diff --git a/.gitmodules b/.gitmodules index 95053091cf..2ede715f49 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "cmake"] path = cmake url = git://git.bro-ids.org/cmake +[submodule "magic"] + path = magic + url = git://git.bro.org/bromagic diff --git a/CMakeLists.txt b/CMakeLists.txt index b95b637770..0f64f304b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} ABSOLUTE) set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic) -set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic) +set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database) configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh @@ -201,9 +201,8 @@ CheckOptionalBuildSources(aux/broctl Broctl INSTALL_BROCTL) CheckOptionalBuildSources(aux/bro-aux Bro-Aux INSTALL_AUX_TOOLS) CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI) -install(DIRECTORY ./magic/ DESTINATION ${BRO_MAGIC_INSTALL_PATH} FILES_MATCHING - PATTERN "COPYING" EXCLUDE - PATTERN "*" +install(DIRECTORY ./magic/database/ + DESTINATION ${BRO_MAGIC_INSTALL_PATH} ) ######################################################################## diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index ddb09bb29c..e7e39d0b3f 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -99,7 +99,7 @@ macro(REST_TARGET srcDir broInput) COMMAND "${CMAKE_COMMAND}" ARGS -E remove_directory .state # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro + COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1) # move generated doc into a new directory tree that # defines the final structure of documents @@ -130,7 +130,7 @@ add_custom_command(OUTPUT proto-analyzers.rst COMMAND "${CMAKE_COMMAND}" ARGS -E remove_directory .state # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro + COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1) # move generated doc into a new directory tree that # defines the final structure of documents diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index f507172161..c5b3f7cbbb 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -74,6 +74,8 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro) +rest_target(${psd} base/files/extract/main.bro) +rest_target(${psd} base/files/hash/main.bro) rest_target(${psd} base/frameworks/analyzer/main.bro) rest_target(${psd} base/frameworks/cluster/main.bro) rest_target(${psd} base/frameworks/cluster/nodes/manager.bro) @@ -83,7 +85,7 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro) rest_target(${psd} base/frameworks/communication/main.bro) rest_target(${psd} base/frameworks/control/main.bro) rest_target(${psd} base/frameworks/dpd/main.bro) -rest_target(${psd} base/frameworks/file-analysis/main.bro) +rest_target(${psd} base/frameworks/files/main.bro) rest_target(${psd} base/frameworks/input/main.bro) rest_target(${psd} base/frameworks/input/readers/ascii.bro) rest_target(${psd} base/frameworks/input/readers/benchmark.bro) @@ -137,25 +139,22 @@ rest_target(${psd} base/protocols/conn/main.bro) rest_target(${psd} base/protocols/conn/polling.bro) rest_target(${psd} base/protocols/dns/consts.bro) rest_target(${psd} base/protocols/dns/main.bro) -rest_target(${psd} base/protocols/ftp/file-analysis.bro) -rest_target(${psd} base/protocols/ftp/file-extract.bro) +rest_target(${psd} base/protocols/ftp/files.bro) rest_target(${psd} base/protocols/ftp/gridftp.bro) rest_target(${psd} base/protocols/ftp/main.bro) rest_target(${psd} base/protocols/ftp/utils-commands.bro) -rest_target(${psd} base/protocols/http/file-analysis.bro) -rest_target(${psd} base/protocols/http/file-extract.bro) -rest_target(${psd} base/protocols/http/file-hash.bro) -rest_target(${psd} base/protocols/http/file-ident.bro) +rest_target(${psd} base/protocols/ftp/utils.bro) +rest_target(${psd} base/protocols/http/entities.bro) +rest_target(${psd} base/protocols/http/files.bro) rest_target(${psd} base/protocols/http/main.bro) rest_target(${psd} base/protocols/http/utils.bro) rest_target(${psd} base/protocols/irc/dcc-send.bro) -rest_target(${psd} base/protocols/irc/file-analysis.bro) +rest_target(${psd} base/protocols/irc/files.bro) rest_target(${psd} base/protocols/irc/main.bro) rest_target(${psd} base/protocols/modbus/consts.bro) rest_target(${psd} base/protocols/modbus/main.bro) -rest_target(${psd} base/protocols/smtp/entities-excerpt.bro) rest_target(${psd} base/protocols/smtp/entities.bro) -rest_target(${psd} base/protocols/smtp/file-analysis.bro) +rest_target(${psd} base/protocols/smtp/files.bro) rest_target(${psd} base/protocols/smtp/main.bro) rest_target(${psd} base/protocols/socks/consts.bro) rest_target(${psd} base/protocols/socks/main.bro) @@ -183,6 +182,8 @@ rest_target(${psd} policy/frameworks/control/controllee.bro) rest_target(${psd} policy/frameworks/control/controller.bro) rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro) rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro) +rest_target(${psd} policy/frameworks/files/detect-MHR.bro) +rest_target(${psd} policy/frameworks/files/hash-all-files.bro) rest_target(${psd} policy/frameworks/intel/do_notice.bro) rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro) rest_target(${psd} policy/frameworks/intel/seen/dns.bro) @@ -216,7 +217,6 @@ rest_target(${psd} policy/protocols/dns/detect-external-names.bro) rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro) rest_target(${psd} policy/protocols/ftp/detect.bro) rest_target(${psd} policy/protocols/ftp/software.bro) -rest_target(${psd} policy/protocols/http/detect-MHR.bro) rest_target(${psd} policy/protocols/http/detect-sqli.bro) rest_target(${psd} policy/protocols/http/detect-webapps.bro) rest_target(${psd} policy/protocols/http/header-names.bro) @@ -228,6 +228,7 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro) rest_target(${psd} policy/protocols/modbus/track-memmap.bro) rest_target(${psd} policy/protocols/smtp/blocklists.bro) rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro) +rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro) rest_target(${psd} policy/protocols/smtp/software.bro) rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro) rest_target(${psd} policy/protocols/ssh/geo-data.bro) diff --git a/magic/COPYING b/magic/COPYING deleted file mode 100644 index 7d2bf1e711..0000000000 --- a/magic/COPYING +++ /dev/null @@ -1,29 +0,0 @@ -# $File: LEGAL.NOTICE,v 1.15 2006/05/03 18:48:33 christos Exp $ -# Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995. -# Software written by Ian F. Darwin and others; -# maintained 1994- Christos Zoulas. -# -# This software is not subject to any export provision of the United States -# Department of Commerce, and may be exported to any country or planet. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice immediately at the beginning of the file, without modification, -# this list of conditions, and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. diff --git a/magic/animation b/magic/animation deleted file mode 100644 index 0cec03d511..0000000000 --- a/magic/animation +++ /dev/null @@ -1,208 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: animation,v 1.47 2013/02/06 14:18:52 christos Exp $ -# animation: file(1) magic for animation/movie formats -# -# animation formats -# MPEG, FLI, DL originally from vax@ccwf.cc.utexas.edu (VaX#n8) -# FLC, SGI, Apple originally from Daniel Quinlan (quinlan@yggdrasil.com) - -# SGI and Apple formats -0 string MOVI Silicon Graphics movie file -!:mime video/x-sgi-movie -4 string moov Apple QuickTime -!:mime video/quicktime -4 string mdat Apple QuickTime movie (unoptimized) -!:mime video/quicktime -#4 string wide Apple QuickTime movie (unoptimized) -#!:mime video/quicktime -#4 string skip Apple QuickTime movie (modified) -#!:mime video/quicktime -#4 string free Apple QuickTime movie (modified) -#!:mime video/quicktime -4 string idsc Apple QuickTime image (fast start) -!:mime image/x-quicktime -#4 string idat Apple QuickTime image (unoptimized) -#!:mime image/x-quicktime -4 string pckg Apple QuickTime compressed archive -!:mime application/x-quicktime-player -4 string/W jP JPEG 2000 image -!:mime image/jp2 -4 string ftyp ISO Media ->8 string isom \b, MPEG v4 system, version 1 -!:mime video/mp4 ->8 string mp41 \b, MPEG v4 system, version 1 -!:mime video/mp4 ->8 string mp42 \b, MPEG v4 system, version 2 -!:mime video/mp4 ->8 string/W jp2 \b, JPEG 2000 -!:mime image/jp2 ->8 string 3ge \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3gg \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3gp \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3gs \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3g2 \b, MPEG v4 system, 3GPP2 -!:mime video/3gpp2 ->8 string mmp4 \b, MPEG v4 system, 3GPP Mobile -!:mime video/mp4 ->8 string avc1 \b, MPEG v4 system, 3GPP JVT AVC -!:mime video/3gpp ->8 string/W M4A \b, MPEG v4 system, iTunes AAC-LC -!:mime audio/mp4 ->8 string/W M4V \b, MPEG v4 system, iTunes AVC-LC -!:mime video/mp4 ->8 string/W qt \b, Apple QuickTime movie -!:mime video/quicktime - -# MPEG sequences -# Scans for all common MPEG header start codes -0 belong&0xFFFFFF00 0x00000100 ->3 byte 0xBA MPEG sequence -!:mime video/mpeg -# GRR too general as it catches also FoxPro Memo example NG.FPT ->3 byte 0xB0 MPEG sequence, v4 -!:mime video/mpeg4-generic ->3 byte 0xB5 MPEG sequence, v4 -!:mime video/mpeg4-generic ->3 byte 0xB3 MPEG sequence -!:mime video/mpeg - -# MPEG ADTS Audio (*.mpx/mxa/aac) -# from dreesen@math.fu-berlin.de -# modified to fully support MPEG ADTS - -# MP3, M1A -# modified by Joerg Jenderek -# GRR the original test are too common for many DOS files -# so don't accept as MP3 until we've tested the rate -0 beshort&0xFFFE 0xFFFA -# rates ->2 byte&0xF0 0x10 MPEG ADTS, layer III, v1, 32 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x20 MPEG ADTS, layer III, v1, 40 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x30 MPEG ADTS, layer III, v1, 48 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x40 MPEG ADTS, layer III, v1, 56 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x50 MPEG ADTS, layer III, v1, 64 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x60 MPEG ADTS, layer III, v1, 80 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x70 MPEG ADTS, layer III, v1, 96 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x80 MPEG ADTS, layer III, v1, 112 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x90 MPEG ADTS, layer III, v1, 128 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xA0 MPEG ADTS, layer III, v1, 160 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xB0 MPEG ADTS, layer III, v1, 192 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xC0 MPEG ADTS, layer III, v1, 224 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xD0 MPEG ADTS, layer III, v1, 256 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xE0 MPEG ADTS, layer III, v1, 320 kbps -!:mime audio/mpeg - -# MP2, M1A -0 beshort&0xFFFE 0xFFFC MPEG ADTS, layer II, v1 -!:mime audio/mpeg - -# MP3, M2A -0 beshort&0xFFFE 0xFFF2 MPEG ADTS, layer III, v2 -!:mime audio/mpeg - -# MPA, M2A -0 beshort&0xFFFE 0xFFF6 MPEG ADTS, layer I, v2 -!:mime audio/mpeg - -# MP3, M25A -0 beshort&0xFFFE 0xFFE2 MPEG ADTS, layer III, v2.5 -!:mime audio/mpeg - -# Stored AAC streams (instead of the MP4 format) -0 string ADIF MPEG ADIF, AAC -!:mime audio/x-hx-aac-adif - -# Live or stored single AAC stream (used with MPEG-2 systems) -0 beshort&0xFFF6 0xFFF0 MPEG ADTS, AAC -!:mime audio/x-hx-aac-adts - -# Live MPEG-4 audio streams (instead of RTP FlexMux) -0 beshort&0xFFE0 0x56E0 MPEG-4 LOAS -!:mime audio/x-mp4a-latm - -# This magic isn't strong enough (matches plausible ISO-8859-1 text) -#0 beshort 0x4DE1 MPEG-4 LO-EP audio stream -#!:mime audio/x-mp4a-latm - -# Summary: FLI animation format -# Created by: Daniel Quinlan -# Modified by (1): Abel Cheung (avoid over-generic detection) -4 leshort 0xAF11 -# standard FLI always has 320x200 resolution and 8 bit color ->8 leshort 320 ->>10 leshort 200 ->>>12 leshort 8 FLI animation, 320x200x8 -!:mime video/x-fli - -# Summary: FLC animation format -# Created by: Daniel Quinlan -# Modified by (1): Abel Cheung (avoid over-generic detection) -4 leshort 0xAF12 -# standard FLC always use 8 bit color ->12 leshort 8 FLC animation -!:mime video/x-flc - -# Microsoft Advanced Streaming Format (ASF) -0 belong 0x3026b275 Microsoft ASF -!:mime video/x-ms-asf - -# MNG Video Format, -0 string \x8aMNG MNG video data, -!:mime video/x-mng - -# JNG Video Format, -0 string \x8bJNG JNG video data, -!:mime video/x-jng - -# VRML (Virtual Reality Modelling Language) -0 string/w #VRML\ V1.0\ ascii VRML 1 file -!:mime model/vrml -0 string/w #VRML\ V2.0\ utf8 ISO/IEC 14772 VRML 97 file -!:mime model/vrml - -# X3D (Extensible 3D) [http://www.web3d.org/specifications/x3d-3.0.dtd] -# From Michel Briand -0 string/t \20 search/1000/cw \4 byte &0x40 -!:mime video/mp2p ->4 byte ^0x40 -!:mime video/mpeg -0 belong 0x000001BB -!:mime video/mpeg -0 belong 0x000001B0 -!:mime video/mp4v-es -0 belong 0x000001B5 -!:mime video/mp4v-es -0 belong 0x000001B3 -!:mime video/mpv -0 belong&0xFF5FFF1F 0x47400010 -!:mime video/mp2t -0 belong 0x00000001 ->4 byte&0x1F 0x07 -!:mime video/h264 diff --git a/magic/archive b/magic/archive deleted file mode 100644 index 35cbef4012..0000000000 --- a/magic/archive +++ /dev/null @@ -1,242 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: archive,v 1.78 2013/02/06 14:18:52 christos Exp $ -# archive: file(1) magic for archive formats (see also "msdos" for self- -# extracting compressed archives) -# -# cpio, ar, arc, arj, hpack, lha/lharc, rar, squish, uc2, zip, zoo, etc. -# pre-POSIX "tar" archives are handled in the C code. - -# POSIX tar archives -257 string ustar\0 POSIX tar archive -!:mime application/x-tar # encoding: posix -257 string ustar\040\040\0 GNU tar archive -!:mime application/x-tar # encoding: gnu - -# cpio archives -# -# Yes, the top two "cpio archive" formats *are* supposed to just be "short". -# The idea is to indicate archives produced on machines with the same -# byte order as the machine running "file" with "cpio archive", and -# to indicate archives produced on machines with the opposite byte order -# from the machine running "file" with "byte-swapped cpio archive". -# -# The SVR4 "cpio(4)" hints that there are additional formats, but they -# are defined as "short"s; I think all the new formats are -# character-header formats and thus are strings, not numbers. -0 short 070707 cpio archive -!:mime application/x-cpio -0 short 0143561 byte-swapped cpio archive -!:mime application/x-cpio # encoding: swapped - -# -# System V Release 1 portable(?) archive format. -# -0 string = System V Release 1 ar archive -!:mime application/x-archive - -# -# Debian package; it's in the portable archive format, and needs to go -# before the entry for regular portable archives, as it's recognized as -# a portable archive whose first member has a name beginning with -# "debian". -# -0 string =!\ndebian -!:mime application/x-debian-package - -# -# MIPS archive; they're in the portable archive format, and need to go -# before the entry for regular portable archives, as it's recognized as -# a portable archive whose first member has a name beginning with -# "__________E". -# -0 string =!\n__________E MIPS archive -!:mime application/x-archive - -# -# BSD/SVR2-and-later portable archive formats. -# -0 string =! current ar archive -!:mime application/x-archive - -# ARC archiver, from Daniel Quinlan (quinlan@yggdrasil.com) -# -# The first byte is the magic (0x1a), byte 2 is the compression type for -# the first file (0x01 through 0x09), and bytes 3 to 15 are the MS-DOS -# filename of the first file (null terminated). Since some types collide -# we only test some types on basis of frequency: 0x08 (83%), 0x09 (5%), -# 0x02 (5%), 0x03 (3%), 0x04 (2%), 0x06 (2%). 0x01 collides with terminfo. -0 lelong&0x8080ffff 0x0000081a ARC archive data, dynamic LZW -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000091a ARC archive data, squashed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000021a ARC archive data, uncompressed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000031a ARC archive data, packed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000041a ARC archive data, squeezed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000061a ARC archive data, crunched -!:mime application/x-arc -# [JW] stuff taken from idarc, obviously ARC successors: -0 lelong&0x8080ffff 0x00000a1a PAK archive data -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000141a ARC+ archive data -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000481a HYP archive data -!:mime application/x-arc - -# ARJ archiver (jason@jarthur.Claremont.EDU) -0 leshort 0xea60 ARJ archive data -!:mime application/x-arj - -# LHARC/LHA archiver (Greg Roelofs, newt@uchicago.edu) -2 string -lh0- LHarc 1.x/ARX archive data [lh0] -!:mime application/x-lharc -2 string -lh1- LHarc 1.x/ARX archive data [lh1] -!:mime application/x-lharc -2 string -lz4- LHarc 1.x archive data [lz4] -!:mime application/x-lharc -2 string -lz5- LHarc 1.x archive data [lz5] -!:mime application/x-lharc -# [never seen any but the last; -lh4- reported in comp.compression:] -2 string -lzs- LHa/LZS archive data [lzs] -!:mime application/x-lha -2 string -lh\40- LHa 2.x? archive data [lh ] -!:mime application/x-lha -2 string -lhd- LHa 2.x? archive data [lhd] -!:mime application/x-lha -2 string -lh2- LHa 2.x? archive data [lh2] -!:mime application/x-lha -2 string -lh3- LHa 2.x? archive data [lh3] -!:mime application/x-lha -2 string -lh4- LHa (2.x) archive data [lh4] -!:mime application/x-lha -2 string -lh5- LHa (2.x) archive data [lh5] -!:mime application/x-lha -2 string -lh6- LHa (2.x) archive data [lh6] -!:mime application/x-lha -2 string -lh7- LHa (2.x)/LHark archive data [lh7] -!:mime application/x-lha - -# RAR archiver (Greg Roelofs, newt@uchicago.edu) -0 string Rar! RAR archive data, -!:mime application/x-rar - -# PKZIP multi-volume archive -0 string PK\x07\x08PK\x03\x04 Zip multi-volume archive data, at least PKZIP v2.50 to extract -!:mime application/zip - -# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu) -0 string PK\003\004 - -# Specialised zip formats which start with a member named 'mimetype' -# (stored uncompressed, with no 'extra field') containing the file's MIME type. -# Check for have 8-byte name, 0-byte extra field, name "mimetype", and -# contents starting with "application/": ->26 string \x8\0\0\0mimetypeapplication/ - -# OpenDocument formats (for OpenOffice 2.x / StarOffice >= 8) -# http://lists.oasis-open.org/archives/office/200505/msg00006.html -# (mimetype contains "application/vnd.oasis.opendocument.") ->>50 string vnd.oasis.opendocument. OpenDocument ->>>73 string text ->>>>77 byte !0x2d Text -!:mime application/vnd.oasis.opendocument.text ->>>>77 string -template Text Template -!:mime application/vnd.oasis.opendocument.text-template ->>>>77 string -web HTML Document Template -!:mime application/vnd.oasis.opendocument.text-web ->>>>77 string -master Master Document -!:mime application/vnd.oasis.opendocument.text-master ->>>73 string graphics ->>>>81 byte !0x2d Drawing -!:mime application/vnd.oasis.opendocument.graphics ->>>>81 string -template Template -!:mime application/vnd.oasis.opendocument.graphics-template ->>>73 string presentation ->>>>85 byte !0x2d Presentation -!:mime application/vnd.oasis.opendocument.presentation ->>>>85 string -template Template -!:mime application/vnd.oasis.opendocument.presentation-template ->>>73 string spreadsheet ->>>>84 byte !0x2d Spreadsheet -!:mime application/vnd.oasis.opendocument.spreadsheet ->>>>84 string -template Template -!:mime application/vnd.oasis.opendocument.spreadsheet-template ->>>73 string chart ->>>>78 byte !0x2d Chart -!:mime application/vnd.oasis.opendocument.chart ->>>>78 string -template Template -!:mime application/vnd.oasis.opendocument.chart-template ->>>73 string formula ->>>>80 byte !0x2d Formula -!:mime application/vnd.oasis.opendocument.formula ->>>>80 string -template Template -!:mime application/vnd.oasis.opendocument.formula-template ->>>73 string database Database -!:mime application/vnd.oasis.opendocument.database ->>>73 string image ->>>>78 byte !0x2d Image -!:mime application/vnd.oasis.opendocument.image ->>>>78 string -template Template -!:mime application/vnd.oasis.opendocument.image-template - -# EPUB (OEBPS) books using OCF (OEBPS Container Format) -# http://www.idpf.org/ocf/ocf1.0/download/ocf10.htm, section 4. -# From: Ralf Brown ->0x1E string mimetypeapplication/epub+zip EPUB document -!:mime application/epub+zip - -# Catch other ZIP-with-mimetype formats -# In a ZIP file, the bytes immediately after a member's contents are -# always "PK". The 2 regex rules here print the "mimetype" member's -# contents up to the first 'P'. Luckily, most MIME types don't contain -# any capital 'P's. This is a kludge. -# (mimetype contains "application/") ->>50 string !epub+zip ->>>50 string !vnd.oasis.opendocument. ->>>>50 string !vnd.sun.xml. ->>>>>50 string !vnd.kde. ->>>>>>38 regex [!-OQ-~]+ Zip data (MIME type "%s"?) -!:mime application/zip -# (mimetype contents other than "application/*") ->26 string \x8\0\0\0mimetype ->>38 string !application/ ->>>38 regex [!-OQ-~]+ Zip data (MIME type "%s"?) -!:mime application/zip - -# Java Jar files ->(26.s+30) leshort 0xcafe Java Jar file data (zip) -!:mime application/jar - -# Generic zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu) -# Next line excludes specialized formats: ->(26.s+30) leshort !0xcafe ->>26 string !\x8\0\0\0mimetype Zip archive data -!:mime application/zip - -# Zoo archiver -20 lelong 0xfdc4a7dc Zoo archive data -!:mime application/x-zoo - -# Shell archives -10 string #\ This\ is\ a\ shell\ archive shell archive text -!:mime application/octet-stream - -# Felix von Leitner -0 string d8:announce BitTorrent file -!:mime application/x-bittorrent - -# EET archive -# From: Tilman Sauerbeck -0 belong 0x1ee7ff00 EET archive -!:mime application/x-eet - -# Symbian installation files -# http://www.thouky.co.uk/software/psifs/sis.html -# http://developer.symbian.com/main/downloads/papers/SymbianOSv91/softwareinstallsis.pdf -8 lelong 0x10000419 Symbian installation file -!:mime application/vnd.symbian.install -0 lelong 0x10201A7A Symbian installation file (Symbian OS 9.x) -!:mime x-epoc/x-sisx-app diff --git a/magic/assembler b/magic/assembler deleted file mode 100644 index 242b6e19e2..0000000000 --- a/magic/assembler +++ /dev/null @@ -1,19 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: assembler,v 1.3 2013/01/04 17:23:28 christos Exp $ -# make: file(1) magic for assembler source -# -0 regex \^[\020\t]*\\.asciiz assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.byte assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.even assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.globl assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.text assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.file assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.type assembler source text -!:mime text/x-asm diff --git a/magic/audio b/magic/audio deleted file mode 100644 index 75a9dc536c..0000000000 --- a/magic/audio +++ /dev/null @@ -1,149 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: audio,v 1.65 2012/10/31 13:38:40 christos Exp $ -# audio: file(1) magic for sound formats (see also "iff") -# -# Jan Nicolai Langfeldt (janl@ifi.uio.no), Dan Quinlan (quinlan@yggdrasil.com), -# and others -# - -# Sun/NeXT audio data -0 string .snd Sun/NeXT audio data: ->12 belong 1 8-bit ISDN mu-law, -!:mime audio/basic ->12 belong 2 8-bit linear PCM [REF-PCM], -!:mime audio/basic ->12 belong 3 16-bit linear PCM, -!:mime audio/basic ->12 belong 4 24-bit linear PCM, -!:mime audio/basic ->12 belong 5 32-bit linear PCM, -!:mime audio/basic ->12 belong 6 32-bit IEEE floating point, -!:mime audio/basic ->12 belong 7 64-bit IEEE floating point, -!:mime audio/basic ->12 belong 23 8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.), -!:mime audio/x-adpcm - -# DEC systems (e.g. DECstation 5000) use a variant of the Sun/NeXT format -# that uses little-endian encoding and has a different magic number -0 lelong 0x0064732E DEC audio data: ->12 lelong 1 8-bit ISDN mu-law, -!:mime audio/x-dec-basic ->12 lelong 2 8-bit linear PCM [REF-PCM], -!:mime audio/x-dec-basic ->12 lelong 3 16-bit linear PCM, -!:mime audio/x-dec-basic ->12 lelong 4 24-bit linear PCM, -!:mime audio/x-dec-basic ->12 lelong 5 32-bit linear PCM, -!:mime audio/x-dec-basic ->12 lelong 6 32-bit IEEE floating point, -!:mime audio/x-dec-basic ->12 lelong 7 64-bit IEEE floating point, -!:mime audio/x-dec-basic ->12 lelong 23 8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.), -!:mime audio/x-dec-basic - -# Creative Labs AUDIO stuff -0 string MThd Standard MIDI data -!:mime audio/midi - -0 string CTMF Creative Music (CMF) data -!:mime audio/x-unknown -0 string SBI SoundBlaster instrument data -!:mime audio/x-unknown -0 string Creative\ Voice\ File Creative Labs voice data -!:mime audio/x-unknown - -# Real Audio (Magic .ra\0375) -0 belong 0x2e7261fd RealAudio sound file -!:mime audio/x-pn-realaudio -0 string .RMF\0\0\0 RealMedia file -!:mime application/vnd.rn-realmedia - -# mime types according to http://www.geocities.com/nevilo/mod.htm: -# audio/it .it -# audio/x-zipped-it .itz -# audio/xm fasttracker modules -# audio/x-s3m screamtracker modules -# audio/s3m screamtracker modules -# audio/x-zipped-mod mdz -# audio/mod mod -# audio/x-mod All modules (mod, s3m, 669, mtm, med, xm, it, mdz, stm, itz, xmz, s3z) - -# -# Taken from loader code from mikmod version 2.14 -# by Steve McIntyre (stevem@chiark.greenend.org.uk) -# added title printing on 2003-06-24 -0 string MAS_UTrack_V00 ->14 string >/0 ultratracker V1.%.1s module sound data -!:mime audio/x-mod -#audio/x-tracker-module - -0 string Extended\ Module: Fasttracker II module sound data -!:mime audio/x-mod -#audio/x-tracker-module - -21 string/c =!SCREAM! Screamtracker 2 module sound data -!:mime audio/x-mod -#audio/x-screamtracker-module -21 string BMOD2STM Screamtracker 2 module sound data -!:mime audio/x-mod -#audio/x-screamtracker-module -1080 string M.K. 4-channel Protracker module sound data -!:mime audio/x-mod -#audio/x-protracker-module -1080 string M!K! 4-channel Protracker module sound data -!:mime audio/x-mod -#audio/x-protracker-module -1080 string FLT4 4-channel Startracker module sound data -!:mime audio/x-mod -#audio/x-startracker-module -1080 string FLT8 8-channel Startracker module sound data -!:mime audio/x-mod -#audio/x-startracker-module -1080 string 4CHN 4-channel Fasttracker module sound data -!:mime audio/x-mod -#audio/x-fasttracker-module -1080 string 6CHN 6-channel Fasttracker module sound data -!:mime audio/x-mod -#audio/x-fasttracker-module -1080 string 8CHN 8-channel Fasttracker module sound data -!:mime audio/x-mod -#audio/x-fasttracker-module -1080 string CD81 8-channel Octalyser module sound data -!:mime audio/x-mod -#audio/x-octalysertracker-module -1080 string OKTA 8-channel Octalyzer module sound data -!:mime audio/x-mod -#audio/x-octalysertracker-module -# Not good enough. -#1082 string CH -#>1080 string >/0 %.2s-channel Fasttracker "oktalyzer" module sound data -1080 string 16CN 16-channel Taketracker module sound data -!:mime audio/x-mod -#audio/x-taketracker-module -1080 string 32CN 32-channel Taketracker module sound data -!:mime audio/x-mod -#audio/x-taketracker-module - -# Impulse tracker module (audio/x-it) -0 string IMPM Impulse Tracker module sound data - -!:mime audio/x-mod - -# Free lossless audio codec -# From: Przemyslaw Augustyniak -0 string fLaC FLAC audio bitstream data -!:mime audio/x-flac - -# Monkey's Audio compressed audio format (.ape) -# From danny.milo@gmx.net (Danny Milosavljevic) -# New version from Abel Cheung -0 string MAC\040 Monkey's Audio compressed format -!:mime audio/x-ape - -# musepak support From: "Jiri Pejchal" -0 string MP+ Musepack audio -!:mime audio/x-musepack diff --git a/magic/c-lang b/magic/c-lang deleted file mode 100644 index 525dc6b599..0000000000 --- a/magic/c-lang +++ /dev/null @@ -1,47 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: c-lang,v 1.16 2011/12/09 08:02:16 rrt Exp $ -# c-lang: file(1) magic for C and related languages programs -# - -# BCPL -0 search/8192 "libhdr" BCPL source text -!:mime text/x-bcpl -0 search/8192 "LIBHDR" BCPL source text -!:mime text/x-bcpl - -# C -0 regex \^#include C source text -!:mime text/x-c -0 regex \^char C source text -!:mime text/x-c -0 regex \^double C source text -!:mime text/x-c -0 regex \^extern C source text -!:mime text/x-c -0 regex \^float C source text -!:mime text/x-c -0 regex \^struct C source text -!:mime text/x-c -0 regex \^union C source text -!:mime text/x-c -0 search/8192 main( C source text -!:mime text/x-c - -# C++ -# The strength of these rules is increased so they beat the C rules above -0 regex \^template C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^virtual C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^class C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^public: C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^private: C++ source text -!:strength + 5 -!:mime text/x-c++ diff --git a/magic/cafebabe b/magic/cafebabe deleted file mode 100644 index 29fefd5f1e..0000000000 --- a/magic/cafebabe +++ /dev/null @@ -1,31 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: cafebabe,v 1.13 2013/02/26 21:04:38 christos Exp $ -# Cafe Babes unite! -# -# Since Java bytecode and Mach-O universal binaries have the same magic number, -# the test must be performed in the same "magic" sequence to get both right. -# The long at offset 4 in a Mach-O universal binary tells the number of -# architectures; the short at offset 4 in a Java bytecode file is the JVM minor -# version and the short at offset 6 is the JVM major version. Since there are only -# only 18 labeled Mach-O architectures at current, and the first released -# Java class format was version 43.0, we can safely choose any number -# between 18 and 39 to test the number of architectures against -# (and use as a hack). Let's not use 18, because the Mach-O people -# might add another one or two as time goes by... -# -### JAVA START ### -0 belong 0xcafebabe -!:mime application/x-java-applet - -0 belong 0xcafed00d JAR compressed with pack200, ->5 byte x version %d. ->4 byte x \b%d -!:mime application/x-java-pack200 - -0 belong 0xcafed00d JAR compressed with pack200, ->5 byte x version %d. ->4 byte x \b%d -!:mime application/x-java-pack200 - -### JAVA END ### diff --git a/magic/commands b/magic/commands deleted file mode 100644 index 6ad7699c5e..0000000000 --- a/magic/commands +++ /dev/null @@ -1,82 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: commands,v 1.44 2013/02/05 15:20:47 christos Exp $ -# commands: file(1) magic for various shells and interpreters -# -#0 string/w : shell archive or script for antique kernel text -0 string/wt #!\ /bin/sh POSIX shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /bin/csh C shell script text executable -!:mime text/x-shellscript -# korn shell magic, sent by George Wu, gwu@clyde.att.com -0 string/wt #!\ /bin/ksh Korn shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /bin/tcsh Tenex C shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/bin/tcsh Tenex C shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/tcsh Tenex C shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/tcsh Tenex C shell script text executable -!:mime text/x-shellscript - -# -# zsh/ash/ae/nawk/gawk magic from cameron@cs.unsw.oz.au (Cameron Simpson) -0 string/wt #!\ /bin/zsh Paul Falstad's zsh script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/bin/zsh Paul Falstad's zsh script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/zsh Paul Falstad's zsh script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/ash Neil Brown's ash script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/ae Neil Brown's ae script text executable -!:mime text/x-shellscript -0 string/wt #!\ /bin/nawk new awk script text executable -!:mime text/x-nawk -0 string/wt #!\ /usr/bin/nawk new awk script text executable -!:mime text/x-nawk -0 string/wt #!\ /usr/local/bin/nawk new awk script text executable -!:mime text/x-nawk -0 string/wt #!\ /bin/gawk GNU awk script text executable -!:mime text/x-gawk -0 string/wt #!\ /usr/bin/gawk GNU awk script text executable -!:mime text/x-gawk -0 string/wt #!\ /usr/local/bin/gawk GNU awk script text executable -!:mime text/x-gawk -# -0 string/wt #!\ /bin/awk awk script text executable -!:mime text/x-awk -0 string/wt #!\ /usr/bin/awk awk script text executable -!:mime text/x-awk - -# bash shell magic, from Peter Tobias (tobias@server.et-inf.fho-emden.de) -0 string/wt #!\ /bin/bash Bourne-Again shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/bin/bash Bourne-Again shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bash Bourne-Again shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/bash Bourne-Again shell script text executable -!:mime text/x-shellscript - -# PHP scripts -# Ulf Harnhammar -0 search/1/c = -0 string =24 regex [0-9.]+ \b, version %s -!:mime text/x-php diff --git a/magic/compress b/magic/compress deleted file mode 100644 index f2598b783f..0000000000 --- a/magic/compress +++ /dev/null @@ -1,77 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: compress,v 1.48 2011/12/07 18:39:43 christos Exp $ -# compress: file(1) magic for pure-compression formats (no archives) -# -# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc. -# -# Formats for various forms of compressed data -# Formats for "compress" proper have been moved into "compress.c", -# because it tries to uncompress it to figure out what's inside. - -# standard unix compress -0 string \037\235 compress'd data -!:mime application/x-compress -!:apple LZIVZIVU - -# gzip (GNU zip, not to be confused with Info-ZIP or PKWARE zip archiver) -# Edited by Chris Chittleborough , March 2002 -# * Original filename is only at offset 10 if "extra field" absent -# * Produce shorter output - notably, only report compression methods -# other than 8 ("deflate", the only method defined in RFC 1952). -0 string \037\213 gzip compressed data -!:mime application/x-gzip - -# packed data, Huffman (minimum redundancy) codes on a byte-by-byte basis -0 string \037\036 packed data -!:mime application/octet-stream - -# -# This magic number is byte-order-independent. -0 short 0x1f1f old packed data -!:mime application/octet-stream - -# XXX - why *two* entries for "compacted data", one of which is -# byte-order independent, and one of which is byte-order dependent? -# -0 short 0x1fff compacted data -!:mime application/octet-stream -# This string is valid for SunOS (BE) and a matching "short" is listed -# in the Ultrix (LE) magic file. -0 string \377\037 compacted data -!:mime application/octet-stream -0 short 0145405 huf output -!:mime application/octet-stream - -# bzip2 -0 string BZh bzip2 compressed data -!:mime application/x-bzip2 - -# lzip -0 string LZIP lzip compressed data -!:mime application/x-lzip - -# 7-zip archiver, from Thomas Klausner (wiz@danbala.tuwien.ac.at) -# http://www.7-zip.org or DOC/7zFormat.txt -# -0 string 7z\274\257\047\034 7-zip archive data, ->6 byte x version %d ->7 byte x \b.%d -!:mime application/x-7z-compressed - -# Type: LZMA -0 lelong&0xffffff =0x5d ->12 leshort =0xff LZMA compressed data, ->>5 lequad =0xffffffffffffffff streamed ->>5 lequad !0xffffffffffffffff non-streamed, size %lld -!:mime application/x-lzma - -# http://tukaani.org/xz/xz-file-format.txt -0 ustring \xFD7zXZ\x00 XZ compressed data -!:mime application/x-xz - -# https://github.com/ckolivas/lrzip/blob/master/doc/magic.header.txt -0 string LRZI LRZIP compressed data ->4 byte x - version %d ->5 byte x \b.%d -!:mime application/x-lrzip diff --git a/magic/database b/magic/database deleted file mode 100644 index f1c09c0629..0000000000 --- a/magic/database +++ /dev/null @@ -1,47 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: database,v 1.32 2013/02/06 14:18:52 christos Exp $ -# database: file(1) magic for various databases -# -# extracted from header/code files by Graeme Wilford (eep2gw@ee.surrey.ac.uk) -# -# -# GDBM magic numbers -# Will be maintained as part of the GDBM distribution in the future. -# -0 belong 0x13579ace GNU dbm 1.x or ndbm database, big endian -!:mime application/x-gdbm -0 lelong 0x13579ace GNU dbm 1.x or ndbm database, little endian -!:mime application/x-gdbm -0 string GDBM GNU dbm 2.x database -!:mime application/x-gdbm -# -# Berkeley DB -# -# Ian Darwin's file /etc/magic files: big/little-endian version. -# -# Hash 1.85/1.86 databases store metadata in network byte order. -# Btree 1.85/1.86 databases store the metadata in host byte order. -# Hash and Btree 2.X and later databases store the metadata in host byte order. - -0 long 0x00061561 Berkeley DB -!:mime application/x-dbm - -# MS Access database -4 string Standard\ Jet\ DB Microsoft Access Database -!:mime application/x-msaccess -4 string Standard\ ACE\ DB Microsoft Access Database -!:mime application/x-msaccess - -# Tokyo Cabinet magic data -# http://tokyocabinet.sourceforge.net/index.html -0 string ToKyO\ CaBiNeT\n Tokyo Cabinet ->14 string x \b (%s) ->32 byte 0 \b, Hash -!:mime application/x-tokyocabinet-hash ->32 byte 1 \b, B+ tree -!:mime application/x-tokyocabinet-btree ->32 byte 2 \b, Fixed-length -!:mime application/x-tokyocabinet-fixed ->32 byte 3 \b, Table -!:mime application/x-tokyocabinet-table diff --git a/magic/diff b/magic/diff deleted file mode 100644 index b6504f17a0..0000000000 --- a/magic/diff +++ /dev/null @@ -1,25 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: diff,v 1.13 2012/06/16 14:43:36 christos Exp $ -# diff: file(1) magic for diff(1) output -# -0 search/1 diff\ diff output text -!:mime text/x-diff -0 search/1 ***\ diff output text -!:mime text/x-diff -0 search/1 Only\ in\ diff output text -!:mime text/x-diff -0 search/1 Common\ subdirectories:\ diff output text -!:mime text/x-diff - -0 search/1 Index: RCS/CVS diff output text -!:mime text/x-diff - -# unified diff -0 search/4096 ---\ ->&0 search/1024 \n ->>&0 search/1 +++\ ->>>&0 search/1024 \n ->>>>&0 search/1 @@ unified diff output text -!:mime text/x-diff -!:strength + 90 diff --git a/magic/elf b/magic/elf deleted file mode 100644 index aaf80cf10e..0000000000 --- a/magic/elf +++ /dev/null @@ -1,43 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# elf: file(1) magic for ELF executables -# -# We have to check the byte order flag to see what byte order all the -# other stuff in the header is in. -# -# What're the correct byte orders for the nCUBE and the Fujitsu VPP500? -# -# Created by: unknown -# Modified by (1): Daniel Quinlan -# Modified by (2): Peter Tobias (core support) -# Modified by (3): Christian 'Dr. Disk' Hechelmann (fix of core support) -# Modified by (4): (VMS Itanium) -# Modified by (5): Matthias Urlichs (Listing of many architectures) -0 string \177ELF ELF ->4 byte 0 invalid class ->4 byte 1 32-bit ->4 byte 2 64-bit ->5 byte 0 invalid byte order ->5 byte 1 LSB ->>16 leshort 0 no file type, -!:strength *2 -!:mime application/octet-stream ->>16 leshort 1 relocatable, -!:mime application/x-object ->>16 leshort 2 executable, -!:mime application/x-executable ->>16 leshort 3 shared object, -!:mime application/x-sharedlib ->>16 leshort 4 core file -!:mime application/x-coredump ->5 byte 2 MSB ->>16 beshort 0 no file type, -!:mime application/octet-stream ->>16 beshort 1 relocatable, -!:mime application/x-object ->>16 beshort 2 executable, -!:mime application/x-executable ->>16 beshort 3 shared object, -!:mime application/x-sharedlib ->>16 beshort 4 core file, -!:mime application/x-coredump diff --git a/magic/epoc b/magic/epoc deleted file mode 100644 index d7397145fb..0000000000 --- a/magic/epoc +++ /dev/null @@ -1,34 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: epoc,v 1.7 2009/09/19 16:28:09 christos Exp $ -# EPOC : file(1) magic for EPOC documents [Psion Series 5/Osaris/Geofox 1] -# Stefan Praszalowicz and Peter Breitenlohner -# Useful information for improving this file can be found at: -# http://software.frodo.looijaard.name/psiconv/formats/Index.html -#------------------------------------------------------------------------------ -0 lelong 0x10000037 Psion Series 5 ->4 lelong 0x10000042 multi-bitmap image -!:mime image/x-epoc-mbm ->4 lelong 0x1000006D ->>8 lelong 0x1000007D Sketch image -!:mime image/x-epoc-sketch ->>8 lelong 0x1000007F Word file -!:mime application/x-epoc-word ->>8 lelong 0x10000085 OPL program (TextEd) -!:mime application/x-epoc-opl ->>8 lelong 0x10000088 Sheet file -!:mime application/x-epoc-sheet ->4 lelong 0x10000073 OPO module -!:mime application/x-epoc-opo ->4 lelong 0x10000074 OPL application -!:mime application/x-epoc-app - - -0 lelong 0x10000050 Psion Series 5 ->4 lelong 0x1000006D database ->>8 lelong 0x10000084 Agenda file -!:mime application/x-epoc-agenda ->>8 lelong 0x10000086 Data file -!:mime application/x-epoc-data ->>8 lelong 0x10000CEA Jotter file -!:mime application/x-epoc-jotter diff --git a/magic/filesystems b/magic/filesystems deleted file mode 100644 index d2178296e0..0000000000 --- a/magic/filesystems +++ /dev/null @@ -1,12 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: filesystems,v 1.76 2013/02/18 18:45:41 christos Exp $ -# filesystems: file(1) magic for different filesystems -# - -# CDROM Filesystems -# Modified for UDF by gerardo.cacciari@gmail.com -32769 string CD001 # -!:mime application/x-iso9660-image -37633 string CD001 ISO 9660 CD-ROM filesystem data (raw 2352 byte sectors) -!:mime application/x-iso9660-image diff --git a/magic/flash b/magic/flash deleted file mode 100644 index b64761b12d..0000000000 --- a/magic/flash +++ /dev/null @@ -1,18 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: flash,v 1.8 2009/09/19 16:28:09 christos Exp $ -# flash: file(1) magic for Macromedia Flash file format -# -# See -# -# http://www.macromedia.com/software/flash/open/ -# -0 string FWS Macromedia Flash data, ->3 byte x version %d -!:mime application/x-shockwave-flash -0 string CWS Macromedia Flash data (compressed), -!:mime application/x-shockwave-flash - -# From: Cal Peake -0 string FLV Macromedia Flash Video -!:mime video/x-flv diff --git a/magic/fonts b/magic/fonts deleted file mode 100644 index 8189131d15..0000000000 --- a/magic/fonts +++ /dev/null @@ -1,32 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: fonts,v 1.25 2013/02/06 14:18:52 christos Exp $ -# fonts: file(1) magic for font data -# - -# X11 font files in SNF (Server Natural Format) format -# updated by Joerg Jenderek at Feb 2013 -# http://computer-programming-forum.com/51-perl/8f22fb96d2e34bab.htm -0 belong 00000004 X11 SNF font data, MSB first -#>104 belong 00000004 X11 SNF font data, MSB first -!:mime application/x-font-sfn -# GRR: line below too general as it catches also Xbase index file t3-CHAR.NDX -0 lelong 00000004 ->104 lelong 00000004 X11 SNF font data, LSB first -!:mime application/x-font-sfn - -# True Type fonts -0 string \000\001\000\000\000 TrueType font data -!:mime application/x-font-ttf - -# Opentype font data from Avi Bercovich -0 string OTTO OpenType font data -!:mime application/vnd.ms-opentype - -# Gurkan Sengun , www.linuks.mine.nu -0 string SplineFontDB: Spline Font Database -!:mime application/vnd.font-fontforge-sfd - -# EOT -34 string LP Embedded OpenType (EOT) -!:mime application/vnd.ms-fontobject diff --git a/magic/fortran b/magic/fortran deleted file mode 100644 index 498eeacf8a..0000000000 --- a/magic/fortran +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: fortran,v 1.6 2009/09/19 16:28:09 christos Exp $ -# FORTRAN source -0 regex/100 \^[Cc][\ \t] FORTRAN program -!:mime text/x-fortran -!:strength - 5 diff --git a/magic/frame b/magic/frame deleted file mode 100644 index b42943bfcd..0000000000 --- a/magic/frame +++ /dev/null @@ -1,31 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# frame: file(1) magic for FrameMaker files -# -# This stuff came on a FrameMaker demo tape, most of which is -# copyright, but this file is "published" as witness the following: -# -# Note that this is the Framemaker Maker Interchange Format, not the -# Normal format which would be application/vnd.framemaker. -# -0 string \6 string 3.0 (3.0) -#>6 string 2.0 (2.0) -#>6 string 1.0 (1.0) -0 string \ - -#------------------------------------------------------------------------------ -# XCF: file(1) magic for the XCF image format used in the GIMP developed -# by Spencer Kimball and Peter Mattis -# ('Bucky' LaDieu, nega@vt.edu) - -0 string gimp\ xcf GIMP XCF image data, -!:mime image/x-xcf diff --git a/magic/gnu b/magic/gnu deleted file mode 100644 index bf1f631751..0000000000 --- a/magic/gnu +++ /dev/null @@ -1,23 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: gnu,v 1.13 2012/01/03 17:16:54 christos Exp $ -# gnu: file(1) magic for various GNU tools -# -# GNU nlsutils message catalog file format -# -# GNU message catalog (.mo and .gmo files) - -# GnuPG -# The format is very similar to pgp -# Note: magic.mime had 0x8501 for the next line instead of 0x8502 -0 beshort 0x8502 GPG encrypted data -!:mime text/PGP # encoding: data - -# This magic is not particularly good, as the keyrings don't have true -# magic. Nevertheless, it covers many keyrings. -0 beshort 0x9901 GPG key public ring -!:mime application/x-gnupg-keyring - -# gettext message catalogue -0 regex \^msgid\ GNU gettext message catalogue text -!:mime text/x-po diff --git a/magic/gnumeric b/magic/gnumeric deleted file mode 100644 index b5edca93c1..0000000000 --- a/magic/gnumeric +++ /dev/null @@ -1,8 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# gnumeric: file(1) magic for Gnumeric spreadsheet -# This entry is only semi-helpful, as Gnumeric compresses its files, so -# they will ordinarily reported as "compressed", but at least -z helps -39 string =4 belong x \b, FORM is %d bytes long -# audio formats ->8 string AIFF \b, AIFF audio -!:mime audio/x-aiff ->8 string AIFC \b, AIFF-C compressed audio -!:mime audio/x-aiff ->8 string 8SVX \b, 8SVX 8-bit sampled sound voice -!:mime audio/x-aiff diff --git a/magic/images b/magic/images deleted file mode 100644 index 281aba4706..0000000000 --- a/magic/images +++ /dev/null @@ -1,255 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: images,v 1.80 2013/02/06 14:18:52 christos Exp $ -# images: file(1) magic for image formats (see also "iff", and "c-lang" for -# XPM bitmaps) -# -# originally from jef@helios.ee.lbl.gov (Jef Poskanzer), -# additions by janl@ifi.uio.no as well as others. Jan also suggested -# merging several one- and two-line files into here. -# -# little magic: PCX (first byte is 0x0a) - -# PBMPLUS images -# The next byte following the magic is always whitespace. -# strength is changed to try these patterns before "x86 boot sector" -0 search/1 P1 ->3 regex =[0-9]*\ [0-9]* Netpbm PBM image text ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-bitmap -0 search/1 P2 ->3 regex =[0-9]*\ [0-9]* Netpbm PGM image text ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-greymap -0 search/1 P3 Netpbm PPM image text ->3 regex =[0-9]*\ [0-9]* Netpbm PPM image text ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-pixmap -0 string P4 ->3 regex =[0-9]*\ [0-9]* Netpbm PBM "rawbits" image data ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-bitmap -0 string P5 ->3 regex =[0-9]*\ [0-9]* Netpbm PGM "rawbits" image data ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-greymap -0 string P6 ->3 regex =[0-9]*\ [0-9]* Netpbm PPM "rawbits" image data ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-pixmap -0 string P7 Netpbm PAM image file -!:mime image/x-portable-pixmap - -# NIFF (Navy Interchange File Format, a modification of TIFF) images -# [GRR: this *must* go before TIFF] -0 string IIN1 NIFF image data -!:mime image/x-niff - -# Canon RAW version 1 (CRW) files are a type of Canon Image File Format -# (CIFF) file. These are apparently all little-endian. -# From: Adam Buchbinder -# URL: http://www.sno.phy.queensu.ca/~phil/exiftool/canon_raw.html -0 string II\x1a\0\0\0HEAPCCDR Canon CIFF raw image data -!:mime image/x-canon-crw - -# Canon RAW version 2 (CR2) files are a kind of TIFF with an extra magic -# number. Put this above the TIFF test to make sure we detect them. -# These are apparently all little-endian. -# From: Adam Buchbinder -# URL: http://libopenraw.freedesktop.org/wiki/Canon_CR2 -0 string II\x2a\0\x10\0\0\0CR Canon CR2 raw image data -!:mime image/x-canon-cr2 - -# Tag Image File Format, from Daniel Quinlan (quinlan@yggdrasil.com) -# The second word of TIFF files is the TIFF version number, 42, which has -# never changed. The TIFF specification recommends testing for it. -0 string MM\x00\x2a TIFF image data, big-endian -!:mime image/tiff -0 string II\x2a\x00 TIFF image data, little-endian -!:mime image/tiff - -0 string MM\x00\x2b Big TIFF image data, big-endian -!:mime image/tiff -0 string II\x2b\x00 Big TIFF image data, little-endian -!:mime image/tiff - -# PNG [Portable Network Graphics, or "PNG's Not GIF"] images -# (Greg Roelofs, newt@uchicago.edu) -# (Albert Cahalan, acahalan@cs.uml.edu) -# -# 137 P N G \r \n ^Z \n [4-byte length] H E A D [HEAD data] [HEAD crc] ... -# -0 string \x89PNG\x0d\x0a\x1a\x0a PNG image data -!:mime image/png - -# possible GIF replacements; none yet released! -# (Greg Roelofs, newt@uchicago.edu) -# -# GRR 950115: this was mine ("Zip GIF"): -0 string GIF94z ZIF image (GIF+deflate alpha) -!:mime image/x-unknown -# -# GRR 950115: this is Jeremy Wohl's Free Graphics Format (better): -# -0 string FGF95a FGF image (GIF+deflate beta) -!:mime image/x-unknown -# -# GRR 950115: this is Thomas Boutell's Portable Bitmap Format proposal -# (best; not yet implemented): -# -0 string PBF PBF image (deflate compression) -!:mime image/x-unknown - -# GIF -0 string GIF8 GIF image data -!:mime image/gif -!:apple 8BIMGIFf - -# From: Joerg Jenderek -# most files with the extension .EPA and some with .BMP -0 string \x11\x06 Award BIOS Logo, 136 x 84 -!:mime image/x-award-bioslogo -0 string \x11\x09 Award BIOS Logo, 136 x 126 -!:mime image/x-award-bioslogo -#0 string \x07\x1f BIOS Logo corrupted? -# http://www.blackfiveservices.co.uk/awbmtools.shtml -# http://biosgfx.narod.ru/v3/ -# http://biosgfx.narod.ru/abr-2/ -0 string AWBM ->4 leshort <1981 Award BIOS bitmap -!:mime image/x-award-bmp - -# PC bitmaps (OS/2, Windows BMP files) (Greg Roelofs, newt@uchicago.edu) -0 string BM ->14 leshort 12 PC bitmap, OS/2 1.x format -!:mime image/x-ms-bmp ->14 leshort 64 PC bitmap, OS/2 2.x format -!:mime image/x-ms-bmp ->14 leshort 40 PC bitmap, Windows 3.x format -!:mime image/x-ms-bmp ->14 leshort 128 PC bitmap, Windows NT/2000 format -!:mime image/x-ms-bmp - -# XPM icons (Greg Roelofs, newt@uchicago.edu) -0 search/1 /*\ XPM\ */ X pixmap image text -!:mime image/x-xpmi - -# DICOM medical imaging data -128 string DICM DICOM medical imaging data -!:mime application/dicom - -# XWD - X Window Dump file. -# As described in /usr/X11R6/include/X11/XWDFile.h -# used by the xwd program. -# Bradford Castalia, idaeim, 1/01 -# updated by Adam Buchbinder, 2/09 -# The following assumes version 7 of the format; the first long is the length -# of the header, which is at least 25 4-byte longs, and the one at offset 8 -# is a constant which is always either 1 or 2. Offset 12 is the pixmap depth, -# which is a maximum of 32. -0 belong >100 ->8 belong <3 ->>12 belong <33 ->>>4 belong 7 XWD X Window Dump image data -!:mime image/x-xwindowdump - -# PCX image files -# From: Dan Fandrich -# updated by Joerg Jenderek at Feb 2013 by http://de.wikipedia.org/wiki/PCX -# http://web.archive.org/web/20100206055706/http://www.qzx.com/pc-gpe/pcx.txt -# GRR: original test was still too general as it catches xbase examples T5.DBT,T6.DBT with 0xa000000 -# test for bytes 0x0a,version byte (0,2,3,4,5),compression byte flag(0,1), bit depth (>0) of PCX or T5.DBT,T6.DBT -0 ubelong&0xffF8fe00 0x0a000000 -# for PCX bit depth > 0 ->3 ubyte >0 -# test for valid versions ->>1 ubyte <6 ->>>1 ubyte !1 PCX -!:mime image/x-pcx - -# Adobe Photoshop -# From: Asbjoern Sloth Toennesen -0 string 8BPS Adobe Photoshop Image -!:mime image/vnd.adobe.photoshop - -# Summary: DjVu image / document -# Extension: .djvu -# Reference: http://djvu.org/docs/DjVu3Spec.djvu -# Submitted by: Stephane Loeuillet -# Modified by (1): Abel Cheung -0 string AT&TFORM ->12 string DJVM DjVu multiple page document -!:mime image/vnd.djvu ->12 string DJVU DjVu image or single page document -!:mime image/vnd.djvu ->12 string DJVI DjVu shared document -!:mime image/vnd.djvu ->12 string THUM DjVu page thumbnails -!:mime image/vnd.djvu - -# Originally by Marc Espie -# Modified by Robert Minsk -# http://www.openexr.com/openexrfilelayout.pdf -0 lelong 20000630 OpenEXR image data, -!:mime image/x-exr - -# SMPTE Digital Picture Exchange Format, SMPTE DPX -# -# ANSI/SMPTE 268M-1994, SMPTE Standard for File Format for Digital -# Moving-Picture Exchange (DPX), v1.0, 18 February 1994 -# Robert Minsk -0 string SDPX DPX image data, big-endian, -!:mime image/x-dpx - -#----------------------------------------------------------------------- -# Hierarchical Data Format, used to facilitate scientific data exchange -# specifications at http://hdf.ncsa.uiuc.edu/ -0 belong 0x0e031301 Hierarchical Data Format (version 4) data -!:mime application/x-hdf -0 string \211HDF\r\n\032\n Hierarchical Data Format (version 5) data -!:mime application/x-hdf - -# http://www.cartesianinc.com/Tech/ -0 string CPC\262 Cartesian Perceptual Compression image -!:mime image/x-cpi - - -# Polar Monitor Bitmap (.pmb) used as logo for Polar Electro watches -# From: Markus Heidelberg -0 string/t [BitmapInfo2] Polar Monitor Bitmap text -!:mime image/x-polar-monitor-bitmap - -# Type: Olympus ORF raw images. -# URL: http://libopenraw.freedesktop.org/wiki/Olympus_ORF -# From: Adam Buchbinder -0 string MMOR Olympus ORF raw image data, big-endian -!:mime image/x-olympus-orf -0 string IIRO Olympus ORF raw image data, little-endian -!:mime image/x-olympus-orf -0 string IIRS Olympus ORF raw image data, little-endian -!:mime image/x-olympus-orf - -# Type: Foveon X3F -# URL: http://www.photofo.com/downloads/x3f-raw-format.pdf -# From: Adam Buchbinder -# Note that the MIME type isn't defined anywhere that I can find; if -# there's a canonical type for this format, it should replace this one. -0 string FOVb Foveon X3F raw image data -!:mime image/x-x3f - -# Paint.NET file -# From Adam Buchbinder -0 string PDN3 Paint.NET image data -!:mime image/x-paintnet diff --git a/magic/java b/magic/java deleted file mode 100644 index 481ffec160..0000000000 --- a/magic/java +++ /dev/null @@ -1,16 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------ -# $File: java,v 1.13 2011/12/08 12:12:46 rrt Exp $ -# Java ByteCode and Mach-O binaries (e.g., Mac OS X) use the -# same magic number, 0xcafebabe, so they are both handled -# in the entry called "cafebabe". -#------------------------------------------------------------ - -0 belong 0xfeedfeed Java KeyStore -!:mime application/x-java-keystore -0 belong 0xcececece Java JCE KeyStore -!:mime application/x-java-jce-keystore - -# Java source -0 regex ^import.*;$ Java source -!:mime text/x-java diff --git a/magic/javascript b/magic/javascript deleted file mode 100644 index a1311d0e71..0000000000 --- a/magic/javascript +++ /dev/null @@ -1,17 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: $ -# javascript: magic for javascript and node.js scripts. -# -0 search/1/w #!/bin/node Node.js script text executable -!:mime application/javascript -0 search/1/w #!/usr/bin/node Node.js script text executable -!:mime application/javascript -0 search/1/w #!/bin/nodejs Node.js script text executable -!:mime application/javascript -0 search/1/w #!/usr/bin/nodejs Node.js script text executable -!:mime application/javascript -0 search/1 #!/usr/bin/env\ node Node.js script text executable -!:mime application/javascript -0 search/1 #!/usr/bin/env\ nodejs Node.js script text executable -!:mime application/javascript diff --git a/magic/jpeg b/magic/jpeg deleted file mode 100644 index 55fedae4b4..0000000000 --- a/magic/jpeg +++ /dev/null @@ -1,31 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: jpeg,v 1.18 2012/08/01 12:12:36 christos Exp $ -# JPEG images -# SunOS 5.5.1 had -# -# 0 string \377\330\377\340 JPEG file -# 0 string \377\330\377\356 JPG file -# -# both of which turn into "JPEG image data" here. -# -0 beshort 0xffd8 JPEG image data -!:mime image/jpeg -!:apple 8BIMJPEG -!:strength +2 - -# From: David Santinoli -0 string \x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A JPEG 2000 -# From: Johan van der Knijff -# Added sub-entries for JP2, JPX, JPM and MJ2 formats; added mimetypes -# https://github.com/bitsgalore/jp2kMagic -# -# Now read value of 'Brand' field, which yields a few possibilities: ->20 string \x6a\x70\x32\x20 Part 1 (JP2) -!:mime image/jp2 ->20 string \x6a\x70\x78\x20 Part 2 (JPX) -!:mime image/jpx ->20 string \x6a\x70\x6d\x20 Part 6 (JPM) -!:mime image/jpm ->20 string \x6d\x6a\x70\x32 Part 3 (MJ2) -!:mime video/mj2 diff --git a/magic/kde b/magic/kde deleted file mode 100644 index 2b66ee611d..0000000000 --- a/magic/kde +++ /dev/null @@ -1,11 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: kde,v 1.4 2009/09/19 16:28:10 christos Exp $ -# kde: file(1) magic for KDE - -0 string/t [KDE\ Desktop\ Entry] KDE desktop entry -!:mime application/x-kdelnk -0 string/t #\ KDE\ Config\ File KDE config file -!:mime application/x-kdelnk -0 string/t #\ xmcd xmcd database file for kscd -!:mime text/x-xmcd diff --git a/magic/kml b/magic/kml deleted file mode 100644 index 608ff0e1b0..0000000000 --- a/magic/kml +++ /dev/null @@ -1,30 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: kml,v 1.2 2009/09/19 16:28:10 christos Exp $ -# Type: Google KML, formerly Keyhole Markup Language -# Future development of this format has been handed -# over to the Open Geospatial Consortium. -# http://www.opengeospatial.org/standards/kml/ -# From: Asbjoern Sloth Toennesen -0 string/t \20 search/400 \ xmlns= ->>&0 regex ['"]http://earth.google.com/kml Google KML document -!:mime application/vnd.google-earth.kml+xml - -#------------------------------------------------------------------------------ -# Type: OpenGIS KML, formerly Keyhole Markup Language -# This standard is maintained by the -# Open Geospatial Consortium. -# http://www.opengeospatial.org/standards/kml/ -# From: Asbjoern Sloth Toennesen ->>&0 regex ['"]http://www.opengis.net/kml OpenGIS KML document -!:mime application/vnd.google-earth.kml+xml - -#------------------------------------------------------------------------------ -# Type: Google KML Archive (ZIP based) -# http://code.google.com/apis/kml/documentation/kml_tut.html -# From: Asbjoern Sloth Toennesen -0 string PK\003\004 ->4 byte 0x14 ->>30 string doc.kml Compressed Google KML Document, including resources. -!:mime application/vnd.google-earth.kmz diff --git a/magic/linux b/magic/linux deleted file mode 100644 index 4a5c935760..0000000000 --- a/magic/linux +++ /dev/null @@ -1,22 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: linux,v 1.46 2013/01/06 21:26:48 christos Exp $ -# linux: file(1) magic for Linux files -# -# Values for Linux/i386 binaries, from Daniel Quinlan -# The following basic Linux magic is useful for reference, but using -# "long" magic is a better practice in order to avoid collisions. -# -# 2 leshort 100 Linux/i386 -# >0 leshort 0407 impure executable (OMAGIC) -# >0 leshort 0410 pure executable (NMAGIC) -# >0 leshort 0413 demand-paged executable (ZMAGIC) -# >0 leshort 0314 demand-paged executable (QMAGIC) -# - -# SYSLINUX boot logo files (from 'ppmtolss16' sources) -# http://www.syslinux.org/wiki/index.php/SYSLINUX#Display_graphic_from_filename: -# file extension .lss .16 -0 lelong =0x1413f33d SYSLINUX' LSS16 image data -# syslinux-4.05/mime/image/x-lss16.xml -!:mime image/x-lss16 diff --git a/magic/lisp b/magic/lisp deleted file mode 100644 index f5a06c8964..0000000000 --- a/magic/lisp +++ /dev/null @@ -1,42 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# lisp: file(1) magic for lisp programs -# -# various lisp types, from Daniel Quinlan (quinlan@yggdrasil.com) - -# updated by Joerg Jenderek -# GRR: This lot is too weak -#0 string ;; -# windows INF files often begin with semicolon and use CRLF as line end -# lisp files are mainly created on unix system with LF as line end -#>2 search/4096 !\r Lisp/Scheme program text -#>2 search/4096 \r Windows INF file - -0 search/4096 (setq\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (defvar\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (defparam\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (defun\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (autoload\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (custom-set-variables\ Lisp/Scheme program text -!:mime text/x-lisp - -# Emacs 18 - this is always correct, but not very magical. -0 string \012( Emacs v18 byte-compiled Lisp data -!:mime application/x-elc -# Emacs 19+ - ver. recognition added by Ian Springer -# Also applies to XEmacs 19+ .elc files; could tell them apart with regexs -# - Chris Chittleborough -0 string ;ELC ->4 byte >18 ->4 byte <32 Emacs/XEmacs v%d byte-compiled Lisp data -!:mime application/x-elc - -# From: David Allouche -0 search/1 \, Seo Sanghyeon - -# Lua scripts -0 search/1/w #!\ /usr/bin/lua Lua script text executable -!:mime text/x-lua -0 search/1/w #!\ /usr/local/bin/lua Lua script text executable -!:mime text/x-lua -0 search/1 #!/usr/bin/env\ lua Lua script text executable -!:mime text/x-lua -0 search/1 #!\ /usr/bin/env\ lua Lua script text executable -!:mime text/x-lua - diff --git a/magic/m4 b/magic/m4 deleted file mode 100644 index 7262fca81b..0000000000 --- a/magic/m4 +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# make: file(1) magic for M4 scripts -# -0 regex \^dnl\ M4 macro processor script text -!:mime text/x-m4 diff --git a/magic/macintosh b/magic/macintosh deleted file mode 100644 index 6398fc2ff2..0000000000 --- a/magic/macintosh +++ /dev/null @@ -1,21 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: macintosh,v 1.21 2010/09/20 19:19:17 rrt Exp $ -# macintosh description -# -# BinHex is the Macintosh ASCII-encoded file format (see also "apple") -# Daniel Quinlan, quinlan@yggdrasil.com -11 string must\ be\ converted\ with\ BinHex BinHex binary text -!:mime application/mac-binhex40 - -# Stuffit archives are the de facto standard of compression for Macintosh -# files obtained from most archives. (franklsm@tuns.ca) -0 string SIT! StuffIt Archive (data) -!:mime application/x-stuffit -!:apple SIT!SIT! - -# Newer StuffIt archives (grant@netbsd.org) -0 string StuffIt StuffIt Archive -!:mime application/x-stuffit -!:apple SIT!SIT! -#>162 string >0 : %s diff --git a/magic/mail.news b/magic/mail.news deleted file mode 100644 index c1a446d4ca..0000000000 --- a/magic/mail.news +++ /dev/null @@ -1,35 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: mail.news,v 1.21 2012/06/21 01:44:52 christos Exp $ -# mail.news: file(1) magic for mail and news -# -# Unfortunately, saved netnews also has From line added in some news software. -#0 string From mail text -0 string/t Relay-Version: old news text -!:mime message/rfc822 -0 string/t #!\ rnews batched news text -!:mime message/rfc822 -0 string/t N#!\ rnews mailed, batched news text -!:mime message/rfc822 -0 string/t Forward\ to mail forwarding text -!:mime message/rfc822 -0 string/t Pipe\ to mail piping text -!:mime message/rfc822 -0 string/tc delivered-to: SMTP mail text -!:mime message/rfc822 -0 string/tc return-path: SMTP mail text -!:mime message/rfc822 -0 string/t Path: news text -!:mime message/news -0 string/t Xref: news text -!:mime message/news -0 string/t From: news or mail text -!:mime message/rfc822 -0 string/t Article saved news text -!:mime message/news -0 string/t Received: RFC 822 mail text -!:mime message/rfc822 - -# TNEF files... -0 lelong 0x223E9F78 Transport Neutral Encapsulation Format -!:mime application/vnd.ms-tnef diff --git a/magic/make b/magic/make deleted file mode 100644 index 83d6a012dd..0000000000 --- a/magic/make +++ /dev/null @@ -1,16 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# make: file(1) magic for makefiles -# -0 regex \^CFLAGS makefile script text -!:mime text/x-makefile -0 regex \^LDFLAGS makefile script text -!:mime text/x-makefile -0 regex \^all: makefile script text -!:mime text/x-makefile -0 regex \^.PRECIOUS makefile script text -!:mime text/x-makefile - -0 regex \^SUBDIRS automake makefile script text -!:mime text/x-makefile diff --git a/magic/marc21 b/magic/marc21 deleted file mode 100644 index 26899d2e70..0000000000 --- a/magic/marc21 +++ /dev/null @@ -1,29 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#-------------------------------------------- -# marc21: file(1) magic for MARC 21 Format -# -# Kevin Ford (kefo@loc.gov) -# -# MARC21 formats are for the representation and communication -# of bibliographic and related information in machine-readable -# form. For more info, see http://www.loc.gov/marc/ - - -# leader position 20-21 must be 45 -20 string 45 - -# leader starts with 5 digits, followed by codes specific to MARC format ->0 regex/1 (^[0-9]{5})[acdnp][^bhlnqsu-z] MARC21 Bibliographic -!:mime application/marc ->0 regex/1 (^[0-9]{5})[acdnosx][z] MARC21 Authority -!:mime application/marc ->0 regex/1 (^[0-9]{5})[cdn][uvxy] MARC21 Holdings -!:mime application/marc -0 regex/1 (^[0-9]{5})[acdn][w] MARC21 Classification -!:mime application/marc ->0 regex/1 (^[0-9]{5})[cdn][q] MARC21 Community -!:mime application/marc - -# leader position 22-23, should be "00" but is it? ->0 regex/1 (^.{21})([^0]{2}) (non-conforming) -!:mime application/marc diff --git a/magic/matroska b/magic/matroska deleted file mode 100644 index c1791413cb..0000000000 --- a/magic/matroska +++ /dev/null @@ -1,17 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: matroska,v 1.7 2012/08/26 10:06:15 christos Exp $ -# matroska: file(1) magic for Matroska files -# -# See http://www.matroska.org/ -# - -# EBML id: -0 belong 0x1a45dfa3 -# DocType id: ->4 search/4096 \x42\x82 -# DocType contents: ->>&1 string webm WebM -!:mime video/webm ->>&1 string matroska Matroska data -!:mime video/x-matroska diff --git a/magic/misctools b/magic/misctools deleted file mode 100644 index 35fddaa61a..0000000000 --- a/magic/misctools +++ /dev/null @@ -1,9 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#----------------------------------------------------------------------------- -# $File: misctools,v 1.12 2010/09/29 18:36:49 rrt Exp $ -# misctools: file(1) magic for miscellaneous UNIX tools. -# -0 string/c BEGIN:VCALENDAR vCalendar calendar file -!:mime text/calendar -0 string/c BEGIN:VCARD vCard visiting card -!:mime text/x-vcard diff --git a/magic/msdos b/magic/msdos deleted file mode 100644 index 59a9d2caac..0000000000 --- a/magic/msdos +++ /dev/null @@ -1,368 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: msdos,v 1.84 2013/02/05 13:55:22 christos Exp $ -# msdos: file(1) magic for MS-DOS files -# - -# .BAT files (Daniel Quinlan, quinlan@yggdrasil.com) -# updated by Joerg Jenderek at Oct 2008,Apr 2011 -0 string/t @ ->1 string/cW \ echo\ off DOS batch file text -!:mime text/x-msdos-batch ->1 string/cW echo\ off DOS batch file text -!:mime text/x-msdos-batch ->1 string/cW rem DOS batch file text -!:mime text/x-msdos-batch ->1 string/cW set\ DOS batch file text -!:mime text/x-msdos-batch - -# Tests for various EXE types. -# -# Many of the compressed formats were extraced from IDARC 1.23 source code. -# -0 string/b MZ DOS MZ -!:mime application/x-dosexec -# All non-DOS EXE extensions have the relocation table more than 0x40 bytes into the file. ->0x18 leshort <0x40 MS-DOS executable -# These traditional tests usually work but not always. When test quality support is -# implemented these can be turned on. -#>>0x18 leshort 0x1c (Borland compiler) -#>>0x18 leshort 0x1e (MS compiler) - -# If the relocation table is 0x40 or more bytes into the file, it's definitely -# not a DOS EXE. ->0x18 leshort >0x3f - -# Maybe it's a PE? ->>(0x3c.l) string PE\0\0 PE ->>>(0x3c.l+24) leshort 0x010b \b32 executable ->>>(0x3c.l+24) leshort 0x020b \b32+ executable ->>>(0x3c.l+24) leshort 0x0107 ROM image ->>>(0x3c.l+24) default x Unknown PE signature ->>>>&0 leshort x 0x%x ->>>(0x3c.l+22) leshort&0x2000 >0 (DLL) ->>>(0x3c.l+92) leshort 1 (native) ->>>(0x3c.l+92) leshort 2 (GUI) ->>>(0x3c.l+92) leshort 3 (console) ->>>(0x3c.l+92) leshort 7 (POSIX) ->>>(0x3c.l+92) leshort 9 (Windows CE) ->>>(0x3c.l+92) leshort 10 (EFI application) ->>>(0x3c.l+92) leshort 11 (EFI boot service driver) ->>>(0x3c.l+92) leshort 12 (EFI runtime driver) ->>>(0x3c.l+92) leshort 13 (EFI ROM) ->>>(0x3c.l+92) leshort 14 (XBOX) ->>>(0x3c.l+92) leshort 15 (Windows boot application) ->>>(0x3c.l+92) default x (Unknown subsystem ->>>>&0 leshort x 0x%x) ->>>(0x3c.l+4) leshort 0x14c Intel 80386 ->>>(0x3c.l+4) leshort 0x166 MIPS R4000 ->>>(0x3c.l+4) leshort 0x168 MIPS R10000 ->>>(0x3c.l+4) leshort 0x184 Alpha ->>>(0x3c.l+4) leshort 0x1a2 Hitachi SH3 ->>>(0x3c.l+4) leshort 0x1a6 Hitachi SH4 ->>>(0x3c.l+4) leshort 0x1c0 ARM ->>>(0x3c.l+4) leshort 0x1c2 ARM Thumb ->>>(0x3c.l+4) leshort 0x1c4 ARMv7 Thumb ->>>(0x3c.l+4) leshort 0x1f0 PowerPC ->>>(0x3c.l+4) leshort 0x200 Intel Itanium ->>>(0x3c.l+4) leshort 0x266 MIPS16 ->>>(0x3c.l+4) leshort 0x268 Motorola 68000 ->>>(0x3c.l+4) leshort 0x290 PA-RISC ->>>(0x3c.l+4) leshort 0x366 MIPSIV ->>>(0x3c.l+4) leshort 0x466 MIPS16 with FPU ->>>(0x3c.l+4) leshort 0xebc EFI byte code ->>>(0x3c.l+4) leshort 0x8664 x86-64 ->>>(0x3c.l+4) leshort 0xc0ee MSIL ->>>(0x3c.l+4) default x Unknown processor type ->>>>&0 leshort x 0x%x ->>>(0x3c.l+22) leshort&0x0200 >0 (stripped to external PDB) ->>>(0x3c.l+22) leshort&0x1000 >0 system file ->>>(0x3c.l+24) leshort 0x010b ->>>>(0x3c.l+232) lelong >0 Mono/.Net assembly ->>>(0x3c.l+24) leshort 0x020b ->>>>(0x3c.l+248) lelong >0 Mono/.Net assembly - -# hooray, there's a DOS extender using the PE format, with a valid PE -# executable inside (which just prints a message and exits if run in win) ->>>(8.s*16) string 32STUB \b, 32rtm DOS extender ->>>(8.s*16) string !32STUB \b, for MS Windows ->>>(0x3c.l+0xf8) string UPX0 \b, UPX compressed ->>>(0x3c.l+0xf8) search/0x140 PEC2 \b, PECompact2 compressed ->>>(0x3c.l+0xf8) search/0x140 UPX2 ->>>>(&0x10.l+(-4)) string PK\3\4 \b, ZIP self-extracting archive (Info-Zip) ->>>(0x3c.l+0xf8) search/0x140 .idata ->>>>(&0xe.l+(-4)) string PK\3\4 \b, ZIP self-extracting archive (Info-Zip) ->>>>(&0xe.l+(-4)) string ZZ0 \b, ZZip self-extracting archive ->>>>(&0xe.l+(-4)) string ZZ1 \b, ZZip self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .rsrc ->>>>(&0x0f.l+(-4)) string a\\\4\5 \b, WinHKI self-extracting archive ->>>>(&0x0f.l+(-4)) string Rar! \b, RAR self-extracting archive ->>>>(&0x0f.l+(-4)) search/0x3000 MSCF \b, InstallShield self-extracting archive ->>>>(&0x0f.l+(-4)) search/32 Nullsoft \b, Nullsoft Installer self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .data ->>>>(&0x0f.l) string WEXTRACT \b, MS CAB-Installer self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .petite\0 \b, Petite compressed ->>>>(0x3c.l+0xf7) byte x ->>>>>(&0x104.l+(-4)) string =!sfx! \b, ACE self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .WISE \b, WISE installer self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .dz\0\0\0 \b, Dzip self-extracting archive ->>>&(0x3c.l+0xf8) search/0x100 _winzip_ \b, ZIP self-extracting archive (WinZip) ->>>&(0x3c.l+0xf8) search/0x100 SharedD \b, Microsoft Installer self-extracting archive ->>>0x30 string Inno \b, InnoSetup self-extracting archive - -# Hmm, not a PE but the relocation table is too high for a traditional DOS exe, -# must be one of the unusual subformats. ->>(0x3c.l) string !PE\0\0 MS-DOS executable - ->>(0x3c.l) string NE \b, NE ->>>(0x3c.l+0x36) byte 1 for OS/2 1.x ->>>(0x3c.l+0x36) byte 2 for MS Windows 3.x ->>>(0x3c.l+0x36) byte 3 for MS-DOS ->>>(0x3c.l+0x36) byte 4 for Windows 386 ->>>(0x3c.l+0x36) byte 5 for Borland Operating System Services ->>>(0x3c.l+0x36) default x ->>>>(0x3c.l+0x36) byte x (unknown OS %x) ->>>(0x3c.l+0x36) byte 0x81 for MS-DOS, Phar Lap DOS extender ->>>(0x3c.l+0x0c) leshort&0x8003 0x8002 (DLL) ->>>(0x3c.l+0x0c) leshort&0x8003 0x8001 (driver) ->>>&(&0x24.s-1) string ARJSFX \b, ARJ self-extracting archive ->>>(0x3c.l+0x70) search/0x80 WinZip(R)\ Self-Extractor \b, ZIP self-extracting archive (WinZip) - ->>(0x3c.l) string LX\0\0 \b, LX ->>>(0x3c.l+0x0a) leshort <1 (unknown OS) ->>>(0x3c.l+0x0a) leshort 1 for OS/2 ->>>(0x3c.l+0x0a) leshort 2 for MS Windows ->>>(0x3c.l+0x0a) leshort 3 for DOS ->>>(0x3c.l+0x0a) leshort >3 (unknown OS) ->>>(0x3c.l+0x10) lelong&0x28000 =0x8000 (DLL) ->>>(0x3c.l+0x10) lelong&0x20000 >0 (device driver) ->>>(0x3c.l+0x10) lelong&0x300 0x300 (GUI) ->>>(0x3c.l+0x10) lelong&0x28300 <0x300 (console) ->>>(0x3c.l+0x08) leshort 1 i80286 ->>>(0x3c.l+0x08) leshort 2 i80386 ->>>(0x3c.l+0x08) leshort 3 i80486 ->>>(8.s*16) string emx \b, emx ->>>>&1 string x %s ->>>&(&0x54.l-3) string arjsfx \b, ARJ self-extracting archive - -# MS Windows system file, supposedly a collection of LE executables ->>(0x3c.l) string W3 \b, W3 for MS Windows - ->>(0x3c.l) string LE\0\0 \b, LE executable ->>>(0x3c.l+0x0a) leshort 1 -# some DOS extenders use LE files with OS/2 header ->>>>0x240 search/0x100 DOS/4G for MS-DOS, DOS4GW DOS extender ->>>>0x240 search/0x200 WATCOM\ C/C++ for MS-DOS, DOS4GW DOS extender ->>>>0x440 search/0x100 CauseWay\ DOS\ Extender for MS-DOS, CauseWay DOS extender ->>>>0x40 search/0x40 PMODE/W for MS-DOS, PMODE/W DOS extender ->>>>0x40 search/0x40 STUB/32A for MS-DOS, DOS/32A DOS extender (stub) ->>>>0x40 search/0x80 STUB/32C for MS-DOS, DOS/32A DOS extender (configurable stub) ->>>>0x40 search/0x80 DOS/32A for MS-DOS, DOS/32A DOS extender (embedded) -# this is a wild guess; hopefully it is a specific signature ->>>>&0x24 lelong <0x50 ->>>>>(&0x4c.l) string \xfc\xb8WATCOM ->>>>>>&0 search/8 3\xdbf\xb9 \b, 32Lite compressed -# another wild guess: if real OS/2 LE executables exist, they probably have higher start EIP -#>>>>(0x3c.l+0x1c) lelong >0x10000 for OS/2 -# fails with DOS-Extenders. ->>>(0x3c.l+0x0a) leshort 2 for MS Windows ->>>(0x3c.l+0x0a) leshort 3 for DOS ->>>(0x3c.l+0x0a) leshort 4 for MS Windows (VxD) ->>>(&0x7c.l+0x26) string UPX \b, UPX compressed ->>>&(&0x54.l-3) string UNACE \b, ACE self-extracting archive - -# looks like ASCII, probably some embedded copyright message. -# and definitely not NE/LE/LX/PE ->>0x3c lelong >0x20000000 ->>>(4.s*512) leshort !0x014c \b, MZ for MS-DOS -# header data too small for extended executable ->2 long !0 ->>0x18 leshort <0x40 ->>>(4.s*512) leshort !0x014c - ->>>>&(2.s-514) string !LE ->>>>>&-2 string !BW \b, MZ for MS-DOS ->>>>&(2.s-514) string LE \b, LE ->>>>>0x240 search/0x100 DOS/4G for MS-DOS, DOS4GW DOS extender -# educated guess since indirection is still not capable enough for complex offset -# calculations (next embedded executable would be at &(&2*512+&0-2) -# I suspect there are only LE executables in these multi-exe files ->>>>&(2.s-514) string BW ->>>>>0x240 search/0x100 DOS/4G ,\b LE for MS-DOS, DOS4GW DOS extender (embedded) ->>>>>0x240 search/0x100 !DOS/4G ,\b BW collection for MS-DOS - -# This sequence skips to the first COFF segment, usually .text ->(4.s*512) leshort 0x014c \b, COFF ->>(8.s*16) string go32stub for MS-DOS, DJGPP go32 DOS extender ->>(8.s*16) string emx ->>>&1 string x for DOS, Win or OS/2, emx %s ->>&(&0x42.l-3) byte x ->>>&0x26 string UPX \b, UPX compressed -# and yet another guess: small .text, and after large .data is unusal, could be 32lite ->>&0x2c search/0xa0 .text ->>>&0x0b lelong <0x2000 ->>>>&0 lelong >0x6000 \b, 32lite compressed - ->(8.s*16) string $WdX \b, WDos/X DOS extender - -# By now an executable type should have been printed out. The executable -# may be a self-uncompressing archive, so look for evidence of that and -# print it out. -# -# Some signatures below from Greg Roelofs, newt@uchicago.edu. -# ->0x35 string \x8e\xc0\xb9\x08\x00\xf3\xa5\x4a\x75\xeb\x8e\xc3\x8e\xd8\x33\xff\xbe\x30\x00\x05 \b, aPack compressed ->0xe7 string LH/2\ Self-Extract \b, %s ->0x1c string UC2X \b, UCEXE compressed ->0x1c string WWP\ \b, WWPACK compressed ->0x1c string RJSX \b, ARJ self-extracting archive ->0x1c string diet \b, diet compressed ->0x1c string LZ09 \b, LZEXE v0.90 compressed ->0x1c string LZ91 \b, LZEXE v0.91 compressed ->0x1c string tz \b, TinyProg compressed ->0x1e string Copyright\ 1989-1990\ PKWARE\ Inc. Self-extracting PKZIP archive -!:mime application/zip -# Yes, this really is "Copr", not "Corp." ->0x1e string PKLITE\ Copr. Self-extracting PKZIP archive -!:mime application/zip -# winarj stores a message in the stub instead of the sig in the MZ header ->0x20 search/0xe0 aRJsfX \b, ARJ self-extracting archive ->0x20 string AIN ->>0x23 string 2 \b, AIN 2.x compressed ->>0x23 string <2 \b, AIN 1.x compressed ->>0x23 string >2 \b, AIN 1.x compressed ->0x24 string LHa's\ SFX \b, LHa self-extracting archive -!:mime application/x-lha ->0x24 string LHA's\ SFX \b, LHa self-extracting archive -!:mime application/x-lha ->0x24 string \ $ARX \b, ARX self-extracting archive ->0x24 string \ $LHarc \b, LHarc self-extracting archive ->0x20 string SFX\ by\ LARC \b, LARC self-extracting archive ->0x40 string aPKG \b, aPackage self-extracting archive ->0x64 string W\ Collis\0\0 \b, Compack compressed ->0x7a string Windows\ self-extracting\ ZIP \b, ZIP self-extracting archive ->>&0xf4 search/0x140 \x0\x40\x1\x0 ->>>(&0.l+(4)) string MSCF \b, WinHKI CAB self-extracting archive ->1638 string -lh5- \b, LHa self-extracting archive v2.13S ->0x17888 string Rar! \b, RAR self-extracting archive - -# Skip to the end of the EXE. This will usually work fine in the PE case -# because the MZ image is hardcoded into the toolchain and almost certainly -# won't match any of these signatures. ->(4.s*512) long x ->>&(2.s-517) byte x ->>>&0 string PK\3\4 \b, ZIP self-extracting archive ->>>&0 string Rar! \b, RAR self-extracting archive ->>>&0 string =!\x11 \b, AIN 2.x self-extracting archive ->>>&0 string =!\x12 \b, AIN 2.x self-extracting archive ->>>&0 string =!\x17 \b, AIN 1.x self-extracting archive ->>>&0 string =!\x18 \b, AIN 1.x self-extracting archive ->>>&7 search/400 **ACE** \b, ACE self-extracting archive ->>>&0 search/0x480 UC2SFX\ Header \b, UC2 self-extracting archive - -# a few unknown ZIP sfxes, no idea if they are needed or if they are -# already captured by the generic patterns above ->(8.s*16) search/0x20 PKSFX \b, ZIP self-extracting archive (PKZIP) -# TODO: how to add this? >FileSize-34 string Windows\ Self-Installing\ Executable \b, ZIP self-extracting archive -# - -# TELVOX Teleinformatica CODEC self-extractor for OS/2: ->49801 string \x79\xff\x80\xff\x76\xff \b, CODEC archive v3.21 ->>49824 leshort =1 \b, 1 file ->>49824 leshort >1 \b, %u files - -# Popular applications -2080 string Microsoft\ Word\ 6.0\ Document %s -!:mime application/msword -2080 string Documento\ Microsoft\ Word\ 6 Spanish Microsoft Word 6 document data -!:mime application/msword -# Pawel Wiecek (for polish Word) -2112 string MSWordDoc Microsoft Word document data -!:mime application/msword -# -0 belong 0x31be0000 Microsoft Word Document -!:mime application/msword -# -0 string/b PO^Q` Microsoft Word 6.0 Document -!:mime application/msword -# -0 string/b \376\067\0\043 Microsoft Office Document -!:mime application/msword -0 string/b \333\245-\0\0\0 Microsoft Office Document -!:mime application/msword -512 string/b \354\245\301 Microsoft Word Document -!:mime application/msword - -# -0 string/b \xDB\xA5\x2D\x00 Microsoft WinWord 2.0 Document -!:mime application/msword -# -2080 string Microsoft\ Excel\ 5.0\ Worksheet %s -!:mime application/vnd.ms-excel -# -0 string/b \xDB\xA5\x2D\x00 Microsoft WinWord 2.0 Document -!:mime application/msword - -2080 string Foglio\ di\ lavoro\ Microsoft\ Exce %s -!:mime application/vnd.ms-excel -# -# Pawel Wiecek (for polish Excel) -2114 string Biff5 Microsoft Excel 5.0 Worksheet -!:mime application/vnd.ms-excel -# Italian MS-Excel -2121 string Biff5 Microsoft Excel 5.0 Worksheet -!:mime application/vnd.ms-excel -0 string/b \x09\x04\x06\x00\x00\x00\x10\x00 Microsoft Excel Worksheet -!:mime application/vnd.ms-excel -# -0 belong 0x00001a00 Lotus 1-2-3 -!:mime application/x-123 -# -0 belong 0x00000200 Lotus 1-2-3 -!:mime application/x-123 -0 string/b WordPro\0 Lotus WordPro -!:mime application/vnd.lotus-wordpro -0 string/b WordPro\r\373 Lotus WordPro -!:mime application/vnd.lotus-wordpro - -# Windows icons (Ian Springer ) -0 string/b \000\000\001\000 MS Windows icon resource -!:mime image/x-icon - -# .PIF files added by Joerg Jenderek from http://smsoft.ru/en/pifdoc.htm -# only for windows versions equal or greater 3.0 -0x171 string MICROSOFT\ PIFEX\0 Windows Program Information File -!:mime application/x-dosexec - -# TNEF magic From "Joomy" -# Microsoft Outlook's Transport Neutral Encapsulation Format (TNEF) -0 leshort 0x223e9f78 TNEF -!:mime application/vnd.ms-tnef - -#------------------------------------------------------------------------------ -# From Stuart Caie (developer of cabextract) -# Microsoft Cabinet files -0 string/b MSCF\0\0\0\0 Microsoft Cabinet archive data -!:mime application/vnd.ms-cab-compressed - -# from http://filext.com by Derek M Jones -# False positive with PPT (also currently this string is too long) -#0 string/b \xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3E\x00\x03\x00\xFE\xFF\x09\x00\x06 Microsoft Installer -0 string/b \320\317\021\340\241\261\032\341 Microsoft Office Document -#>48 byte 0x1B Excel Document -#!:mime application/vnd.ms-excel ->546 string bjbj Microsoft Word Document -!:mime application/msword ->546 string jbjb Microsoft Word Document -!:mime application/msword - -0 string/b \224\246\056 Microsoft Word Document -!:mime application/msword - -512 string R\0o\0o\0t\0\ \0E\0n\0t\0r\0y Microsoft Word Document -!:mime application/msword - -# MS eBook format (.lit) -0 string/b ITOLITLS Microsoft Reader eBook Data ->8 lelong x \b, version %u -!:mime application/x-ms-reader diff --git a/magic/neko b/magic/neko deleted file mode 100644 index 50163a0861..0000000000 --- a/magic/neko +++ /dev/null @@ -1,12 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------ -# $File: java,v 1.12 2009/09/19 16:28:10 christos Exp $ - -# From: Mikhail Gusarov -# NekoVM (http://nekovm.org/) bytecode -0 string NEKO NekoVM bytecode ->4 lelong x (%d global symbols, ->8 lelong x %d global fields, ->12 lelong x %d bytecode ops) -!:mime application/x-nekovm-bytecode - diff --git a/magic/pascal b/magic/pascal deleted file mode 100644 index 911eea3c0c..0000000000 --- a/magic/pascal +++ /dev/null @@ -1,11 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pascal: file(1) magic for Pascal source -# -0 search/8192 (input, Pascal source text -!:mime text/x-pascal -0 regex \^program Pascal source text -!:mime text/x-pascal -0 regex \^record Pascal source text -!:mime text/x-pascal diff --git a/magic/pdf b/magic/pdf deleted file mode 100644 index 761006ffe6..0000000000 --- a/magic/pdf +++ /dev/null @@ -1,8 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pdf: file(1) magic for Portable Document Format -# - -0 string %PDF- PDF document -!:mime application/pdf diff --git a/magic/perl b/magic/perl deleted file mode 100644 index 12ec33b73a..0000000000 --- a/magic/perl +++ /dev/null @@ -1,26 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: perl,v 1.19 2012/06/20 21:16:25 christos Exp $ -# perl: file(1) magic for Larry Wall's perl language. -# -# The `eval' lines recognizes an outrageously clever hack. -# Keith Waclena -# Send additions to -0 search/1/w #!\ /bin/perl Perl script text executable -!:mime text/x-perl -0 search/1 eval\ "exec\ /bin/perl Perl script text -!:mime text/x-perl -0 search/1/w #!\ /usr/bin/perl Perl script text executable -!:mime text/x-perl -0 search/1 eval\ "exec\ /usr/bin/perl Perl script text -!:mime text/x-perl -0 search/1/w #!\ /usr/local/bin/perl Perl script text executable -!:mime text/x-perl -0 search/1 eval\ "exec\ /usr/local/bin/perl Perl script text -!:mime text/x-perl -0 search/1 eval\ '(exit\ $?0)'\ &&\ eval\ 'exec Perl script text -!:mime text/x-perl -0 search/1 #!/usr/bin/env\ perl Perl script text executable -!:mime text/x-perl -0 search/1 #!\ /usr/bin/env\ perl Perl script text executable -!:mime text/x-perl diff --git a/magic/pgp b/magic/pgp deleted file mode 100644 index 2bdfb77981..0000000000 --- a/magic/pgp +++ /dev/null @@ -1,27 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pgp: file(1) magic for Pretty Good Privacy -# see http://lists.gnupg.org/pipermail/gnupg-devel/1999-September/016052.html -# -0 beshort 0x9900 PGP key public ring -!:mime application/x-pgp-keyring -0 beshort 0x9501 PGP key security ring -!:mime application/x-pgp-keyring -0 beshort 0x9500 PGP key security ring -!:mime application/x-pgp-keyring -0 beshort 0xa600 PGP encrypted data -#!:mime application/pgp-encrypted -#0 string -----BEGIN\040PGP text/PGP armored data -!:mime text/PGP # encoding: armored data -#>15 string PUBLIC\040KEY\040BLOCK- public key block -#>15 string MESSAGE- message -#>15 string SIGNED\040MESSAGE- signed message -#>15 string PGP\040SIGNATURE- signature - -2 string ---BEGIN\ PGP\ PUBLIC\ KEY\ BLOCK- PGP public key block -!:mime application/pgp-keys -0 string -----BEGIN\040PGP\40MESSAGE- PGP message -!:mime application/pgp -0 string -----BEGIN\040PGP\40SIGNATURE- PGP signature -!:mime application/pgp-signature diff --git a/magic/pkgadd b/magic/pkgadd deleted file mode 100644 index 602b4ec21d..0000000000 --- a/magic/pkgadd +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pkgadd: file(1) magic for SysV R4 PKG Datastreams -# -0 string #\ PaCkAgE\ DaTaStReAm pkg Datastream (SVR4) -!:mime application/x-svr4-package diff --git a/magic/printer b/magic/printer deleted file mode 100644 index cdce275b12..0000000000 --- a/magic/printer +++ /dev/null @@ -1,14 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: printer,v 1.24 2011/05/08 16:34:51 christos Exp $ -# printer: file(1) magic for printer-formatted files -# - -# PostScript, updated by Daniel Quinlan (quinlan@yggdrasil.com) -0 string %! PostScript document text -!:mime application/postscript -!:apple ASPSTEXT -# Some PCs have the annoying habit of adding a ^D as a document separator -0 string \004%! PostScript document text -!:mime application/postscript -!:apple ASPSTEXT diff --git a/magic/python b/magic/python deleted file mode 100644 index 1cd724bc59..0000000000 --- a/magic/python +++ /dev/null @@ -1,46 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: python,v 1.21 2012/06/21 01:12:51 christos Exp $ -# python: file(1) magic for python -# - -0 search/1/w #!\ /usr/bin/python Python script text executable -!:mime text/x-python -0 search/1/w #!\ /usr/local/bin/python Python script text executable -!:mime text/x-python -0 search/1 #!/usr/bin/env\ python Python script text executable -!:mime text/x-python -0 search/1 #!\ /usr/bin/env\ python Python script text executable -!:mime text/x-python - -# from module.submodule import func1, func2 -0 regex \^from\\s+(\\w|\\.)+\\s+import.*$ Python script text executable -!:mime text/x-python - -# def __init__ (self, ...): -0 search/4096 def\ __init__ ->&0 search/64 self Python script text executable -!:mime text/x-python - -# comments -0 search/4096 ''' ->&0 regex .*'''$ Python script text executable -!:mime text/x-python - -0 search/4096 """ ->&0 regex .*"""$ Python script text executable -!:mime text/x-python - -# try: -# except: or finally: -# block -0 search/4096 try: ->&0 regex \^\\s*except.*: Python script text executable -!:mime text/x-python ->&0 search/4096 finally: Python script text executable -!:mime text/x-python - -# def name(args, args): -0 regex \^(\ |\\t)*def\ +[a-zA-Z]+ ->&0 regex \ *\\(([a-zA-Z]|,|\ )*\\):$ Python script text executable -!:mime text/x-python diff --git a/magic/riff b/magic/riff deleted file mode 100644 index 929dc9aa89..0000000000 --- a/magic/riff +++ /dev/null @@ -1,36 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: riff,v 1.22 2011/09/06 11:00:06 christos Exp $ -# riff: file(1) magic for RIFF format -# See -# -# http://www.seanet.com/users/matts/riffmci/riffmci.htm -# -# AVI section extended by Patrik Radman -# -0 string RIFF RIFF (little-endian) data -# Microsoft WAVE format (*.wav) ->8 string WAVE \b, WAVE audio -!:mime audio/x-wav -# Corel Draw Picture ->8 string CDRA \b, Corel Draw Picture -!:mime image/x-coreldraw -# AVI == Audio Video Interleave ->8 string AVI\040 \b, AVI -!:mime video/x-msvideo - -#------------------------------------------------------------------------------ -# Sony Wave64 -# see http://www.vcs.de/fileadmin/user_upload/MBS/PDF/Whitepaper/Informations_about_Sony_Wave64.pdf -# 128 bit RIFF-GUID { 66666972-912E-11CF-A5D6-28DB04C10000 } in little-endian -0 string riff\x2E\x91\xCF\x11\xA5\xD6\x28\xDB\x04\xC1\x00\x00 Sony Wave64 RIFF data -# 128 bit + total file size (64 bits) so 24 bytes -# then WAVE-GUID { 65766177-ACF3-11D3-8CD1-00C04F8EDB8A } ->24 string wave\xF3\xAC\xD3\x11\x8C\xD1\x00\xC0\x4F\x8E\xDB\x8A \b, WAVE 64 audio -!:mime audio/x-w64 - -#------------------------------------------------------------------------------ -# MBWF/RF64 -# see EBU TECH 3306 http://tech.ebu.ch/docs/tech/tech3306-2009.pdf -0 string RF64\xff\xff\xff\xffWAVEds64 MBWF/RF64 audio -!:mime audio/x-wav diff --git a/magic/rpm b/magic/rpm deleted file mode 100644 index 2558ebeef1..0000000000 --- a/magic/rpm +++ /dev/null @@ -1,12 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: rpm,v 1.11 2011/06/14 12:47:41 christos Exp $ -# -# RPM: file(1) magic for Red Hat Packages Erik Troan (ewt@redhat.com) -# -0 belong 0xedabeedb RPM -!:mime application/x-rpm - -#delta RPM Daniel Novotny (dnovotny@redhat.com) -0 string drpm Delta RPM -!:mime application/x-rpm diff --git a/magic/rtf b/magic/rtf deleted file mode 100644 index 0719264e47..0000000000 --- a/magic/rtf +++ /dev/null @@ -1,9 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# rtf: file(1) magic for Rich Text Format (RTF) -# -# Duncan P. Simpson, D.P.Simpson@dcs.warwick.ac.uk -# -0 string {\\rtf Rich Text Format data, -!:mime text/rtf diff --git a/magic/ruby b/magic/ruby deleted file mode 100644 index 41682a89ad..0000000000 --- a/magic/ruby +++ /dev/null @@ -1,28 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: ruby,v 1.4 2010/07/08 20:24:13 christos Exp $ -# ruby: file(1) magic for Ruby scripting language -# URL: http://www.ruby-lang.org/ -# From: Reuben Thomas - -# Ruby scripts -0 search/1/w #!\ /usr/bin/ruby Ruby script text executable -!:mime text/x-ruby -0 search/1/w #!\ /usr/local/bin/ruby Ruby script text executable -!:mime text/x-ruby -0 search/1 #!/usr/bin/env\ ruby Ruby script text executable -!:mime text/x-ruby -0 search/1 #!\ /usr/bin/env\ ruby Ruby script text executable -!:mime text/x-ruby - -# What looks like ruby, but does not have a shebang -# (modules and such) -# From: Lubomir Rintel -0 regex \^[\ \t]*require[\ \t]'[A-Za-z_/]+' ->0 regex include\ [A-Z]|def\ [a-z]|\ do$ ->>0 regex \^[\ \t]*end([\ \t]*[;#].*)?$ Ruby script text -!:mime text/x-ruby -0 regex \^[\ \t]*(class|module)[\ \t][A-Z] ->0 regex (modul|includ)e\ [A-Z]|def\ [a-z] ->>0 regex \^[\ \t]*end([\ \t]*[;#].*)?$ Ruby module source text -!:mime text/x-ruby diff --git a/magic/sc b/magic/sc deleted file mode 100644 index 75333b3916..0000000000 --- a/magic/sc +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# sc: file(1) magic for "sc" spreadsheet -# -38 string Spreadsheet sc spreadsheet file -!:mime application/x-sc diff --git a/magic/sgml b/magic/sgml deleted file mode 100644 index 64efa2c153..0000000000 --- a/magic/sgml +++ /dev/null @@ -1,82 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: sgml,v 1.28 2012/04/28 21:20:26 christos Exp $ -# Type: SVG Vectorial Graphics -# From: Noel Torres -0 string \15 string >\0 ->>19 search/4096 \>19 search/4096 \15 string >\0 ->>19 search/4096/cWbt \15 string >\0 ->>19 search/4096/cWbt \15 string >\0 ->>19 search/4096/cWbt \ - -# Although we may know the offset of certain text fields in TeX DVI -# and font files, we can't use them reliably because they are not -# zero terminated. [but we do anyway, christos] -0 string \367\002 TeX DVI file -!:mime application/x-dvi - -# There is no way to detect TeX Font Metric (*.tfm) files without -# breaking them apart and reading the data. The following patterns -# match most *.tfm files generated by METAFONT or afm2tfm. -2 string \000\021 TeX font metric data -!:mime application/x-tex-tfm -2 string \000\022 TeX font metric data -!:mime application/x-tex-tfm - -# Texinfo and GNU Info, from Daniel Quinlan (quinlan@yggdrasil.com) -0 search/1 \\input\ texinfo Texinfo source text -!:mime text/x-texinfo -0 search/1 This\ is\ Info\ file GNU Info text -!:mime text/x-info - -# TeX documents, from Daniel Quinlan (quinlan@yggdrasil.com) -0 search/4096 \\input TeX document text -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\section LaTeX document text -!:mime text/x-tex -!:strength + 18 -0 search/4096 \\setlength LaTeX document text -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\documentstyle LaTeX document text -!:mime text/x-tex -!:strength + 18 -0 search/4096 \\chapter LaTeX document text -!:mime text/x-tex -!:strength + 18 -0 search/4096 \\documentclass LaTeX 2e document text -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\relax LaTeX auxiliary file -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\contentsline LaTeX table of contents -!:mime text/x-tex -!:strength + 15 -0 search/4096 %\ -*-latex-*- LaTeX document text -!:mime text/x-tex diff --git a/magic/troff b/magic/troff deleted file mode 100644 index 7f60b1d9b3..0000000000 --- a/magic/troff +++ /dev/null @@ -1,22 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# troff: file(1) magic for *roff -# -# updated by Daniel Quinlan (quinlan@yggdrasil.com) - -# troff input -0 search/1 .\\" troff or preprocessor input text -!:mime text/troff -0 search/1 '\\" troff or preprocessor input text -!:mime text/troff -0 search/1 '.\\" troff or preprocessor input text -!:mime text/troff -0 search/1 \\" troff or preprocessor input text -!:mime text/troff -0 search/1 ''' troff or preprocessor input text -!:mime text/troff -0 regex/20 \^\\.[A-Za-z0-9][A-Za-z0-9][\ \t] troff or preprocessor input text -!:mime text/troff -0 regex/20 \^\\.[A-Za-z0-9][A-Za-z0-9]$ troff or preprocessor input text -!:mime text/troff diff --git a/magic/vorbis b/magic/vorbis deleted file mode 100644 index 4d25c3c3cd..0000000000 --- a/magic/vorbis +++ /dev/null @@ -1,26 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# vorbis: file(1) magic for Ogg/Vorbis files -# -# From Felix von Leitner -# Extended by Beni Cherniavsky -# Further extended by Greg Wooledge -# -# Most (everything but the number of channels and bitrate) is commented -# out with `##' as it's not interesting to the average user. The most -# probable things advanced users would want to uncomment are probably -# the number of comments and the encoder version. -# -# FIXME: The first match has been made a search, so that it can skip -# over prepended ID3 tags. This will work for MIME type detection, but -# won't work for detecting other properties of the file (they all need -# to be made relative to the search). In any case, if the file has ID3 -# tags, the ID3 information will be printed, not the Ogg information, -# so until that's fixed, this doesn't matter. -# FIXME[2]: Disable the above for now, since search assumes text mode. -# -# --- Ogg Framing --- -#0 search/1000 OggS Ogg data -0 string OggS Ogg data -!:mime application/ogg diff --git a/magic/warc b/magic/warc deleted file mode 100644 index 2a2aeb6fae..0000000000 --- a/magic/warc +++ /dev/null @@ -1,14 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: warc,v 1.2 2009/09/19 16:28:13 christos Exp $ -# warc: file(1) magic for WARC files - -0 string WARC/ WARC Archive ->5 string x version %.4s -!:mime application/warc - -#------------------------------------------------------------------------------ -# Arc File Format from Internet Archive -# see http://www.archive.org/web/researcher/ArcFileFormat.php -0 string filedesc:// Internet Archive File -!:mime application/x-ia-arc diff --git a/magic/windows b/magic/windows deleted file mode 100644 index 6a529782a9..0000000000 --- a/magic/windows +++ /dev/null @@ -1,19 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: windows,v 1.4 2009/09/19 16:28:13 christos Exp $ -# windows: file(1) magic for Microsoft Windows -# -# This file is mainly reserved for files where programs -# using them are run almost always on MS Windows 3.x or -# above, or files only used exclusively in Windows OS, -# where there is no better category to allocate for. -# For example, even though WinZIP almost run on Windows -# only, it is better to treat them as "archive" instead. -# For format usable in DOS, such as generic executable -# format, please specify under "msdos" file. -# - -# From: Pal Tamas -# Autorun File -0 string/c [autorun]\r\n Microsoft Windows Autorun file. -!:mime application/x-setupscript. diff --git a/magic/wordprocessors b/magic/wordprocessors deleted file mode 100644 index 7de3413c0a..0000000000 --- a/magic/wordprocessors +++ /dev/null @@ -1,43 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: wordprocessors,v 1.16 2012/10/29 17:36:49 christos Exp $ -# wordprocessors: file(1) magic fo word processors. -# - -# Hangul (Korean) Word Processor File -# From: Won-Kyu Park -512 string R\0o\0o\0t\0 Hangul (Korean) Word Processor File 2000 -!:mime application/x-hwp - -# Quark Express from http://www.garykessler.net/library/file_sigs.html -2 string MMXPR3 Motorola Quark Express Document (English) -!:mime application/x-quark-xpress-3 - -#------------------------------------------------------------------------------ -# ichitaro456: file(1) magic for Just System Word Processor Ichitaro -# -# Contributor kenzo-: -# Reversed-engineered JS Ichitaro magic numbers -# - -0 string DOC ->43 byte 0x14 Just System Word Processor Ichitaro v4 -!:mime application/x-ichitaro4 - -0 string DOC ->43 byte 0x15 Just System Word Processor Ichitaro v5 -!:mime application/x-ichitaro5 - -0 string DOC ->43 byte 0x16 Just System Word Processor Ichitaro v6 -!:mime application/x-ichitaro6 - -# Type: Freemind mindmap documents -# From: Jamie Thompson -0 string/w \ -0 string \ 0 ) + f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h); + + add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h]; + } + +event file_timeout(f: fa_file) &priority=10 + { + set_info(f); + f$info$timedout = T; + } + +event file_state_remove(f: fa_file) &priority=10 + { + set_info(f); + } + +event file_state_remove(f: fa_file) &priority=-10 + { + Log::write(Files::LOG, f$info); + } + +function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool + { + local result = (tag !in registered_protocols); + registered_protocols[tag] = reg; + return result; + } + +function describe(f: fa_file): string + { + local tag = Analyzer::get_tag(f$source); + if ( tag !in registered_protocols ) + return ""; + + local handler = registered_protocols[tag]; + return handler$describe(f); + } + +event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5 + { + if ( tag !in registered_protocols ) + return; + + local handler = registered_protocols[tag]; + set_file_handle(handler$get_file_handle(c, is_orig)); + } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 30e0013517..d046e6b076 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -68,6 +68,25 @@ export { ## the notice policy. iconn: icmp_conn &optional; + ## A file record if the notice is relted to a file. The + ## reference to the actual fa_file record will be deleted after applying + ## the notice policy. + f: fa_file &optional; + + ## A file unique ID if this notice is related to a file. If the $f + ## field is provided, this will be automatically filled out. + fuid: string &log &optional; + + ## A mime type if the notice is related to a file. If the $f field + ## is provided, this will be automatically filled out. + file_mime_type: string &log &optional; + + ## Frequently files can be "described" to give a bit more context. + ## This field will typically be automatically filled out from an + ## fa_file record. For example, if a notice was related to a + ## file over HTTP, the URL of the request would be shown. + file_desc: string &log &optional; + ## The transport protocol. Filled automatically when either conn, iconn ## or p is specified. proto: transport_proto &log &optional; @@ -460,10 +479,28 @@ function apply_policy(n: Notice::Info) if ( ! n?$ts ) n$ts = network_time(); + if ( n?$f ) + { + if ( ! n?$fuid ) + n$fuid = n$f$id; + + if ( ! n?$file_mime_type && n$f?$mime_type ) + n$file_mime_type = n$f$mime_type; + + n$file_desc = Files::describe(n$f); + + if ( n$f?$conns && |n$f$conns| == 1 ) + { + for ( id in n$f$conns ) + n$conn = n$f$conns[id]; + } + } + if ( n?$conn ) { if ( ! n?$id ) n$id = n$conn$id; + if ( ! n?$uid ) n$uid = n$conn$uid; } @@ -513,13 +550,15 @@ function apply_policy(n: Notice::Info) if ( ! n?$suppress_for ) n$suppress_for = default_suppression_interval; - # Delete the connection record if it's there so we aren't sending that - # to remote machines. It can cause problems due to the size of the - # connection record. + # Delete the connection and file records if they're there so we + # aren't sending that to remote machines. It can cause problems + # due to the size of those records. if ( n?$conn ) delete n$conn; if ( n?$iconn ) delete n$iconn; + if ( n?$f ) + delete n$f; } function internal_NOTICE(n: Notice::Info) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 6ad788f8c2..9876ad03f7 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -328,7 +328,7 @@ type fa_file: record { ## An identification of the source of the file data. E.g. it may be ## a network protocol over which it was transferred, or a local file ## path which was read, or some other input source. - source: string &optional; + source: string; ## If the source of this file is is a network connection, this field ## may be set to indicate the directionality. @@ -3050,6 +3050,6 @@ const snaplen = 8192 &redef; @load base/frameworks/logging @load base/frameworks/input @load base/frameworks/analyzer -@load base/frameworks/file-analysis +@load base/frameworks/files @load base/bif diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 6aa8ff5e26..72ba0bf115 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -49,4 +49,7 @@ @load base/protocols/syslog @load base/protocols/tunnels +@load base/files/hash +@load base/files/extract + @load base/misc/find-checksum-offloading diff --git a/scripts/base/protocols/ftp/__load__.bro b/scripts/base/protocols/ftp/__load__.bro index f3226de69d..ebb09e702c 100644 --- a/scripts/base/protocols/ftp/__load__.bro +++ b/scripts/base/protocols/ftp/__load__.bro @@ -1,7 +1,7 @@ @load ./utils-commands @load ./main -@load ./file-analysis -@load ./file-extract +@load ./utils +@load ./files @load ./gridftp -@load-sigs ./dpd.sig \ No newline at end of file +@load-sigs ./dpd.sig diff --git a/scripts/base/protocols/ftp/file-analysis.bro b/scripts/base/protocols/ftp/file-analysis.bro deleted file mode 100644 index 2d7609197a..0000000000 --- a/scripts/base/protocols/ftp/file-analysis.bro +++ /dev/null @@ -1,48 +0,0 @@ -@load ./main -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module FTP; - -export { - ## Default file handle provider for FTP. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -function get_handle_string(c: connection): string - { - return cat(Analyzer::ANALYZER_FTP_DATA, " ", c$start_time, " ", id_string(c$id)); - } - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) return ""; - - local info: FTP::Info = ftp_data_expected[c$id$resp_h, c$id$resp_p]; - - if ( info$passive ) - # FTP client initiates data channel. - if ( is_orig ) - # Don't care about FTP client data. - return ""; - else - # Do care about FTP server data. - return get_handle_string(c); - else - # FTP server initiates dta channel. - if ( is_orig ) - # Do care about FTP server data. - return get_handle_string(c); - else - # Don't care about FTP client data. - return ""; - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_FTP_DATA ) return; - set_file_handle(FTP::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/ftp/file-extract.bro b/scripts/base/protocols/ftp/file-extract.bro deleted file mode 100644 index 2b7bb8cd50..0000000000 --- a/scripts/base/protocols/ftp/file-extract.bro +++ /dev/null @@ -1,90 +0,0 @@ -##! File extraction support for FTP. - -@load ./main -@load base/utils/files - -module FTP; - -export { - ## Pattern of file mime types to extract from FTP transfers. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## The on-disk prefix for files to be extracted from FTP-data transfers. - const extraction_prefix = "ftp-item" &redef; -} - -redef record Info += { - ## On disk file where it was extracted to. - extraction_file: string &log &optional; - - ## Indicates if the current command/response pair should attempt to - ## extract the file if a file was transferred. - extract_file: bool &default=F; -}; - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "FTP_DATA" ) return; - - if ( f?$mime_type && extract_file_types in f$mime_type ) - { - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=get_extraction_name(f)]); - return; - } - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in ftp_data_expected ) next; - - local s = ftp_data_expected[cid$resp_h, cid$resp_p]; - - if ( ! s$extract_file ) next; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=get_extraction_name(f)]); - return; - } - } - -event file_state_remove(f: fa_file) &priority=4 - { - if ( ! f?$source ) return; - if ( f$source != "FTP_DATA" ) return; - if ( ! f?$info ) return; - - for ( filename in f$info$extracted_files ) - { - local s: FTP::Info; - s$ts = network_time(); - s$tags = set(); - s$user = ""; - s$extraction_file = filename; - - if ( f?$conns ) - for ( cid in f$conns ) - { - s$uid = f$conns[cid]$uid; - s$id = cid; - } - - Log::write(FTP::LOG, s); - } - } - -event log_ftp(rec: Info) &priority=-10 - { - delete rec$extraction_file; - delete rec$extract_file; - } diff --git a/scripts/base/protocols/ftp/files.bro b/scripts/base/protocols/ftp/files.bro new file mode 100644 index 0000000000..9ed17ab2a4 --- /dev/null +++ b/scripts/base/protocols/ftp/files.bro @@ -0,0 +1,60 @@ +@load ./main +@load ./utils +@load base/utils/conn-ids +@load base/frameworks/files + +module FTP; + +export { + redef record Info += { + ## File unique ID. + fuid: string &optional &log; + }; + + ## Default file handle provider for FTP. + global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Describe the file being transferred. + global describe_file: function(f: fa_file): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) + return ""; + + return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig); + } + +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "FTP" ) + return ""; + + for ( cid in f$conns ) + { + if ( f$conns[cid]?$ftp ) + return FTP::describe(f$conns[cid]$ftp); + } + return ""; + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, + [$get_file_handle = FTP::get_file_handle, + $describe = FTP::describe_file]); + } + + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) + return; + + local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p]; + ftp$fuid = f$id; + if ( f?$mime_type ) + ftp$mime_type = f$mime_type; + } diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 7bf9d6cc4c..7e66b63f40 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -63,8 +63,6 @@ export { reply_code: count &log &optional; ## Reply message from the server in response to the command. reply_msg: string &log &optional; - ## Arbitrary tags that may indicate a particular attribute of this command. - tags: set[string] &log; ## Expected FTP data channel. data_channel: ExpectedDataChannel &log &optional; @@ -104,6 +102,8 @@ export { global log_ftp: event(rec: Info); } +@load ./utils + # Add the state tracking information variable to the connection record redef record connection += { ftp: Info &optional; @@ -171,37 +171,26 @@ function set_ftp_session(c: connection) function ftp_message(s: Info) { - # If it either has a tag associated with it (something detected) - # or it's a deliberately logged command. - if ( |s$tags| > 0 || (s?$cmdarg && s$cmdarg$cmd in logged_commands) ) + s$ts=s$cmdarg$ts; + s$command=s$cmdarg$cmd; + + s$arg = s$cmdarg$arg; + if ( s$cmdarg$cmd in file_cmds ) + s$arg = build_url_ftp(s); + + if ( s$arg == "" ) + delete s$arg; + + if ( s?$password && + ! s$capture_password && + to_lower(s$user) !in guest_ids ) { - if ( s?$password && - ! s$capture_password && - to_lower(s$user) !in guest_ids ) - { - s$password = ""; - } - - local arg = s$cmdarg$arg; - if ( s$cmdarg$cmd in file_cmds ) - { - local comp_path = build_path_compressed(s$cwd, arg); - if ( comp_path[0] != "/" ) - comp_path = cat("/", comp_path); - - arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path); - } - - s$ts=s$cmdarg$ts; - s$command=s$cmdarg$cmd; - if ( arg == "" ) - delete s$arg; - else - s$arg=arg; - - Log::write(FTP::LOG, s); + s$password = ""; } + if ( s?$cmdarg && s$command in logged_commands) + Log::write(FTP::LOG, s); + # The MIME and file_size fields are specific to file transfer commands # and may not be used in all commands so they need reset to "blank" # values after logging. @@ -209,8 +198,6 @@ function ftp_message(s: Info) delete s$file_size; # Same with data channel. delete s$data_channel; - # Tags are cleared everytime too. - s$tags = set(); } function add_expected_data_channel(s: Info, chan: ExpectedDataChannel) @@ -218,8 +205,9 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel) s$passive = chan$passive; s$data_channel = chan; ftp_data_expected[chan$resp_h, chan$resp_p] = s; - Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, Analyzer::ANALYZER_FTP_DATA, - 5mins); + Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, + Analyzer::ANALYZER_FTP_DATA, + 5mins); } event ftp_request(c: connection, command: string, arg: string) &priority=5 diff --git a/scripts/base/protocols/ftp/utils.bro b/scripts/base/protocols/ftp/utils.bro new file mode 100644 index 0000000000..8b92a37764 --- /dev/null +++ b/scripts/base/protocols/ftp/utils.bro @@ -0,0 +1,47 @@ +##! Utilities specific for FTP processing. + +@load ./main +@load base/utils/addrs + +module FTP; + +export { + ## Creates a URL from an :bro:type:`FTP::Info` record. + ## + ## rec: An :bro:type:`FTP::Info` record. + ## + ## Returns: A URL, not prefixed by "ftp://". + global build_url: function(rec: Info): string; + + ## Creates a URL from an :bro:type:`FTP::Info` record. + ## + ## rec: An :bro:type:`FTP::Info` record. + ## + ## Returns: A URL prefixed with "ftp://". + global build_url_ftp: function(rec: Info): string; + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; +} + +function build_url(rec: Info): string + { + if ( !rec?$arg ) + return ""; + + local comp_path = build_path_compressed(rec$cwd, rec$arg); + if ( comp_path[0] != "/" ) + comp_path = cat("/", comp_path); + + return fmt("%s%s", addr_to_uri(rec$id$resp_h), comp_path); + } + +function build_url_ftp(rec: Info): string + { + return fmt("ftp://%s", build_url(rec)); + } + +function describe(rec: Info): string + { + return build_url_ftp(rec); + } diff --git a/scripts/base/protocols/http/__load__.bro b/scripts/base/protocols/http/__load__.bro index 8f426c1521..20fcd5f1ec 100644 --- a/scripts/base/protocols/http/__load__.bro +++ b/scripts/base/protocols/http/__load__.bro @@ -1,8 +1,6 @@ @load ./main +@load ./entities @load ./utils -@load ./file-analysis -@load ./file-ident -@load ./file-hash -@load ./file-extract +@load ./files @load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/http/entities.bro b/scripts/base/protocols/http/entities.bro new file mode 100644 index 0000000000..e9376a0c0c --- /dev/null +++ b/scripts/base/protocols/http/entities.bro @@ -0,0 +1,109 @@ +##! Analysis and logging for MIME entities found in HTTP sessions. + +@load base/frameworks/files +@load base/utils/strings +@load base/utils/files +@load ./main + +module HTTP; + +export { + type Entity: record { + ## Filename for the entity if discovered from a header. + filename: string &optional; + }; + + redef record Info += { + ## An ordered vector of file unique IDs. + orig_fuids: vector of string &log &optional; + + ## An ordered vector of mime types. + orig_mime_types: vector of string &log &optional; + + ## An ordered vector of file unique IDs. + resp_fuids: vector of string &log &optional; + + ## An ordered vector of mime types. + resp_mime_types: vector of string &log &optional; + + ## The current entity. + current_entity: Entity &optional; + ## Current number of MIME entities in the HTTP request message body. + orig_mime_depth: count &default=0; + ## Current number of MIME entities in the HTTP response message body. + resp_mime_depth: count &default=0; + }; +} + +event http_begin_entity(c: connection, is_orig: bool) &priority=10 + { + set_state(c, F, is_orig); + + if ( is_orig ) + ++c$http$orig_mime_depth; + else + ++c$http$resp_mime_depth; + + c$http$current_entity = Entity(); + } + +event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3 + { + if ( name == "CONTENT-DISPOSITION" && + /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) + { + c$http$current_entity$filename = extract_filename_from_content_disposition(value); + } + else if ( name == "CONTENT-TYPE" && + /[nN][aA][mM][eE][:blank:]*=/ in value ) + { + c$http$current_entity$filename = extract_filename_from_content_disposition(value); + } + } + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( f$source == "HTTP" && c?$http ) + { + if ( c$http?$current_entity && c$http$current_entity?$filename ) + f$info$filename = c$http$current_entity$filename; + + if ( f$is_orig ) + { + if ( ! c$http?$orig_mime_types ) + c$http$orig_fuids = string_vec(f$id); + else + c$http$orig_fuids[|c$http$orig_fuids|] = f$id; + + if ( f?$mime_type ) + { + if ( ! c$http?$orig_mime_types ) + c$http$orig_mime_types = string_vec(f$mime_type); + else + c$http$orig_mime_types[|c$http$orig_mime_types|] = f$mime_type; + } + } + else + { + if ( ! c$http?$resp_mime_types ) + c$http$resp_fuids = string_vec(f$id); + else + c$http$resp_fuids[|c$http$resp_fuids|] = f$id; + + if ( f?$mime_type ) + { + if ( ! c$http?$resp_mime_types ) + c$http$resp_mime_types = string_vec(f$mime_type); + else + c$http$resp_mime_types[|c$http$resp_mime_types|] = f$mime_type; + } + } + } + + } + +event http_end_entity(c: connection, is_orig: bool) &priority=5 + { + if ( c?$http && c$http?$current_entity ) + delete c$http$current_entity; + } diff --git a/scripts/base/protocols/http/file-analysis.bro b/scripts/base/protocols/http/file-analysis.bro deleted file mode 100644 index d6da8c4f69..0000000000 --- a/scripts/base/protocols/http/file-analysis.bro +++ /dev/null @@ -1,54 +0,0 @@ -@load ./main -@load ./utils -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module HTTP; - -export { - redef record HTTP::Info += { - ## Number of MIME entities in the HTTP request message body so far. - request_mime_level: count &default=0; - ## Number of MIME entities in the HTTP response message body so far. - response_mime_level: count &default=0; - }; - - ## Default file handle provider for HTTP. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -event http_begin_entity(c: connection, is_orig: bool) &priority=5 - { - if ( ! c?$http ) - return; - - if ( is_orig ) - ++c$http$request_mime_level; - else - ++c$http$response_mime_level; - } - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( ! c?$http ) return ""; - - local mime_level: count = - is_orig ? c$http$request_mime_level : c$http$response_mime_level; - local mime_level_str: string = mime_level > 1 ? cat(mime_level) : ""; - - if ( c$http$range_request ) - return cat(Analyzer::ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h, " ", - build_url(c$http)); - - return cat(Analyzer::ANALYZER_HTTP, " ", c$start_time, " ", is_orig, " ", - c$http$trans_depth, mime_level_str, " ", id_string(c$id)); - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_HTTP ) return; - set_file_handle(HTTP::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/http/file-extract.bro b/scripts/base/protocols/http/file-extract.bro deleted file mode 100644 index a8c6039395..0000000000 --- a/scripts/base/protocols/http/file-extract.bro +++ /dev/null @@ -1,100 +0,0 @@ -##! Extracts the items from HTTP traffic, one per file. At this time only -##! the message body from the server can be extracted with this script. - -@load ./main -@load ./file-analysis - -module HTTP; - -export { - ## Pattern of file mime types to extract from HTTP response entity bodies. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## The on-disk prefix for files to be extracted from HTTP entity bodies. - const extraction_prefix = "http-item" &redef; - - redef record Info += { - ## On-disk location where files in request body were extracted. - extracted_request_files: vector of string &log &optional; - - ## On-disk location where files in response body were extracted. - extracted_response_files: vector of string &log &optional; - - ## Indicates if the response body is to be extracted or not. Must be - ## set before or by the first :bro:see:`file_new` for the file content. - extract_file: bool &default=F; - }; -} - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - -function add_extraction_file(c: connection, is_orig: bool, fn: string) - { - if ( is_orig ) - { - if ( ! c$http?$extracted_request_files ) - c$http$extracted_request_files = vector(); - c$http$extracted_request_files[|c$http$extracted_request_files|] = fn; - } - else - { - if ( ! c$http?$extracted_response_files ) - c$http$extracted_response_files = vector(); - c$http$extracted_response_files[|c$http$extracted_response_files|] = fn; - } - } - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$conns ) return; - - local fname: string; - local c: connection; - - if ( f?$mime_type && extract_file_types in f$mime_type ) - { - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - - for ( cid in f$conns ) - { - c = f$conns[cid]; - if ( ! c?$http ) next; - add_extraction_file(c, f$is_orig, fname); - } - - return; - } - - local extracting: bool = F; - - for ( cid in f$conns ) - { - c = f$conns[cid]; - - if ( ! c?$http ) next; - - if ( ! c$http$extract_file ) next; - - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - extracting = T; - break; - } - - if ( extracting ) - for ( cid in f$conns ) - { - c = f$conns[cid]; - if ( ! c?$http ) next; - add_extraction_file(c, f$is_orig, fname); - } - } diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro deleted file mode 100644 index 34d91e45bb..0000000000 --- a/scripts/base/protocols/http/file-hash.bro +++ /dev/null @@ -1,68 +0,0 @@ -##! Calculate hashes for HTTP body transfers. - -@load ./main -@load ./file-analysis - -module HTTP; - -export { - redef record Info += { - ## MD5 sum for a file transferred over HTTP calculated from the - ## response body. - md5: string &log &optional; - - ## This value can be set per-transfer to determine per request - ## if a file should have an MD5 sum generated. It must be - ## set to T at the time of or before the first chunk of body data. - calc_md5: bool &default=F; - }; - - ## Generate MD5 sums for these filetypes. - const generate_md5 = /application\/x-dosexec/ # Windows and DOS executables - | /application\/x-executable/ # *NIX executable binary - &redef; -} - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - - if ( f?$mime_type && generate_md5 in f$mime_type ) - { - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); - return; - } - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$http ) next; - - if ( ! c$http$calc_md5 ) next; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); - return; - } - } - -event file_state_remove(f: fa_file) &priority=4 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$conns ) return; - if ( ! f?$info ) return; - if ( ! f$info?$md5 ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$http ) next; - - c$http$md5 = f$info$md5; - } - } diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro deleted file mode 100644 index 7ed4b58a37..0000000000 --- a/scripts/base/protocols/http/file-ident.bro +++ /dev/null @@ -1,105 +0,0 @@ -##! Identification of file types in HTTP response bodies with file content sniffing. - -@load base/frameworks/notice -@load ./main -@load ./utils -@load ./file-analysis - -module HTTP; - -export { - redef enum Notice::Type += { - ## Indicates when the file extension doesn't seem to match the file - ## contents. - Incorrect_File_Type, - }; - - redef record Info += { - ## Mime type of response body identified by content sniffing. - mime_type: string &log &optional; - }; - - ## Mapping between mime type strings (without character set) and - ## regular expressions for URLs. - ## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the - ## pattern doesn't match the mime type that was discovered. - const mime_types_extensions: table[string] of pattern = { - ["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/, - } &redef; - - ## A pattern for filtering out :bro:enum:`HTTP::Incorrect_File_Type` urls - ## that are not noteworthy before a notice is created. Each - ## pattern added should match the complete URL (the matched URLs include - ## "http://" at the beginning). - const ignored_incorrect_file_type_urls = /^$/ &redef; -} - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$mime_type ) return; - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$http ) next; - - c$http$mime_type = f$mime_type; - - local mime_str: string = c$http$mime_type; - - if ( mime_str !in mime_types_extensions ) next; - if ( ! c$http?$uri ) next; - if ( mime_types_extensions[mime_str] in c$http$uri ) next; - - local url = build_url_http(c$http); - - if ( url == ignored_incorrect_file_type_urls ) next; - - local message = fmt("%s %s %s", mime_str, c$http$method, url); - NOTICE([$note=Incorrect_File_Type, - $msg=message, - $conn=c]); - } - } - -event file_over_new_connection(f: fa_file, c: connection) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$mime_type ) return; - if ( ! c?$http ) return; - - # Spread the mime around (e.g. for partial content, file_type event only - # happens once for the first connection, but if there's subsequent - # connections to transfer the same file, they'll be lacking the mime_type - # field if we don't do this). - c$http$mime_type = f$mime_type; - } - -# Tracks byte-range request / partial content response mime types, indexed -# by [connection, uri] pairs. This is needed because a person can pipeline -# byte-range requests over multiple connections to the same uri. Without -# the tracking, only the first request in the pipeline for each connection -# would get a mime_type field assigned to it (by the FileAnalysis policy hooks). -global partial_types: table[conn_id, string] of string &read_expire=5mins; - -# Priority 4 so that it runs before the handler that will write to http.log. -event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) - &priority=4 - { - if ( ! c$http$range_request ) return; - if ( ! c$http?$uri ) return; - - if ( c$http?$mime_type ) - { - partial_types[c$id, c$http$uri] = c$http$mime_type; - return; - } - - if ( [c$id, c$http$uri] in partial_types ) - c$http$mime_type = partial_types[c$id, c$http$uri]; - } diff --git a/scripts/base/protocols/http/files.bro b/scripts/base/protocols/http/files.bro new file mode 100644 index 0000000000..840b5a2372 --- /dev/null +++ b/scripts/base/protocols/http/files.bro @@ -0,0 +1,56 @@ +@load ./main +@load ./entities +@load ./utils +@load base/utils/conn-ids +@load base/frameworks/files + +module HTTP; + +export { + ## Default file handle provider for HTTP. + global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Default file describer for HTTP. + global describe_file: function(f: fa_file): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + if ( ! c?$http ) + return ""; + + if ( c$http$range_request && ! is_orig ) + { + # Any multipart responses from the server are pieces of same file + # that correspond to range requests, so don't use mime depth to + # identify the file. + return cat(Analyzer::ANALYZER_HTTP, is_orig, c$id$orig_h, build_url(c$http)); + } + else + { + local mime_depth = is_orig ? c$http$orig_mime_depth : c$http$resp_mime_depth; + return cat(Analyzer::ANALYZER_HTTP, c$start_time, is_orig, + c$http$trans_depth, mime_depth, id_string(c$id)); + } + } + +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "HTTP" ) + return ""; + + for ( cid in f$conns ) + { + if ( f$conns[cid]?$http ) + return build_url_http(f$conns[cid]$http); + } + return ""; + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_HTTP, + [$get_file_handle = HTTP::get_file_handle, + $describe = HTTP::describe_file]); + } diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 6d06376183..e103bc6b41 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -1,5 +1,5 @@ -##! Implements base functionality for HTTP analysis. The logging model is -##! to log request/response pairs and all relevant metadata together in +##! Implements base functionality for HTTP analysis. The logging model is +##! to log request/response pairs and all relevant metadata together in ##! a single record. @load base/utils/numbers @@ -15,10 +15,10 @@ export { ## Placeholder. EMPTY }; - + ## This setting changes if passwords used in Basic-Auth are captured or not. const default_capture_password = F &redef; - + type Info: record { ## Timestamp for when the request happened. ts: time &log; @@ -26,7 +26,7 @@ export { uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; - ## Represents the pipelined depth into the connection of this + ## Represents the pipelined depth into the connection of this ## request/response transaction. trans_depth: count &log; ## Verb used in the HTTP request (GET, POST, HEAD, etc.). @@ -60,24 +60,24 @@ export { ## A set of indicators of various attributes discovered and ## related to a particular request/response pair. tags: set[Tags] &log; - + ## Username if basic-auth is performed for the request. username: string &log &optional; ## Password if basic-auth is performed for the request. password: string &log &optional; - + ## Determines if the password will be captured for this request. capture_password: bool &default=default_capture_password; - + ## All of the headers that may indicate if the request was proxied. proxied: set[string] &log &optional; ## Indicates if this request can assume 206 partial content in ## response. - range_request: bool &default=F; + range_request: bool &default=F; }; - - ## Structure to maintain state for an HTTP connection with multiple + + ## Structure to maintain state for an HTTP connection with multiple ## requests and responses. type State: record { ## Pending requests. @@ -87,7 +87,7 @@ export { ## Current response in the pending queue. current_response: count &default=0; }; - + ## A list of HTTP headers typically used to indicate proxied requests. const proxy_headers: set[string] = { "FORWARDED", @@ -100,8 +100,8 @@ export { } &redef; ## A list of HTTP methods. Other methods will generate a weird. Note - ## that the HTTP analyzer will only accept methods consisting solely - ## of letters ``[A-Za-z]``. + ## that the HTTP analyzer will only accept methods consisting solely + ## of letters ``[A-Za-z]``. const http_methods: set[string] = { "GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE", "CONNECT", @@ -111,8 +111,8 @@ export { "POLL", "REPORT", "SUBSCRIBE", "BMOVE", "SEARCH" } &redef; - - ## Event that can be handled to access the HTTP record as it is sent on + + ## Event that can be handled to access the HTTP record as it is sent on ## to the logging framework. global log_http: event(rec: Info); } @@ -147,12 +147,12 @@ function new_http_session(c: connection): Info tmp$ts=network_time(); tmp$uid=c$uid; tmp$id=c$id; - # $current_request is set prior to the Info record creation so we + # $current_request is set prior to the Info record creation so we # can use the value directly here. tmp$trans_depth = c$http_state$current_request; return tmp; } - + function set_state(c: connection, request: bool, is_orig: bool) { if ( ! c?$http_state ) @@ -160,19 +160,19 @@ function set_state(c: connection, request: bool, is_orig: bool) local s: State; c$http_state = s; } - + # These deal with new requests and responses. if ( request || c$http_state$current_request !in c$http_state$pending ) c$http_state$pending[c$http_state$current_request] = new_http_session(c); if ( ! is_orig && c$http_state$current_response !in c$http_state$pending ) c$http_state$pending[c$http_state$current_response] = new_http_session(c); - + if ( is_orig ) c$http = c$http_state$pending[c$http_state$current_request]; else c$http = c$http_state$pending[c$http_state$current_response]; } - + event http_request(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string) &priority=5 { @@ -181,17 +181,17 @@ event http_request(c: connection, method: string, original_URI: string, local s: State; c$http_state = s; } - + ++c$http_state$current_request; set_state(c, T, T); - + c$http$method = method; c$http$uri = unescaped_URI; if ( method !in http_methods ) event conn_weird("unknown_HTTP_method", c, method); } - + event http_reply(c: connection, version: string, code: count, reason: string) &priority=5 { if ( ! c?$http_state ) @@ -199,7 +199,7 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p local s: State; c$http_state = s; } - + # If the last response was an informational 1xx, we're still expecting # the real response to the request, so don't create a new Info record yet. if ( c$http_state$current_response !in c$http_state$pending || @@ -207,7 +207,7 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p ! code_in_range(c$http_state$pending[c$http_state$current_response]$status_code, 100, 199)) ) ++c$http_state$current_response; set_state(c, F, F); - + c$http$status_code = code; c$http$status_msg = reason; if ( code_in_range(code, 100, 199) ) @@ -216,33 +216,33 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p c$http$info_msg = reason; } } - + event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5 { set_state(c, F, is_orig); - + if ( is_orig ) # client headers { if ( name == "REFERER" ) c$http$referrer = value; - + else if ( name == "HOST" ) # The split is done to remove the occasional port value that shows up here. c$http$host = split1(value, /:/)[1]; else if ( name == "RANGE" ) c$http$range_request = T; - + else if ( name == "USER-AGENT" ) c$http$user_agent = value; - + else if ( name in proxy_headers ) { if ( ! c$http?$proxied ) c$http$proxied = set(); add c$http$proxied[fmt("%s -> %s", name, value)]; } - + else if ( name == "AUTHORIZATION" ) { if ( /^[bB][aA][sS][iI][cC] / in value ) @@ -264,25 +264,19 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr } } } - - else # server headers - { - if ( name == "CONTENT-DISPOSITION" && - /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) - c$http$filename = extract_filename_from_content_disposition(value); - } + } - + event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5 { set_state(c, F, is_orig); - + if ( is_orig ) c$http$request_body_len = stat$body_length; else c$http$response_body_len = stat$body_length; } - + event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = -5 { # The reply body is done so we're ready to log. @@ -311,4 +305,4 @@ event connection_state_remove(c: connection) &priority=-5 } } } - + diff --git a/scripts/base/protocols/http/utils.bro b/scripts/base/protocols/http/utils.bro index a74a2fe696..fe8c076780 100644 --- a/scripts/base/protocols/http/utils.bro +++ b/scripts/base/protocols/http/utils.bro @@ -32,6 +32,9 @@ export { ## ## Returns: A URL prefixed with "http://". global build_url_http: function(rec: Info): string; + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; } @@ -62,3 +65,8 @@ function build_url_http(rec: Info): string { return fmt("http://%s", build_url(rec)); } + +function describe(rec: Info): string + { + return build_url_http(rec); + } diff --git a/scripts/base/protocols/irc/__load__.bro b/scripts/base/protocols/irc/__load__.bro index 2e60cda0a6..0834604683 100644 --- a/scripts/base/protocols/irc/__load__.bro +++ b/scripts/base/protocols/irc/__load__.bro @@ -1,5 +1,5 @@ @load ./main @load ./dcc-send -@load ./file-analysis +@load ./files @load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/irc/dcc-send.bro b/scripts/base/protocols/irc/dcc-send.bro index 3194766946..d95eb97517 100644 --- a/scripts/base/protocols/irc/dcc-send.bro +++ b/scripts/base/protocols/irc/dcc-send.bro @@ -2,7 +2,7 @@ ##! ##! There is a major problem with this script in the cluster context because ##! we might see A send B a message that a DCC connection is to be expected, -##! but that connection will actually be between B and C which could be +##! but that connection will actually be between B and C which could be ##! analyzed on a different worker. ##! @@ -15,12 +15,6 @@ module IRC; export { - ## Pattern of file mime types to extract from IRC DCC file transfers. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## On-disk prefix for files to be extracted from IRC DCC file transfers. - const extraction_prefix = "irc-dcc-item" &redef; - redef record Info += { ## DCC filename requested. dcc_file_name: string &log &optional; @@ -28,101 +22,10 @@ export { dcc_file_size: count &log &optional; ## Sniffed mime type of the file. dcc_mime_type: string &log &optional; - - ## The file handle for the file to be extracted - extraction_file: string &log &optional; - - ## A boolean to indicate if the current file transfer should be extracted. - extract_file: bool &default=F; }; } -global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins; - -function set_dcc_mime(f: fa_file) - { - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; - - local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; - - s$dcc_mime_type = f$mime_type; - } - } - -function set_dcc_extraction_file(f: fa_file, filename: string) - { - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; - - local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; - - s$extraction_file = filename; - } - } - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - -# this handler sets the IRC::Info mime type -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "IRC_DATA" ) return; - if ( ! f?$mime_type ) return; - - set_dcc_mime(f); - } - -# this handler check if file extraction is desired -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "IRC_DATA" ) return; - - local fname: string; - - if ( f?$mime_type && extract_file_types in f$mime_type ) - { - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - set_dcc_extraction_file(f, fname); - return; - } - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; - - local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; - - if ( ! s$extract_file ) next; - - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - s$extraction_file = fname; - return; - } - } +global dcc_expected_transfers: table[addr, port] of Info &synchronized &read_expire=5mins; function log_dcc(f: fa_file) { @@ -141,24 +44,21 @@ function log_dcc(f: fa_file) Log::write(IRC::LOG, irc); irc$command = tmp; - # Delete these values in case another DCC transfer + # Delete these values in case another DCC transfer # happens during the IRC session. - delete irc$extract_file; - delete irc$extraction_file; delete irc$dcc_file_name; delete irc$dcc_file_size; delete irc$dcc_mime_type; + delete dcc_expected_transfers[cid$resp_h, cid$resp_p]; return; } } event file_new(f: fa_file) &priority=-5 { - if ( ! f?$source ) return; - if ( f$source != "IRC_DATA" ) return; - - log_dcc(f); + if ( f$source == "IRC_DATA" ) + log_dcc(f); } event irc_dcc_message(c: connection, is_orig: bool, diff --git a/scripts/base/protocols/irc/file-analysis.bro b/scripts/base/protocols/irc/file-analysis.bro deleted file mode 100644 index 89cbe7990c..0000000000 --- a/scripts/base/protocols/irc/file-analysis.bro +++ /dev/null @@ -1,25 +0,0 @@ -@load ./dcc-send.bro -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module IRC; - -export { - ## Default file handle provider for IRC. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( is_orig ) return ""; - return cat(Analyzer::ANALYZER_IRC_DATA, " ", c$start_time, " ", id_string(c$id)); - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_IRC_DATA ) return; - set_file_handle(IRC::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/irc/files.bro b/scripts/base/protocols/irc/files.bro new file mode 100644 index 0000000000..7e077c8331 --- /dev/null +++ b/scripts/base/protocols/irc/files.bro @@ -0,0 +1,39 @@ +@load ./dcc-send +@load base/utils/conn-ids +@load base/frameworks/files + +module IRC; + +export { + redef record Info += { + ## File unique ID. + fuid: string &log &optional; + }; + + ## Default file handle provider for IRC. + global get_file_handle: function(c: connection, is_orig: bool): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + return cat(Analyzer::ANALYZER_IRC_DATA, c$start_time, c$id, is_orig); + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, + [$get_file_handle = IRC::get_file_handle]); + } + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) + return; + + local irc = dcc_expected_transfers[c$id$resp_h, c$id$resp_p]; + irc$fuid = f$id; + if ( irc?$dcc_file_name ) + f$info$filename = irc$dcc_file_name; + if ( f?$mime_type ) + irc$dcc_mime_type = f$mime_type; + } diff --git a/scripts/base/protocols/smtp/__load__.bro b/scripts/base/protocols/smtp/__load__.bro index 3e3fde6947..bf37447ecb 100644 --- a/scripts/base/protocols/smtp/__load__.bro +++ b/scripts/base/protocols/smtp/__load__.bro @@ -1,6 +1,5 @@ @load ./main @load ./entities -@load ./entities-excerpt -@load ./file-analysis +@load ./files @load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/smtp/entities.bro b/scripts/base/protocols/smtp/entities.bro index b58766e51d..d21ba8722d 100644 --- a/scripts/base/protocols/smtp/entities.bro +++ b/scripts/base/protocols/smtp/entities.bro @@ -1,5 +1,6 @@ ##! Analysis and logging for MIME entities found in SMTP sessions. +@load base/frameworks/files @load base/utils/strings @load base/utils/files @load ./main @@ -7,217 +8,55 @@ module SMTP; export { - redef enum Log::ID += { ENTITIES_LOG }; - - type EntityInfo: record { - ## This is the timestamp of when the MIME content transfer began. - ts: time &log; - uid: string &log; - id: conn_id &log; - ## A count to represent the depth of this message transaction in a - ## single connection where multiple messages were transferred. - trans_depth: count &log; - ## The filename seen in the Content-Disposition header. - filename: string &log &optional; - ## Track how many bytes of the MIME encoded file have been seen. - content_len: count &log &default=0; - ## The mime type of the entity discovered through magic bytes identification. - mime_type: string &log &optional; - - ## The calculated MD5 sum for the MIME entity. - md5: string &log &optional; - ## Optionally calculate the file's MD5 sum. Must be set prior to the - ## first data chunk being see in an event. - calc_md5: bool &default=F; - - ## Optionally write the file to disk. Must be set prior to first - ## data chunk being seen in an event. - extract_file: bool &default=F; - ## Store the file handle here for the file currently being extracted. - extraction_file: string &log &optional; + type Entity: record { + ## Filename for the entity if discovered from a header. + filename: string &optional; }; redef record Info += { - ## The in-progress entity information. - current_entity: EntityInfo &optional; + ## The current entity being seen. + entity: Entity &optional; }; redef record State += { - ## Track the number of MIME encoded files transferred during a session. - mime_level: count &default=0; + ## Track the number of MIME encoded files transferred + ## during a session. + mime_depth: count &default=0; }; - - ## Generate MD5 sums for these filetypes. - const generate_md5 = /application\/x-dosexec/ # Windows and DOS executables - | /application\/x-executable/ # *NIX executable binary - &redef; - - ## Pattern of file mime types to extract from MIME bodies. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## The on-disk prefix for files to be extracted from MIME entity bodies. - const extraction_prefix = "smtp-entity" &redef; - - ## If set, never generate MD5s. This is mainly for testing purposes to create - ## reproducable output in the case that the decision whether to create - ## checksums depends on environment specifics. - const never_calc_md5 = F &redef; - - global log_mime: event(rec: EntityInfo); } -event bro_init() &priority=5 - { - Log::create_stream(SMTP::ENTITIES_LOG, [$columns=EntityInfo, $ev=log_mime]); - } - -function set_session(c: connection, new_entity: bool) - { - if ( ! c$smtp?$current_entity || new_entity ) - { - local info: EntityInfo; - info$ts=network_time(); - info$uid=c$uid; - info$id=c$id; - info$trans_depth=c$smtp$trans_depth; - - c$smtp$current_entity = info; - ++c$smtp_state$mime_level; - } - } - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - event mime_begin_entity(c: connection) &priority=10 { - if ( ! c?$smtp ) return; - - set_session(c, T); + c$smtp$entity = Entity(); + ++c$smtp_state$mime_depth; } -event file_new(f: fa_file) &priority=5 +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - if ( ! f?$conns ) return; - - local fname: string; - local extracting: bool = F; - - for ( cid in f$conns ) + if ( f$source == "SMTP" && c?$smtp ) { - local c: connection = f$conns[cid]; - - if ( ! c?$smtp ) next; - if ( ! c$smtp?$current_entity ) next; - - if ( c$smtp$current_entity$extract_file ) - { - if ( ! extracting ) - { - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, - [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - extracting = T; - } - - c$smtp$current_entity$extraction_file = fname; - } - - if ( c$smtp$current_entity$calc_md5 ) - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); + if ( c$smtp?$entity && c$smtp$entity?$filename ) + f$info$filename = c$smtp$entity$filename; + f$info$depth = c$smtp_state$mime_depth; } } -function check_extract_by_type(f: fa_file) +event mime_one_header(c: connection, h: mime_header_rec) &priority=5 { - if ( extract_file_types !in f$mime_type ) return; - - if ( f?$info && FileAnalysis::ANALYZER_EXTRACT in f$info$analyzers ) + if ( ! c?$smtp ) return; - local fname: string = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - if ( ! c?$smtp ) next; - c$smtp$current_entity$extraction_file = fname; - } - } - -function check_md5_by_type(f: fa_file) - { - if ( never_calc_md5 ) return; - if ( generate_md5 !in f$mime_type ) return; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); - } - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - if ( ! f?$mime_type ) return; - - if ( f?$conns ) - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$smtp ) next; - if ( ! c$smtp?$current_entity ) next; - - c$smtp$current_entity$mime_type = f$mime_type; - } - - check_extract_by_type(f); - check_md5_by_type(f); - } - -event file_state_remove(f: fa_file) &priority=4 - { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$smtp ) next; - if ( ! c$smtp?$current_entity ) next; - # Only log if there was some content. - if ( f$seen_bytes == 0 ) next; - - if ( f?$info && f$info?$md5 ) - c$smtp$current_entity$md5 = f$info$md5; - - c$smtp$current_entity$content_len = f$seen_bytes; - Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity); - delete c$smtp$current_entity; - return; - } - } - -event mime_one_header(c: connection, h: mime_header_rec) - { - if ( ! c?$smtp ) return; - if ( h$name == "CONTENT-DISPOSITION" && /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) - c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); + c$smtp$entity$filename = extract_filename_from_content_disposition(h$value); if ( h$name == "CONTENT-TYPE" && /[nN][aA][mM][eE][:blank:]*=/ in h$value ) - c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); + c$smtp$entity$filename = extract_filename_from_content_disposition(h$value); + } + +event mime_end_entity(c: connection) &priority=5 + { + if ( c?$smtp && c$smtp?$entity ) + delete c$smtp$entity; } diff --git a/scripts/base/protocols/smtp/file-analysis.bro b/scripts/base/protocols/smtp/file-analysis.bro deleted file mode 100644 index 68ec6390dd..0000000000 --- a/scripts/base/protocols/smtp/file-analysis.bro +++ /dev/null @@ -1,27 +0,0 @@ -@load ./main -@load ./entities -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module SMTP; - -export { - ## Default file handle provider for SMTP. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( ! c?$smtp ) return ""; - return cat(Analyzer::ANALYZER_SMTP, " ", c$start_time, " ", c$smtp$trans_depth, " ", - c$smtp_state$mime_level); - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_SMTP ) return; - set_file_handle(SMTP::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/smtp/files.bro b/scripts/base/protocols/smtp/files.bro new file mode 100644 index 0000000000..f9ae2ab05f --- /dev/null +++ b/scripts/base/protocols/smtp/files.bro @@ -0,0 +1,53 @@ +@load ./main +@load ./entities +@load base/utils/conn-ids +@load base/frameworks/files + +module SMTP; + +export { + redef record Info += { + ## An ordered vector of file unique IDs seen attached to + ## the message. + fuids: vector of string &log &default=string_vec(); + }; + + ## Default file handle provider for SMTP. + global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Default file describer for SMTP. + global describe_file: function(f: fa_file): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + return cat(Analyzer::ANALYZER_SMTP, c$start_time, c$smtp$trans_depth, + c$smtp_state$mime_depth); + } + +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "SMTP" ) + return ""; + + for ( cid in f$conns ) + { + local c = f$conns[cid]; + return SMTP::describe(c$smtp); + } + return ""; + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_SMTP, + [$get_file_handle = SMTP::get_file_handle, + $describe = SMTP::describe_file]); + } + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( c?$smtp ) + c$smtp$fuids[|c$smtp$fuids|] = f$id; + } diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index 0d510e645d..f54fcdd4c8 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -72,7 +72,10 @@ export { ## ALL_HOSTS - always capture the entire path. ## NO_HOSTS - never capture the path. const mail_path_capture = ALL_HOSTS &redef; - + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; + global log_smtp: event(rec: Info); } @@ -271,3 +274,29 @@ event connection_state_remove(c: connection) &priority=-5 if ( c?$smtp ) smtp_message(c); } + +function describe(rec: Info): string + { + if ( rec?$mailfrom && rec?$rcptto ) + { + local one_to = ""; + for ( to in rec$rcptto ) + { + one_to = to; + break; + } + local abbrev_subject = ""; + if ( rec?$subject ) + { + if ( |rec$subject| > 20 ) + { + abbrev_subject = rec$subject[0:20] + "..."; + } + } + + return fmt("%s -> %s%s%s", rec$mailfrom, one_to, + (|rec$rcptto|>1 ? fmt(" (plus %d others)", |rec$rcptto|-1) : ""), + (abbrev_subject != "" ? fmt(": %s", abbrev_subject) : "")); + } + return ""; + } diff --git a/scripts/base/utils/files.bro b/scripts/base/utils/files.bro index 76d2ecea4f..b88ae5763e 100644 --- a/scripts/base/utils/files.bro +++ b/scripts/base/utils/files.bro @@ -6,22 +6,28 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str { local conn_info = fmt("%s:%d-%s:%d", addr_to_uri(c$id$orig_h), c$id$orig_p, addr_to_uri(c$id$resp_h), c$id$resp_p); - + if ( prefix != "" ) conn_info = fmt("%s_%s", prefix, conn_info); if ( suffix != "" ) conn_info = fmt("%s_%s", conn_info, suffix); - + return conn_info; } - -## For CONTENT-DISPOSITION headers, this function can be used to extract + +## For CONTENT-DISPOSITION headers, this function can be used to extract ## the filename. function extract_filename_from_content_disposition(data: string): string { - local filename = sub(data, /^.*[nN][aA][mM][eE][[:blank:]]*=[[:blank:]]*/, ""); + local filename = sub(data, /^.*[nN][aA][mM][eE][[:blank:]]*\*?=[[:blank:]]*/, ""); + # Remove quotes around the filename if they are there. if ( /^\"/ in filename ) - filename = split_n(filename, /\"/, F, 2)[2]; - return filename; + filename = split_n(filename, /\"/, F, 2)[2]; + + # Remove the language and encoding if it's there. + if ( /^[a-zA-Z0-9\!#$%&+-^_`{}~]+'[a-zA-Z0-9\!#$%&+-^_`{}~]*'/ in filename ) + filename = sub(filename, /^.+'.*'/, ""); + + return unescape_URI(filename); } diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro new file mode 100644 index 0000000000..8a2e33b7f4 --- /dev/null +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -0,0 +1,57 @@ +##! Detect file downloads that have hash values matching files in Team +##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). + +@load base/frameworks/files +@load base/frameworks/notice +@load frameworks/files/hash-all-files + +module TeamCymruMalwareHashRegistry; + +export { + redef enum Notice::Type += { + ## The hash value of a file transferred over HTTP matched in the + ## malware hash registry. + Match + }; + + ## File types to attempt matching against the Malware Hash Registry. + const match_file_types = /application\/x-dosexec/ | + /application\/vnd.ms-cab-compressed/ | + /application\/pdf/ | + /application\/x-shockwave-flash/ | + /application\/x-java-applet/ | + /application\/jar/ | + /video\/mp4/ &redef; + + ## The malware hash registry runs each malware sample through several A/V engines. + ## Team Cymru returns a percentage to indicate how many A/V engines flagged the + ## sample as malicious. This threshold allows you to require a minimum detection + ## rate. + const notice_threshold = 10 &redef; +} + +event file_hash(f: fa_file, kind: string, hash: string) + { + if ( kind=="sha1" && match_file_types in f$mime_type ) + { + local hash_domain = fmt("%s.malware.hash.cymru.com", hash); + when ( local MHR_result = lookup_hostname_txt(hash_domain) ) + { + # Data is returned as " " + local MHR_answer = split1(MHR_result, / /); + if ( |MHR_answer| == 2 ) + { + local mhr_first_detected = double_to_time(to_double(MHR_answer[1])); + local mhr_detect_rate = to_count(MHR_answer[2]); + + local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected); + if ( mhr_detect_rate >= notice_threshold ) + { + local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); + local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); + } + } + } + } + } diff --git a/scripts/policy/frameworks/files/hash-all-files.bro b/scripts/policy/frameworks/files/hash-all-files.bro new file mode 100644 index 0000000000..931857c2bc --- /dev/null +++ b/scripts/policy/frameworks/files/hash-all-files.bro @@ -0,0 +1,7 @@ +# Perform MD5 and SHA1 hashing on all files. + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_MD5); + Files::add_analyzer(f, Files::ANALYZER_SHA1); + } diff --git a/scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro b/scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro index a3ba410641..aa9c322bcf 100644 --- a/scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro +++ b/scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro @@ -1,11 +1,12 @@ @load base/frameworks/intel -@load base/protocols/smtp/file-analysis +@load base/protocols/smtp @load base/utils/urls @load ./where-locations event intel_mime_data(f: fa_file, data: string) { - if ( ! f?$conns ) return; + if ( ! f?$conns ) + return; for ( cid in f$conns ) { @@ -21,11 +22,8 @@ event intel_mime_data(f: fa_file, data: string) } } -event file_new(f: fa_file) &priority=5 +event file_new(f: fa_file) { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_DATA_EVENT, - $stream_event=intel_mime_data]); + if ( f$source == "SMTP" ) + Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT, [$stream_event=intel_mime_data]); } diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro deleted file mode 100644 index 0594276c93..0000000000 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ /dev/null @@ -1,44 +0,0 @@ -##! Detect file downloads over HTTP that have MD5 sums matching files in Team -##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). -##! By default, not all file transfers will have MD5 sums calculated. Read the -##! documentation for the :doc:base/protocols/http/file-hash.bro script to see -##! how to configure which transfers will have hashes calculated. - -@load base/frameworks/notice -@load base/protocols/http - -module HTTP; - -export { - redef enum Notice::Type += { - ## The MD5 sum of a file transferred over HTTP matched in the - ## malware hash registry. - Malware_Hash_Registry_Match - }; - - ## The malware hash registry runs each malware sample through several A/V engines. - ## Team Cymru returns a percentage to indicate how many A/V engines flagged the - ## sample as malicious. This threshold allows you to require a minimum detection - ## rate (default: 50%). - const MHR_threshold = 50 &redef; -} - -event log_http(rec: HTTP::Info) - { - if ( rec?$md5 ) - { - local hash_domain = fmt("%s.malware.hash.cymru.com", rec$md5); - when ( local MHR_result = lookup_hostname_txt(hash_domain) ) - { - # Data is returned as " " - local MHR_answer = split1(MHR_result, / /); - if ( |MHR_answer| == 2 && to_count(MHR_answer[2]) >= MHR_threshold ) - { - local url = HTTP::build_url_http(rec); - local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url); - NOTICE([$note=Malware_Hash_Registry_Match, - $msg=message, $id=rec$id]); - } - } - } - } diff --git a/scripts/base/protocols/smtp/entities-excerpt.bro b/scripts/policy/protocols/smtp/entities-excerpt.bro similarity index 83% rename from scripts/base/protocols/smtp/entities-excerpt.bro rename to scripts/policy/protocols/smtp/entities-excerpt.bro index 1ecd100571..0e596ebdd1 100644 --- a/scripts/base/protocols/smtp/entities-excerpt.bro +++ b/scripts/policy/protocols/smtp/entities-excerpt.bro @@ -1,16 +1,16 @@ ##! This script is for optionally adding a body excerpt to the SMTP ##! entities log. -@load ./entities +@load base/protocols/smtp/entities module SMTP; export { - redef record SMTP::EntityInfo += { + redef record SMTP::Entity+= { ## The entity body excerpt. excerpt: string &log &default=""; }; - + ## This is the default value for how much of the entity body should be ## included for all MIME entities. The lesser of this value and ## :bro:see:`default_file_bof_buffer_size` will be used. @@ -31,7 +31,6 @@ event file_new(f: fa_file) &priority=5 if ( ! c?$smtp ) next; if ( default_entity_excerpt_len > 0 ) - c$smtp$current_entity$excerpt = - f$bof_buffer[0:default_entity_excerpt_len]; + c$smtp$entity$excerpt = f$bof_buffer[0:default_entity_excerpt_len]; } } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index dfebd9923a..e4b3a44e7a 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -64,7 +64,14 @@ # Detect logins using "interesting" hostnames. @load protocols/ssh/interesting-hostnames -# Detect MD5 sums in Team Cymru's Malware Hash Registry. -@load protocols/http/detect-MHR # Detect SQL injection attacks. @load protocols/http/detect-sqli + +#### Network File Handling #### + +# Enable MD5 and SHA1 hashing for all files. +@load frameworks/files/hash-all-files + +# Detect SHA1 sums in Team Cymru's Malware Hash Registry. +@load frameworks/files/detect-MHR + diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 809fc1d1ec..dcf50b538e 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -25,6 +25,8 @@ @load frameworks/intel/seen/smtp.bro @load frameworks/intel/seen/ssl.bro @load frameworks/intel/seen/where-locations.bro +@load frameworks/files/detect-MHR.bro +@load frameworks/files/hash-all-files.bro @load frameworks/packet-filter/shunt.bro @load frameworks/software/version-changes.bro @load frameworks/software/vulnerable.bro @@ -51,7 +53,6 @@ @load protocols/ftp/detect-bruteforcing.bro @load protocols/ftp/detect.bro @load protocols/ftp/software.bro -@load protocols/http/detect-MHR.bro @load protocols/http/detect-sqli.bro @load protocols/http/detect-webapps.bro @load protocols/http/header-names.bro @@ -63,6 +64,7 @@ @load protocols/modbus/track-memmap.bro @load protocols/smtp/blocklists.bro @load protocols/smtp/detect-suspicious-orig.bro +@load protocols/smtp/entities-excerpt.bro @load protocols/smtp/software.bro @load protocols/ssh/detect-bruteforcing.bro @load protocols/ssh/geo-data.bro diff --git a/src/analyzer/analyzer.bif b/src/analyzer/analyzer.bif index 7f3cc6ed94..4d70816075 100644 --- a/src/analyzer/analyzer.bif +++ b/src/analyzer/analyzer.bif @@ -43,3 +43,9 @@ function __name%(atype: Analyzer::Tag%) : string %{ return new StringVal(analyzer_mgr->GetAnalyzerName(atype)); %} + +function __tag%(name: string%) : Analyzer::Tag + %{ + analyzer::Tag t = analyzer_mgr->GetAnalyzerTag(name->CheckString()); + return t.AsEnumVal()->Ref(); + %} diff --git a/src/event.bif b/src/event.bif index 6f363cb961..e4d6f8c844 100644 --- a/src/event.bif +++ b/src/event.bif @@ -897,7 +897,7 @@ event event_queue_flush_point%(%); event get_file_handle%(tag: Analyzer::Tag, c: connection, is_orig: bool%); ## Indicates that an analysis of a new file has begun. The analysis can be -## augmented at this time via :bro:see:`FileAnalysis::add_analyzer`. +## augmented at this time via :bro:see:`Files::add_analyzer`. ## ## f: The file. ## @@ -911,8 +911,10 @@ event file_new%(f: fa_file%); ## ## c: The new connection over which the file is seen being transferred. ## +## is_orig: true if the originator of *c* is the one sending the file. +## ## .. bro:see:: file_new file_timeout file_gap file_state_remove -event file_over_new_connection%(f: fa_file, c: connection%); +event file_over_new_connection%(f: fa_file, c: connection, is_orig: bool%); ## Indicates that file analysis has timed out because no activity was seen ## for the file in a while. @@ -920,8 +922,8 @@ event file_over_new_connection%(f: fa_file, c: connection%); ## f: The file. ## ## .. bro:see:: file_new file_over_new_connection file_gap file_state_remove -## default_file_timeout_interval FileAnalysis::set_timeout_interval -## FileAnalysis::set_timeout_interval +## default_file_timeout_interval Files::set_timeout_interval +## Files::set_timeout_interval event file_timeout%(f: fa_file%); ## Indicates that a chunk of the file is missing. diff --git a/src/file_analysis/Analyzer.h b/src/file_analysis/Analyzer.h index 2589dfbe35..0a5aa9e25c 100644 --- a/src/file_analysis/Analyzer.h +++ b/src/file_analysis/Analyzer.h @@ -96,7 +96,7 @@ public: */ static FA_Tag ArgsTag(const RecordVal* args) { - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; return args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum(); } diff --git a/src/file_analysis/AnalyzerSet.cc b/src/file_analysis/AnalyzerSet.cc index e350e8b0d8..c710d8b085 100644 --- a/src/file_analysis/AnalyzerSet.cc +++ b/src/file_analysis/AnalyzerSet.cc @@ -15,7 +15,7 @@ static void analyzer_del_func(void* v) AnalyzerSet::AnalyzerSet(File* arg_file) : file(arg_file) { TypeList* t = new TypeList(); - t->Append(BifType::Record::FileAnalysis::AnalyzerArgs->Ref()); + t->Append(BifType::Record::Files::AnalyzerArgs->Ref()); analyzer_hash = new CompositeHash(t); Unref(t); analyzer_map.SetDeleteFunc(analyzer_del_func); diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index ee590a23a7..9e44e327e3 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -75,7 +75,8 @@ void File::StaticInit() File::File(const string& file_id, Connection* conn, analyzer::Tag tag, bool is_orig) : id(file_id), val(0), postpone_timeout(false), first_chunk(true), - missed_bof(false), need_reassembly(false), done(false), analyzers(this) + missed_bof(false), need_reassembly(false), done(false), + did_file_new_event(false), analyzers(this) { StaticInit(); @@ -89,7 +90,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag, // add source, connection, is_orig fields SetSource(analyzer_mgr->GetAnalyzerName(tag)); val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL)); - UpdateConnectionFields(conn); + UpdateConnectionFields(conn, is_orig); } UpdateLastActivityTime(); @@ -99,6 +100,13 @@ File::~File() { DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str()); Unref(val); + + // Queue may not be empty in the case where only content gaps were seen. + while ( ! fonc_queue.empty() ) + { + delete_vals(fonc_queue.front().second); + fonc_queue.pop(); + } } void File::UpdateLastActivityTime() @@ -111,18 +119,15 @@ double File::GetLastActivityTime() const return val->Lookup(last_active_idx)->AsTime(); } -void File::UpdateConnectionFields(Connection* conn) +void File::UpdateConnectionFields(Connection* conn, bool is_orig) { if ( ! conn ) return; Val* conns = val->Lookup(conns_idx); - bool is_first = false; - if ( ! conns ) { - is_first = true; conns = empty_connection_table(); val->Assign(conns_idx, conns); } @@ -133,12 +138,18 @@ void File::UpdateConnectionFields(Connection* conn) Val* conn_val = conn->BuildConnVal(); conns->AsTableVal()->Assign(idx, conn_val); - if ( ! is_first && FileEventAvailable(file_over_new_connection) ) + if ( FileEventAvailable(file_over_new_connection) ) { val_list* vl = new val_list(); vl->append(val->Ref()); vl->append(conn_val->Ref()); - FileEvent(file_over_new_connection, vl); + vl->append(new Val(is_orig, TYPE_BOOL)); + + if ( did_file_new_event ) + FileEvent(file_over_new_connection, vl); + else + fonc_queue.push(pair( + file_over_new_connection, vl)); } } @@ -435,6 +446,18 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl) { mgr.QueueEvent(h, vl); + if ( h == file_new ) + { + did_file_new_event = true; + + while ( ! fonc_queue.empty() ) + { + pair p = fonc_queue.front(); + mgr.QueueEvent(p.first, p.second); + fonc_queue.pop(); + } + } + if ( h == file_new || h == file_timeout ) { // immediate feedback is required for these events. diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index ac54c75bc5..794734d24b 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -3,7 +3,9 @@ #ifndef FILE_ANALYSIS_FILE_H #define FILE_ANALYSIS_FILE_H +#include #include +#include #include #include "Conn.h" @@ -171,8 +173,9 @@ protected: * Updates the "conn_ids" and "conn_uids" fields in #val record with the * \c conn_id and UID taken from \a conn. * @param conn the connection over which a part of the file has been seen. + * @param is_orig true if the connection originator is sending the file. */ - void UpdateConnectionFields(Connection* conn); + void UpdateConnectionFields(Connection* conn, bool is_orig); /** * Increment a byte count field of #val record by \a size. @@ -239,7 +242,9 @@ private: bool missed_bof; /**< Flags that we missed start of file. */ bool need_reassembly; /**< Whether file stream reassembly is needed. */ bool done; /**< If this object is about to be deleted. */ + bool did_file_new_event; /**< Whether the file_new event has been done. */ AnalyzerSet analyzers; /**< A set of attached file analyzer. */ + queue > fonc_queue; struct BOF_Buffer { BOF_Buffer() : full(false), replayed(false), size(0) {} diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index a7f7a29c18..7a92e92109 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -19,8 +19,8 @@ string Manager::salt; Manager::Manager() { - tag_enum_type = new EnumType("FileAnalysis::Tag"); - ::ID* id = install_ID("Tag", "FileAnalysis", true, true); + tag_enum_type = new EnumType("Files::Tag"); + ::ID* id = install_ID("Tag", "Files", true, true); add_type(id, tag_enum_type, 0, 0); } @@ -42,7 +42,7 @@ void Manager::RegisterAnalyzerComponent(Component* component) { const char* cname = component->CanonicalName(); - if ( tag_enum_type->Lookup("FileAnalysis", cname) != -1 ) + if ( tag_enum_type->Lookup("Files", cname) != -1 ) reporter->FatalError("File Analyzer %s defined more than once", cname); DBG_LOG(DBG_FILE_ANALYSIS, "Registering analyzer %s (tag %s)", @@ -54,7 +54,7 @@ void Manager::RegisterAnalyzerComponent(Component* component) component->Tag().AsEnumVal()->InternalInt(), component)); string id = fmt("ANALYZER_%s", cname); - tag_enum_type->AddName("FileAnalysis", id.c_str(), + tag_enum_type->AddName("Files", id.c_str(), component->Tag().AsEnumVal()->InternalInt(), true); } @@ -76,7 +76,7 @@ void Manager::Terminate() string Manager::HashHandle(const string& handle) const { if ( salt.empty() ) - salt = BifConst::FileAnalysis::salt->CheckString(); + salt = BifConst::Files::salt->CheckString(); char tmp[20]; uint64 hash[2]; @@ -249,7 +249,7 @@ File* Manager::GetFile(const string& file_id, Connection* conn, rval->UpdateLastActivityTime(); if ( update_conn ) - rval->UpdateConnectionFields(conn); + rval->UpdateConnectionFields(conn, is_orig); } return rval; @@ -351,7 +351,7 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig) bool Manager::IsDisabled(analyzer::Tag tag) { if ( ! disabled ) - disabled = internal_const_val("FileAnalysis::disable")->AsTableVal(); + disabled = internal_const_val("Files::disable")->AsTableVal(); Val* index = new Val(tag, TYPE_COUNT); Val* yield = disabled->Lookup(index); diff --git a/src/file_analysis/analyzer/data_event/DataEvent.cc b/src/file_analysis/analyzer/data_event/DataEvent.cc index 159c8c19cd..1b04111c44 100644 --- a/src/file_analysis/analyzer/data_event/DataEvent.cc +++ b/src/file_analysis/analyzer/data_event/DataEvent.cc @@ -17,7 +17,7 @@ DataEvent::DataEvent(RecordVal* args, File* file, file_analysis::Analyzer* DataEvent::Instantiate(RecordVal* args, File* file) { - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; int chunk_off = AnalyzerArgs->FieldOffset("chunk_event"); int stream_off = AnalyzerArgs->FieldOffset("stream_event"); diff --git a/src/file_analysis/analyzer/extract/Extract.cc b/src/file_analysis/analyzer/extract/Extract.cc index cbe176d4ca..ef37425003 100644 --- a/src/file_analysis/analyzer/extract/Extract.cc +++ b/src/file_analysis/analyzer/extract/Extract.cc @@ -29,7 +29,7 @@ Extract::~Extract() file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file) { - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; Val* v = args->Lookup(AnalyzerArgs->FieldOffset("extract_filename")); if ( ! v ) diff --git a/src/file_analysis/analyzer/hash/events.bif b/src/file_analysis/analyzer/hash/events.bif index b4a8de1c74..e03cbf359a 100644 --- a/src/file_analysis/analyzer/hash/events.bif +++ b/src/file_analysis/analyzer/hash/events.bif @@ -7,6 +7,6 @@ ## ## hash: The result of the hashing. ## -## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5 -## FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256 +## .. bro:see:: Files::add_analyzer Files::ANALYZER_MD5 +## Files::ANALYZER_SHA1 Files::ANALYZER_SHA256 event file_hash%(f: fa_file, kind: string, hash: string%); diff --git a/src/file_analysis/file_analysis.bif b/src/file_analysis/file_analysis.bif index 06ae9450dd..b6c80ac800 100644 --- a/src/file_analysis/file_analysis.bif +++ b/src/file_analysis/file_analysis.bif @@ -1,6 +1,6 @@ ##! Internal functions and types used by the file analysis framework. -module FileAnalysis; +module Files; %%{ #include "file_analysis/Manager.h" @@ -8,40 +8,46 @@ module FileAnalysis; type AnalyzerArgs: record; -## :bro:see:`FileAnalysis::set_timeout_interval`. -function FileAnalysis::__set_timeout_interval%(file_id: string, t: interval%): bool +## :bro:see:`Files::set_timeout_interval`. +function Files::__set_timeout_interval%(file_id: string, t: interval%): bool %{ bool result = file_mgr->SetTimeoutInterval(file_id->CheckString(), t); return new Val(result, TYPE_BOOL); %} -## :bro:see:`FileAnalysis::add_analyzer`. -function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool +## :bro:see:`Files::add_analyzer`. +function Files::__add_analyzer%(file_id: string, args: any%): bool %{ - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); bool result = file_mgr->AddAnalyzer(file_id->CheckString(), rv); Unref(rv); return new Val(result, TYPE_BOOL); %} -## :bro:see:`FileAnalysis::remove_analyzer`. -function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool +## :bro:see:`Files::remove_analyzer`. +function Files::__remove_analyzer%(file_id: string, args: any%): bool %{ - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); bool result = file_mgr->RemoveAnalyzer(file_id->CheckString(), rv); Unref(rv); return new Val(result, TYPE_BOOL); %} -## :bro:see:`FileAnalysis::stop`. -function FileAnalysis::__stop%(file_id: string%): bool +## :bro:see:`Files::stop`. +function Files::__stop%(file_id: string%): bool %{ bool result = file_mgr->IgnoreFile(file_id->CheckString()); return new Val(result, TYPE_BOOL); %} +## :bro:see:`Files::analyzer_name`. +function Files::__analyzer_name%(tag: Files::Tag%) : string + %{ + return new StringVal(file_mgr->GetAnalyzerName(tag->InternalInt())); + %} + module GLOBAL; ## For use within a :bro:see:`get_file_handle` handler to set a unique @@ -58,4 +64,4 @@ function set_file_handle%(handle: string%): any return 0; %} -const FileAnalysis::salt: string; +const Files::salt: string; diff --git a/src/util.cc b/src/util.cc index 23abbacc3f..10a1ad94be 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1593,12 +1593,26 @@ void operator delete[](void* v) #endif +// Being selective of which components of MAGIC_NO_CHECK_BUILTIN are actually +// known to be problematic, but keeping rest of libmagic's builtin checks. +#define DISABLE_LIBMAGIC_BUILTIN_CHECKS ( \ +/* MAGIC_NO_CHECK_COMPRESS | */ \ +/* MAGIC_NO_CHECK_TAR | */ \ +/* MAGIC_NO_CHECK_SOFT | */ \ +/* MAGIC_NO_CHECK_APPTYPE | */ \ +/* MAGIC_NO_CHECK_ELF | */ \ +/* MAGIC_NO_CHECK_TEXT | */ \ + MAGIC_NO_CHECK_CDF | \ + MAGIC_NO_CHECK_TOKENS \ +/* MAGIC_NO_CHECK_ENCODING */ \ +) + void bro_init_magic(magic_t* cookie_ptr, int flags) { if ( ! cookie_ptr || *cookie_ptr ) return; - *cookie_ptr = magic_open(flags|MAGIC_NO_CHECK_TOKENS); + *cookie_ptr = magic_open(flags|DISABLE_LIBMAGIC_BUILTIN_CHECKS); // Use our custom database for mime types, but the default database // from libmagic for the verbose file type. diff --git a/testing/btest/Baseline/core.tunnels.ayiya/http.log b/testing/btest/Baseline/core.tunnels.ayiya/http.log index cd49c4cc89..cc0cf32148 100644 --- a/testing/btest/Baseline/core.tunnels.ayiya/http.log +++ b/testing/btest/Baseline/core.tunnels.ayiya/http.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-20 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html - - - +#open 2013-07-25-21-12-29 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - - - meGKu6goEyd text/html 1257655302.514424 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 2 GET ipv6.google.com /csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - - 1257655303.603569 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 3 GET ipv6.google.com /gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - - -#close 2013-05-21-21-11-20 +#close 2013-07-25-21-12-29 diff --git a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log index e88be88763..8f9ac07c96 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-21 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - - -1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - - -#close 2013-05-21-21-11-21 +#open 2013-07-25-16-23-41 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - - - 6jqjOyeITn5 application/x-shockwave-flash +1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - - - A0xot7xPc22 application/x-shockwave-flash +#close 2013-07-25-16-23-41 diff --git a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log index 8f2893caa7..45b88b7813 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-22 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - application/octet-stream - - - -#close 2013-05-21-21-11-22 +#open 2013-07-25-21-12-32 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - - - oypNlaRdgs7 application/octet-stream +#close 2013-07-25-21-12-32 diff --git a/testing/btest/Baseline/core.tunnels.teredo/http.log b/testing/btest/Baseline/core.tunnels.teredo/http.log index 4e3cdfd61d..1ecf0884e2 100644 --- a/testing/btest/Baseline/core.tunnels.teredo/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo/http.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-21 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain - - - -1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - - -1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - - -1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - application/xml - - - -#close 2013-05-21-21-11-21 +#open 2013-07-25-16-23-17 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - tZX578lAmo3 text/plain - - +1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - - - nkfWSsPnjX7 text/html +1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - - - fk5lVax7K37 text/html +1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - - - 6wF1NFmBUza application/xml +#close 2013-07-25-16-23-17 diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log index 65ec33186e..0c8c448e30 100644 --- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-22 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - - -1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - - -#close 2013-05-21-21-11-22 +#open 2013-07-25-16-22-21 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - - - RzAMHHXJral text/html +1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - - - vOmb3ToMKRg text/html +#close 2013-07-25-16-22-21 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 5879c504e2..0caafdf107 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -85,10 +85,11 @@ scripts/base/init-bare.bro scripts/base/frameworks/analyzer/main.bro scripts/base/frameworks/packet-filter/utils.bro build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/file-analysis/__load__.bro - scripts/base/frameworks/file-analysis/main.bro + scripts/base/frameworks/files/__load__.bro + scripts/base/frameworks/files/main.bro build/scripts/base/bif/file_analysis.bif.bro - build/scripts/base/bif/__load__.bro + scripts/base/utils/site.bro + scripts/base/utils/patterns.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2013-07-25-17-54-33 +#close 2013-07-25-19-59-47 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 2a820f4270..deffbe364b 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-07-25-17-54-33 +#open 2013-07-23-05-48-10 #fields name #types string scripts/base/init-bare.bro @@ -85,13 +85,12 @@ scripts/base/init-bare.bro scripts/base/frameworks/analyzer/main.bro scripts/base/frameworks/packet-filter/utils.bro build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/file-analysis/__load__.bro - scripts/base/frameworks/file-analysis/main.bro + scripts/base/frameworks/files/__load__.bro + scripts/base/frameworks/files/main.bro build/scripts/base/bif/file_analysis.bif.bro - build/scripts/base/bif/__load__.bro + scripts/base/utils/site.bro + scripts/base/utils/patterns.bro scripts/base/init-default.bro - scripts/base/utils/site.bro - scripts/base/utils/patterns.bro scripts/base/utils/addrs.bro scripts/base/utils/conn-ids.bro scripts/base/utils/directions-and-hosts.bro @@ -159,8 +158,8 @@ scripts/base/init-default.bro scripts/base/protocols/ftp/__load__.bro scripts/base/protocols/ftp/utils-commands.bro scripts/base/protocols/ftp/main.bro - scripts/base/protocols/ftp/file-analysis.bro - scripts/base/protocols/ftp/file-extract.bro + scripts/base/protocols/ftp/utils.bro + scripts/base/protocols/ftp/files.bro scripts/base/protocols/ftp/gridftp.bro scripts/base/protocols/ssl/__load__.bro scripts/base/protocols/ssl/consts.bro @@ -168,15 +167,13 @@ scripts/base/init-default.bro scripts/base/protocols/ssl/mozilla-ca-list.bro scripts/base/protocols/http/__load__.bro scripts/base/protocols/http/main.bro + scripts/base/protocols/http/entities.bro scripts/base/protocols/http/utils.bro - scripts/base/protocols/http/file-analysis.bro - scripts/base/protocols/http/file-ident.bro - scripts/base/protocols/http/file-hash.bro - scripts/base/protocols/http/file-extract.bro + scripts/base/protocols/http/files.bro scripts/base/protocols/irc/__load__.bro scripts/base/protocols/irc/main.bro scripts/base/protocols/irc/dcc-send.bro - scripts/base/protocols/irc/file-analysis.bro + scripts/base/protocols/irc/files.bro scripts/base/protocols/modbus/__load__.bro scripts/base/protocols/modbus/consts.bro scripts/base/protocols/modbus/main.bro @@ -184,8 +181,7 @@ scripts/base/init-default.bro scripts/base/protocols/smtp/__load__.bro scripts/base/protocols/smtp/main.bro scripts/base/protocols/smtp/entities.bro - scripts/base/protocols/smtp/entities-excerpt.bro - scripts/base/protocols/smtp/file-analysis.bro + scripts/base/protocols/smtp/files.bro scripts/base/protocols/socks/__load__.bro scripts/base/protocols/socks/consts.bro scripts/base/protocols/socks/main.bro @@ -195,6 +191,10 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/consts.bro scripts/base/protocols/syslog/main.bro scripts/base/protocols/tunnels/__load__.bro + scripts/base/files/hash/__load__.bro + scripts/base/files/hash/main.bro + scripts/base/files/extract/__load__.bro + scripts/base/files/extract/main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-07-25-17-54-33 +#close 2013-07-23-05-48-10 diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index be7e6e5692..dd61de5424 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-32 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170691.550143 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-33 +#open 2013-07-25-21-10-36 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374786635.573905 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-21-10-37 diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index be7e6e5692..dd61de5424 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-32 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170691.550143 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-33 +#open 2013-07-25-21-10-36 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374786635.573905 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-21-10-37 diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index ae693399c3..aebe4dea7b 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-40 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170699.511968 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-41 +#open 2013-07-25-20-26-59 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374784018.898860 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-20-27-00 diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index ae693399c3..b70ba733bd 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-40 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170699.511968 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-41 +#open 2013-07-25-21-05-37 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374786336.338273 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-21-05-38 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out index ddc3449a4c..cbd60840bf 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION file_stream, file #0, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea file_chunk, file #0, 1500, 0, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea file_stream, file #0, 1024, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out index 4b572d5df9..eb62690f91 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout index 160a51a543..e78f5c8c17 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout @@ -2,6 +2,7 @@ FILE_NEW file #0, 0, 0 MIME_TYPE application/x-dosexec +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] @@ -11,6 +12,7 @@ FILE_NEW file #1, 0, 0 MIME_TYPE application/octet-stream +FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_TIMEOUT FILE_STATE_REMOVE diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out index f7182027aa..13cfe5de58 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out @@ -4,3 +4,4 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out index 4463db6958..eba43b94a4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER The Nationa MIME_TYPE text/x-pascal +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 16557, 0 [orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out index 2b46d02042..d42db4b90a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER {^J "origin MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 197, 0 [orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out index bb2f622969..219aad4eff 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out index 4b6fa76c0c..da42f4fd68 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER test^M^J MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 6, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] @@ -17,6 +18,7 @@ FILE_BOF_BUFFER test2^M^J MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 7, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] @@ -30,6 +32,7 @@ FILE_BOF_BUFFER test3^M^J MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #2, 7, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] @@ -43,6 +46,7 @@ FILE_BOF_BUFFER {^J "data": MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #3, 465, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out index f8f2538e92..077fb5282c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out @@ -3,6 +3,7 @@ file #0, 0, 0 MIME_TYPE application/pdf FILE_OVER_NEW_CONNECTION +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 555523, 0 [orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out index b2a0cb66a2..9c05f311f3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out @@ -2,6 +2,7 @@ FILE_NEW file #0, 0, 0 MIME_TYPE application/x-dosexec +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] @@ -11,6 +12,7 @@ FILE_NEW file #1, 0, 0 MIME_TYPE application/octet-stream +FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_STATE_REMOVE file #1, 206024, 0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out index 7c5e9dfeca..d85a9de314 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out @@ -3,6 +3,7 @@ file #0, 0, 0 MIME_TYPE application/octet-stream FILE_OVER_NEW_CONNECTION +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 498702, 0 [orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out index 02ac2f0a7e..b85485cd1a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER /*^J******** MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 2675, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -17,6 +18,7 @@ FILE_BOF_BUFFER //-- Google MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 21421, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -30,6 +32,7 @@ FILE_BOF_BUFFER GIF89a^D\0^D\0\xb3 MIME_TYPE image/gif +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #2, 94, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -44,6 +47,7 @@ FILE_BOF_BUFFER \x89PNG^M^J^Z^J\0\0\0 MIME_TYPE image/png +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #3, 2349, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -58,6 +62,7 @@ FILE_BOF_BUFFER \x89PNG^M^J^Z^J\0\0\0 MIME_TYPE image/png +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #4, 27579, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out index 3103ecb39e..cedc396254 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER hello world MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 11, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] @@ -18,6 +19,7 @@ FILE_BOF_BUFFER {^J "origin MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 366, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out index 36da7bdeed..fcd30b2253 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out @@ -4,6 +4,21 @@ FILE_BOF_BUFFER PK^C^D^T\0\0\0^H\0\xae MIME_TYPE application/zip +FILE_OVER_NEW_CONNECTION +FILE_NEW +file #1, 0, 0 +FILE_BOF_BUFFER +\0\0^Ex\0\0^J\xf0\0\0^P +MIME_TYPE +application/octet-stream +FILE_OVER_NEW_CONNECTION +FILE_STATE_REMOVE +file #1, 124, 0 +[orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] +source: IRC_DATA +MD5: 35288fd50a74c7d675909ff83424d7a1 +SHA1: 8a98f177cb47e6bf771bf57c2f7e94c4b5e79ffa +SHA256: b24dde52b933a0d76e885ab418cb6d697b14a4e2fef45fce66e12ecc5a6a81aa FILE_STATE_REMOVE file #0, 42208, 0 [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log deleted file mode 100644 index f95a70d50a..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path file_analysis -#open 2013-06-07-18-51-45 -#fields id parent_id source is_orig last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size mime_type timedout conn_uids extracted_files md5 sha1 sha256 -#types string string string bool time count count count count interval count string bool table[string] table[string] string string string -BYYd1GSNX5c - HTTP F 1362692527.009775 4705 4705 0 0 120.000000 1024 text/plain F UWkUyAuUGXf BYYd1GSNX5c-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 -#close 2013-06-07-18-51-46 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log new file mode 100644 index 0000000000..2663184b88 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path files +#open 2013-07-25-16-57-31 +#fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted +#types time string table[addr] table[addr] table[string] string count table[string] string string interval bool bool count count count count bool string string string string string +1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file +#close 2013-07-25-16-57-31 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out index ac4e6e50fa..57f1f97b9c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER Hello^M^J^M^J ^M MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 79, 0 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] @@ -17,6 +18,7 @@ FILE_BOF_BUFFER pub/NetBSD/README.export-control -lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control -total 98028 -total 98028 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log deleted file mode 100644 index e77f59dc44..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log +++ /dev/null @@ -1,21 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path ftp -#open 2013-06-07-18-57-22 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file -#types time string addr port addr port string string string string string count count string table[string] bool addr addr port string -1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) (empty) T 141.142.220.235 199.233.217.249 56666 - -1329843175.791528 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - 226 Transfer complete. (empty) - - - - - -1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) (empty) T 141.142.220.235 199.233.217.249 56667 - -1329843193.984222 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 - - - - - - - (empty) - - - - ftp-item-pVhQhhFsB2b.dat -1329843193.984222 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 - - - - - - - (empty) - - - - ftp-item-fFCPkV1sEsc.dat -1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 33582 - -1329843194.095782 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - 226 Transfer complete. (empty) - - - - - -1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 37835 - -1329843199.968212 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 - - - - - - - (empty) - - - - ftp-item-g3zS3MuJFh.dat -1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -1329843200.079930 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 - - - - - - - (empty) - - - - ftp-item-lMf4UWRkEO5.dat -#close 2013-06-07-18-57-22 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log index afa4c97830..4cc6d67761 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log @@ -3,13 +3,13 @@ #empty_field (empty) #unset_field - #path ftp -#open 2013-04-12-16-32-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file -#types time string addr port addr port string string string string string count count string table[string] bool addr addr port string -1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) (empty) T 141.142.220.235 199.233.217.249 56666 - -1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) (empty) T 141.142.220.235 199.233.217.249 56667 - -1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 33582 - -1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 37835 - -1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -#close 2013-04-12-16-32-25 +#open 2013-07-27-01-49-02 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid +#types time string addr port addr port string string string string string count count string bool addr addr port string +1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) T 141.142.220.235 199.233.217.249 56666 - +1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) T 141.142.220.235 199.233.217.249 56667 - +1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - 4VAnSiNGSQh +1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 33582 4VAnSiNGSQh +1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 37835 4VAnSiNGSQh +1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - aJg8mtdsS86 +#close 2013-07-27-01-49-02 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log index 85207806c4..d6f57bcf45 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log @@ -3,14 +3,14 @@ #empty_field (empty) #unset_field - #path ftp -#open 2013-04-12-16-32-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file -#types time string addr port addr port string string string string string count count string table[string] bool addr addr port string -1329327783.207785 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57086|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57086 - -1329327786.415755 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57087|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57087 - -1329327787.180814 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57088|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57088 - -1329327787.396984 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. (empty) - - - - - -1329327795.355248 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189| - - 200 EPRT command successful. (empty) F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49189 - -1329327795.463946 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. (empty) - - - - - -1329327799.799327 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190| - - 200 EPRT command successful. (empty) F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49190 - -#close 2013-04-12-16-32-25 +#open 2013-07-27-01-49-13 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid +#types time string addr port addr port string string string string string count count string bool addr addr port string +1329327783.207785 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57086|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57086 - +1329327786.415755 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57087|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57087 - +1329327787.180814 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57088|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57088 - +1329327787.396984 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. - - - - - +1329327795.355248 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49189 4YhNtGvCehl +1329327795.463946 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. - - - - 4YhNtGvCehl +1329327799.799327 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49190 4YhNtGvCehl +#close 2013-07-27-01-49-13 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log index 051f1c6266..04c80407f6 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2013-04-02-02-19-21 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double -1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2013-04-02-02-19-21 +#open 2013-07-23-05-19-25 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double +1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 - - - tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-07-23-05-19-25 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index edbee28991..a81c0d4a2d 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-24 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - - -#close 2013-05-21-21-11-24 +#open 2013-07-25-19-39-08 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - 8TXBHVmBGD7 text/plain ATGo7hdUXdi text/html +#close 2013-07-25-19-39-08 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item.dat b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item.dat deleted file mode 100644 index 73c369dd14..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item.dat +++ /dev/null @@ -1,304 +0,0 @@ - -ICIR - -ICIR
-

-ICIR (The ICSI Center for Internet Research) -is a -non-profit -research institute at -ICSI -in -Berkeley, -California.
-For the three years from 1999 to 2001 we were named -ACIRI, the AT&T Center for Internet Research at ICSI, -and were funded by AT&T.
- -The goals of ICIR are to: -

    -
  • Pursue research on the Internet architecture and related networking issues, -
  • -Participate actively in the research (SIGCOMM and IRTF) and -standards (IETF) communities, -
  • Bridge the gap between the Internet research community and commercial -interests by providing a neutral forum where topics of mutual technical -interest can be addressed. -
-

- -

- -
- - - - - - - - - - -
- -

-People -

- - -		 - -

-Publications -

- - -

-Projects -

- - - -		 - -

Research

-   Transport and Congestion - - -   Traffic and Topology -
    -
  • -IDMaps -(Internet Distance Mapping). -
  • The -Internet Traffic Archive. -
  • -MINC -(Multicast-based Inference of Network-internal Characteristics). -
  • -NIMI -(National Internet Measurement Infrastructure). -
- -

- -Collaborators -

- - - -
-
- -
-

Information for visitors and local users.

-
-Last modified: June 2004. Copyright notice. - -Older versions of this web page, in its ACIRI incarnation.. -
-For more information about this server, mail www@aciri.org. -
-To report unusual activity by any of our hosts, mail abuse@aciri.org. - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log deleted file mode 100644 index 53b80e5e9e..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path http -#open 2013-06-07-19-04-27 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - http-item-54zlJFqn0x6.dat -#close 2013-06-07-19-04-27 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log index 54a75f4697..674e355631 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log @@ -3,56 +3,56 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - text/html - - - -1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - text/html - - - -1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - text/html - - - -1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - text/html - - - -1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +#open 2013-07-25-19-41-27 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - - - VTrFjxi3V27 text/html +1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - jeoiUX9q8v9 text/html +1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 6dL7NPgFhil text/html +1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - cix6gzDRCob text/html +1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - - - tCZHDKUkBdi text/html +1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - - - iVzFNTeQnnc text/html +1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - - - boBAqw2JcFi text/html +1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - r3w183FJvW3 text/html +1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - bncugeoItlf text/html +1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - - - NkYD5vo8Gy text/html +1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - - - S85THffBTLh text/html +1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 2m6kUZZS0wd text/html +1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - UoqtpOgJZSk text/html +1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - mqs8p4wwsS7 text/html +1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - S36eCQJUY5k text/html +1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - - - LeNRDWYrpS7 text/html +1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - ZwKUASlWzYk text/html +1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - uj62KNQhsG3 text/html +1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - taBxWzrYquk text/html +1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - - - bHBxZULKI0k text/html +1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - - - t6k8zHaGZk5 text/html +1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - c11un7ZO6nc text/html +1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - iWCHzW5XJWk text/html +1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - dzvHktkjD9a text/html +1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - - - vEO9iYqh3Zc text/html +1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - - - 8seYaeRVuV2 text/html +1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - 0kkHkmLHFl3 text/html +1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - koHEYsvMVBa text/html +1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 50tlwxQjBCb text/html +1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - - - DdECXqOZjXh text/html +1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - - - LIZQeBP0Coi text/html +1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - hjPo0BdP973 text/html +1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - d6K2onvteNa text/html +1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - BY1g634OMv6 text/html +1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - - - 5aAa2m40fZd text/html +1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - - - y3Syn85ve8e text/html +1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - - - P92nMD5z6D4 text/html +1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - qIPObDBIhSj text/html +1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - su86MWxyjne text/html +1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - - - r2aysGE6ve8 text/html +1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - - - Zosv3c0p2Zb text/html +1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - - - L02QmCl2lX4 text/html +1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - uh9TwTMdWI9 text/html +1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 4gLQ9WVkuYd text/html 1354328924.559704 E490YZTUozc 128.2.6.136 46606 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - - 1354328928.625437 YIeWJmXWNWj 128.2.6.136 46607 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - - 1354328932.692706 ydiZblvsYri 128.2.6.136 46608 173.194.75.103 80 1 HEAD www.google.com /HTTP/1.1 - - 0 0 400 Bad Request - - - (empty) - - - - - - - -1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -#close 2013-05-21-21-11-25 +1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - NIV5LGdqSk2 text/html +1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - SlC7NZIgx1d text/html +#close 2013-07-25-19-41-27 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log deleted file mode 100644 index 97e797b4fb..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ /dev/null @@ -1,14 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path http -#open 2013-05-21-21-11-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - text/plain - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - text/plain - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - image/gif - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - - -#close 2013-05-21-21-11-25 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index e22fb53103..6779485f91 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string vector[string] vector[string] -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - - -#close 2013-05-21-21-11-25 +#open 2013-07-25-19-43-06 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - XRu8VItOvLc text/plain +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - m1D1wMxW9y8 text/plain +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - ZwnCaxWANNb image/gif +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - 3WVi9g0Caei image/png +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - ta9bGBff1Wl image/png +#close 2013-07-25-19-43-06 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log b/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log index 0bd15badef..ae71680dfa 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-06-07-19-57-15 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369159408.455878 UWkUyAuUGXf 141.142.228.5 57262 54.243.88.146 80 1 POST httpbin.org /post - curl/7.30.0 370 465 200 OK - - - (empty) - - - text/plain - http-item-lcf92jVphSl.dat,http-item-z8gOS6arddh.dat,http-item-tBYz7eElzTb.dat http-item-GVJrSB2Vxk6.dat -#close 2013-06-07-19-57-15 +#open 2013-07-25-19-50-23 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1369159408.455878 UWkUyAuUGXf 141.142.228.5 57262 54.243.88.146 80 1 POST httpbin.org /post - curl/7.30.0 370 465 200 OK - - - (empty) - - - UB09X6VFGTd,wFP689pOsIa,g5yDIGBH4i5 text/plain,text/plain,text/plain yv4qm3EsdOc text/plain +#close 2013-07-25-19-50-23 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index 64bdb41861..8249c94938 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path irc -#open 2013-03-27-18-51-40 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size extraction_file +#open 2013-07-25-19-51-43 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size fuid #types time string addr port addr port string string string string string string count string 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - 1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - -#close 2013-03-27-18-51-40 +#close 2013-07-25-19-51-43 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item.dat b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item.dat deleted file mode 100644 index d4ec9e374b..0000000000 Binary files a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item.dat and /dev/null differ diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log deleted file mode 100644 index 28ca448e05..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ /dev/null @@ -1,13 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path irc -#open 2013-06-07-19-08-42 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size dcc_mime_type extraction_file -#types time string addr port addr port string string string string string string count string string -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 application/zip irc-dcc-item-A3OSdqG9zvk.dat -#close 2013-06-07-19-08-42 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log index ba16578dfb..b56b8afab6 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path smtp -#open 2009-10-05-06-06-12 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent -#types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string -1254722768.219663 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 -#close 2009-10-05-06-06-16 +#open 2013-07-25-19-52-35 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent fuids +#types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string vector[string] +1254722768.219663 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 A1IqG95k9Tk,VUcocHqaWva,JJPHrvZaGJj +#close 2013-07-25-19-52-35 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions deleted file mode 100644 index 45d776a8e9..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions +++ /dev/null @@ -1,277 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2) - "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of - #included directly or indirectly)! - (available when right-clicking the class-browser - (still can be overriden by using "-c" command line parameter). - (the same filename as the project's but with extension ".layout"). If you - - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow) - - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20 - - Strip executable - -c - 20 - Instead open the file in an already launched Dev-C++. - It used to be a checkbox, allowing only two states (on or off), but there is - The user can define this in the class browser's context menu under "View mode". - Well, it adds caching to code-completion. Depending on the cache size, - a third relevant option now: "Project classes" so it didn't fit the purpose... - and selecting "View mode"). - cause of many errors (although it should be fixed by now), we are giving the - class inheritance and visibility (shows items only from files - code-completion and the user has all the commands (belonging to the files - compiler: -D__DEBUG__ - displayed in the editor when the mouse moves over a word. Since this was the - have your project under CVS control, you ''ll know why this had to happen... - he added in the cache) at his fingertips. If, for example, the user adds - include files can now be generated. - information definitions - it in the private resource) - its counterpart are highlighted - resource) - the program may take a bit longer to start-up, but provides very fast - the requested CVS action - then we even get a stack trace in the bug report! - user the option to disable this feature. - wxWindows! You get the picture... -* "Build priority" per-unit -* "Compile as C++" per-unit -* "Default" button in Compiler Options is back -* "Include file in compilation process" per-unit -* "Include file in linking process" per-unit -* Added "Add Library" button in Project Options -* Added "Classes" toolbar -* Added "External programs" in Tools/Environment Options (for units "Open with") -* Added "Files" tab in CVS form to allow selection of more than one file for -* Added "Open with" in project units context menu -* Added "Tip of the day" system. -* Added *working* function arguments hint -* Added CVS "login" and "logout" commands -* Added CVS commands "Add" and "Remove" -* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory -* Added bracket highlighting. When the caret is on a bracket, that bracket and -* Added configuration option for "Templates Directory" in "Environment Options" -* Added display of project filename, project output and a summary of the project files in Project Options General tab. -* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards -* Added file's date/time stamp in File/Properties window -* Added new WebUpdate module (inactive temporarily). -* Added new code for code-completion caching of files (disabled - work in progress). -* Added new compiler/linker options: 20 -* Added new file menu entry: Save Project As -* Added new option in class-browser: Use colors -* Added possibility to include in a Template the Project's directories (include, libs and ressources) -* Added support for GCC > 3.2 -* Added support for macros in the "default source code" (Tools/Editor Options/Code) -* Added support for the "interface" keyword -* Added support for the '::' member access operator in code-completion -* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value") -* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup -* Added two new macros: and -* Allow customizing of per-unit compile command in projects -* Allow user to specify an alternate configuration file in Environment Options 20 -* Backtrace in debugging -* Big speed up in function parameters listing while editing -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug-fix for double quotes in devcpp.cfg file read by vUpdate -* CPU Window (still in development) -* CVS support -* Caching of result set of code-completion for speed-up. -* Changed position of compiler/linker parameters in Project Options. -* Changed tint of Class browser pictures colors to match the New Look style -* Class-parser speed-up (50% to 85% improvement timed!!!) -* Code-completion updates -* Compiler set per-project -* Compiler settings per-project -* Compiling progress window -* Current windows listing in Window menu -* Debug variable browser -* Debug variables are now resent during next debug session -* Dev-C++ now traps access violation of your programs (and of itself too ;) -* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features. -* Each project creates a _private.h file containing version -* Editor colors are initialized properly on Dev-C++ first-run -* Environment options : "Show progress window" and "Auto-close progress window" -* Error messages parsing improved -* Fixed many class browser bugs, including some that had to do with class folders. -* Fixed pre-compilation dependency checks to work correctly -* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0 -* Fixed the dreaded "Clock skew detected" compiler warning! -* Folders in Project and Class Browser -* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2) -* Implemented new compiler settings framework -* Implemented search in help files for the word at cursor (context sensitive help) -* Implemented the "File/Export/Project to HTML" function. -* Improved Indent/Unindent and Remove Comment -* Improved WebUpdate module -* Improved automatic indent -* Improved code completion cache -* Improved editor -* Improved help file -* Improved installer -* Lots of bug fixes. -* Lots of bugfixes -* MSVC import now creates the folders structure of the original VC project -* Made whole bottom report control floating instead of only debug output. -* Makefile can now be customized. -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many code-completion updates. Now takes into account context, -* Modified the behaviour of the -c param : 20 -* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings) -* Necessary UI changes in Project Options -* Nested folders in project view -* New "Abort compilation" button -* New WebUpdater module. -* New class browser option: "Show inherited members" -* New code tooltip display -* New debug feature for DLLs: attach to a running process -* New environment options : "watch variable under mouse" and "Report watch errors" -* New feature: compile current file only -* New option "Execution/Parameters" (and "Debug/Parameters"). -* New option in Editor Options (code-completion): Use code-completion cache. -* New option in Editor Options: Show editor hints. User can disable the hints -* New project option: Use custom Makefile. 20 -* New splash screen and association icons -* Now checks for vRoach existance when sending a crash report -* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20 -* Other bug fixes -* Possibility of changing compilers and tools filename. -* Printing settings are now saved -* Profiling support -* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows. -* Project version info (creates the relevant VERSIONINFO struct in the private -* Removed "Only show classes from current file" option in class browser settings. -* Resource errors are now reported in the Resource sheet -* Resource files are treated as ordinary files now -* Run to cursor -* Saving of custom syntax parameter group -* Send custom commands to GDB -* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744) -* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744) -* Separated layout info from project file. It is now kept in a different file -* Support XP Themes (creates the CommonControls 6.0 manifest file and includes -* Support for latest Mingw compiler system builds -* ToDo list -* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data) -* Updates in "Project Options/Files" code -* Watched Variables not in correct context are now kept and updated when it is needed -* WebUpdate should now report installation problems from PackMan -* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP -* When adding debugging symbols on request, remove "-s" option from linker -* When compiling the current file only, no dependency checks are performed -* When compiling with debugging symbols, an extra definition is passed to the -* When creating a DLL, the created static lib respects now the project-defined output directory -* When running a source file in explorer, don't spawn new instance. -* Window list (in Window menu) -* XP Theme support -* added ENTER key for opening file in project browser, DEL to delete from the project. -* back to gcc 2.95.3 -* bug fixes -* bug fixes -* new update/packages checker (vUpdate) -* support for DLL application hosting, for debugging and executing DLLs under Dev-C++. -* ~300% Speed-up in class parser -Find the attachment -GPS -Hello -I send u smtp pcap file -Version 4.9.4.1 (5.0 beta 4.1): -Version 4.9.5.0 (5.0 beta 5): -Version 4.9.5.1 -Version 4.9.5.2 -Version 4.9.5.3 -Version 4.9.5.4 -Version 4.9.5.5 -Version 4.9.6.5 -Version 4.9.6.6 -Version 4.9.6.7 -Version 4.9.6.8 -Version 4.9.6.9 -Version 4.9.7.0 -Version 4.9.7.1 -Version 4.9.7.2 -Version 4.9.7.3 -Version 4.9.7.4 -Version 4.9.7.5 -Version 4.9.7.6 -Version 4.9.7.7 -Version 4.9.7.8 -Version 4.9.7.9 -Version 4.9.8.0 -Version 4.9.8.1 -Version 4.9.8.2 -Version 4.9.8.3 -Version 4.9.8.4 -Version 4.9.8.5 -Version 4.9.8.7 -Version 4.9.8.9 -Version 4.9.9.0 -Version 4.9.9.1 -version 4.9.6.1 -version 4.9.6.2 -version 4.9.6.3 -version 4.9.6.4 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount deleted file mode 100644 index 0cfbf08886..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log deleted file mode 100644 index 865694e8a2..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log +++ /dev/null @@ -1,12 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path smtp_entities -#open 2013-06-07-19-32-56 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt -#types time string addr port addr port count string count string string string string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain - smtp-entity-mR3f2AAKo11.dat (empty) -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 text/html - - (empty) -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain - smtp-entity-ZNp0KBSLByc.dat (empty) -#close 2013-06-07-19-32-56 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log deleted file mode 100644 index 135c644855..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log +++ /dev/null @@ -1,12 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path smtp_entities -#open 2013-03-26-20-39-07 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt -#types time string addr port addr port count string count string string string string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain 92bca2e6cdcde73647125da7dccbdd07 - (empty) -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 text/html - - (empty) -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain a968bb0f9f9d95835b2e74c845877e87 - (empty) -#close 2013-03-26-20-39-07 diff --git a/testing/btest/Baseline/scripts.base.utils.files/output b/testing/btest/Baseline/scripts.base.utils.files/output index ab92c3a624..4d53bcedd3 100644 --- a/testing/btest/Baseline/scripts.base.utils.files/output +++ b/testing/btest/Baseline/scripts.base.utils.files/output @@ -1,3 +1,6 @@ +Economy +US-$ rates +\xa3 rates test-prefix_141.142.220.118:48649-208.80.152.118:80_test-suffix test-prefix_141.142.220.118:48649-208.80.152.118:80 141.142.220.118:48649-208.80.152.118:80_test-suffix diff --git a/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log b/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log index f2cf09cab6..54b04aafae 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log +++ b/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path notice -#open 2013-04-28-22-36-26 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double -1367188586.649122 - - - - - - Software::Vulnerable_Version 1.2.3.4 is running Java 1.7.0.15 which is vulnerable. Java 1.7.0.15 1.2.3.4 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - -1367188586.649122 - - - - - - Software::Vulnerable_Version 1.2.3.5 is running Java 1.6.0.43 which is vulnerable. Java 1.6.0.43 1.2.3.5 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2013-04-28-22-36-26 +#open 2013-07-25-19-54-45 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double +1374782085.726121 - - - - - - - - - Software::Vulnerable_Version 1.2.3.4 is running Java 1.7.0.15 which is vulnerable. Java 1.7.0.15 1.2.3.4 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +1374782085.726121 - - - - - - - - - Software::Vulnerable_Version 1.2.3.5 is running Java 1.6.0.43 which is vulnerable. Java 1.6.0.43 1.2.3.5 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-07-25-19-54-45 diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 4a13833094..7ccf99eea8 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -7,7 +7,7 @@ IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` -BROMAGIC=%(testbase)s/../../magic +BROMAGIC=%(testbase)s/../../magic/database BRO_SEED_FILE=%(testbase)s/random.seed TZ=UTC LC_ALL=C diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index 249ebc3754..d227417c15 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -41,16 +41,25 @@ redef ssl_ca_certificate = "../ca_cert.pem"; redef ssl_private_key = "../bro.pem"; redef ssl_passphrase = "my-password"; +# Make sure the HTTP connection really gets out. +# (We still miss one final connection event because we shutdown before +# it gets propagated but that's ok.) +redef tcp_close_delay = 0secs; + # File-analysis fields in http.log won't get set on receiver side correctly, # one problem is with the way serialization may send a unique ID in place # of a full value and expect the remote side to associate that unique ID with -# a value it received at an earlier time. So sometimes modifications the sender -# makes to the value aren't seen on the receiver (in this case, the mime_type -# field). -event file_new(f: fa_file) &priority=10 +# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver. +function myfh(c: connection, is_orig: bool): string { - delete f$mime_type; - FileAnalysis::stop(f); + return ""; + } + +event bro_init() + { + # Ignore all http files. + Files::register_protocol(Analyzer::ANALYZER_HTTP, + [$get_file_handle = myfh]); } @TEST-END-FILE diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index 21f46cf4b3..1edf14fee7 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -39,12 +39,17 @@ redef tcp_close_delay = 0secs; # File-analysis fields in http.log won't get set on receiver side correctly, # one problem is with the way serialization may send a unique ID in place # of a full value and expect the remote side to associate that unique ID with -# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver (in this case, the mime_type -# field). -event file_new(f: fa_file) &priority=10 +# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver. +function myfh(c: connection, is_orig: bool): string { - delete f$mime_type; - FileAnalysis::stop(f); + return ""; + } + +event bro_init() + { + # Ignore all http files. + Files::register_protocol(Analyzer::ANALYZER_HTTP, + [$get_file_handle = myfh]); } @TEST-END-FILE diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro index 1f15a4221f..a3704618bd 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro @@ -11,8 +11,8 @@ redef test_get_file_name = function(f: fa_file): string event file_new(f: fa_file) &priority=-10 { for ( tag in test_file_analyzers ) - FileAnalysis::remove_analyzer(f, tag); + Files::remove_analyzer(f, tag); local filename = test_get_file_name(f); - FileAnalysis::remove_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=filename]); + Files::remove_analyzer(f, Files::ANALYZER_EXTRACT, + [$extract_filename=filename]); } diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro index 8ec4704cdb..c9eac4c31d 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro @@ -20,7 +20,7 @@ redef default_file_timeout_interval = 2sec; event file_timeout(f: fa_file) { if ( timeout_cnt < 1 ) - FileAnalysis::set_timeout_interval(f, f$timeout_interval); + Files::set_timeout_interval(f, f$timeout_interval); else terminate(); ++timeout_cnt; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro index e994706010..dd40c69684 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro @@ -4,5 +4,5 @@ event file_new(f: fa_file) { - FileAnalysis::stop(f); + Files::stop(f); } diff --git a/testing/btest/scripts/base/frameworks/file-analysis/irc.bro b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro index 2b93a59a8f..9fd8e06613 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/irc.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro @@ -4,7 +4,17 @@ redef test_file_analysis_source = "IRC_DATA"; -redef test_get_file_name = function(f: fa_file): string +global first: bool = T; + +function myfile(f: fa_file): string { - return "thefile"; - }; + if ( first ) + { + first = F; + return "thefile"; + } + else + return ""; + } + +redef test_get_file_name = myfile; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/logging.bro b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro index 9792017962..1d1f5fd721 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/logging.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT -# @TEST-EXEC: btest-diff file_analysis.log +# @TEST-EXEC: btest-diff files.log redef test_file_analysis_source = "HTTP"; diff --git a/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro b/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro deleted file mode 100644 index 8cbacdbf6f..0000000000 --- a/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro +++ /dev/null @@ -1,10 +0,0 @@ -# This tests FTP file extraction. -# -# @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace %INPUT -# @TEST-EXEC: btest-diff conn.log -# @TEST-EXEC: btest-diff ftp.log -# @TEST-EXEC: cat ftp-item-*.dat | sort > extractions -# @TEST-EXEC: btest-diff extractions - -redef FTP::logged_commands += {"LIST"}; -redef FTP::extract_file_types=/.*/; diff --git a/testing/btest/scripts/base/protocols/http/http-extract-files.bro b/testing/btest/scripts/base/protocols/http/http-extract-files.bro deleted file mode 100644 index 6156009821..0000000000 --- a/testing/btest/scripts/base/protocols/http/http-extract-files.bro +++ /dev/null @@ -1,6 +0,0 @@ -# @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT -# @TEST-EXEC: btest-diff http.log -# @TEST-EXEC: mv http-item-*.dat http-item.dat -# @TEST-EXEC: btest-diff http-item.dat - -redef HTTP::extract_file_types += /text\/html/; diff --git a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro deleted file mode 100644 index b35e491b4d..0000000000 --- a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro +++ /dev/null @@ -1,6 +0,0 @@ -# This tests md5 calculation for a specified mime type. - -# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output -# @TEST-EXEC: btest-diff http.log - -redef HTTP::generate_md5 += /image\/png/; diff --git a/testing/btest/scripts/base/protocols/http/multipart-extract.bro b/testing/btest/scripts/base/protocols/http/multipart-extract.bro index c2789750a3..a919a844b2 100644 --- a/testing/btest/scripts/base/protocols/http/multipart-extract.bro +++ b/testing/btest/scripts/base/protocols/http/multipart-extract.bro @@ -1,5 +1,9 @@ # @TEST-EXEC: bro -C -r $TRACES/http/multipart.trace %INPUT # @TEST-EXEC: btest-diff http.log -# @TEST-EXEC: cat http-item-* | sort > extractions +# @TEST-EXEC: cat extract_files/http-item-* | sort > extractions -redef HTTP::extract_file_types += /.*/; +event file_new(f: fa_file) + { + local fname = fmt("http-item-%s", f$id); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); + } diff --git a/testing/btest/scripts/base/protocols/irc/dcc-extract.test b/testing/btest/scripts/base/protocols/irc/dcc-extract.test deleted file mode 100644 index cbfc6890da..0000000000 --- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test +++ /dev/null @@ -1,11 +0,0 @@ -# This tests that the contents of a DCC transfer negotiated with IRC can be -# correctly extracted. - -# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT -# @TEST-EXEC: btest-diff irc.log -# @TEST-EXEC: mv irc-dcc-item-*.dat irc-dcc-item.dat -# @TEST-EXEC: btest-diff irc-dcc-item.dat -# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test" -# @TEST-EXEC: test -e test-*.dat - -redef IRC::extract_file_types=/.*/; diff --git a/testing/btest/scripts/base/protocols/smtp/mime-extract.test b/testing/btest/scripts/base/protocols/smtp/mime-extract.test deleted file mode 100644 index 0caa5d530c..0000000000 --- a/testing/btest/scripts/base/protocols/smtp/mime-extract.test +++ /dev/null @@ -1,11 +0,0 @@ -# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT -# @TEST-EXEC: btest-diff smtp_entities.log -# @TEST-EXEC: cat smtp-entity-*.dat | sort > extractions -# @TEST-EXEC: btest-diff extractions -# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test" -# @TEST-EXEC: cnt=0 && for f in test-*.dat; do cnt=$((cnt+1)); done && echo $cnt >filecount -# @TEST-EXEC: btest-diff filecount - -@load base/protocols/smtp - -redef SMTP::extract_file_types=/text\/plain/; diff --git a/testing/btest/scripts/base/protocols/smtp/mime.test b/testing/btest/scripts/base/protocols/smtp/mime.test deleted file mode 100644 index 8e7a336987..0000000000 --- a/testing/btest/scripts/base/protocols/smtp/mime.test +++ /dev/null @@ -1,6 +0,0 @@ -# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT -# @TEST-EXEC: btest-diff smtp_entities.log - -@load base/protocols/smtp - -redef SMTP::generate_md5=/text\/plain/; diff --git a/testing/btest/scripts/base/utils/files.test b/testing/btest/scripts/base/utils/files.test index 84eff13187..3324522030 100644 --- a/testing/btest/scripts/base/utils/files.test +++ b/testing/btest/scripts/base/utils/files.test @@ -11,3 +11,10 @@ event connection_established(c: connection) print generate_extraction_filename("", c, "test-suffix"); print generate_extraction_filename("", c, ""); } + +event bro_init() + { + print extract_filename_from_content_disposition("attachment; filename=Economy"); + print extract_filename_from_content_disposition("attachment; name=\"US-$ rates\""); + print extract_filename_from_content_disposition("attachment; filename*=iso-8859-1'en'%A3%20rates"); + } \ No newline at end of file diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index b631ba2457..fb5873418a 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -7,7 +7,7 @@ IgnoreFiles = *.tmp *.swp #* *.trace .gitignore *.skeleton [environment] BROPATH=`bash -c %(testbase)s/../../../build/bro-path-dev`:%(testbase)s/../scripts -BROMAGIC=%(testbase)s/../../../magic +BROMAGIC=%(testbase)s/../../../magic/database BRO_SEED_FILE=%(testbase)s/../random.seed TZ=UTC LC_ALL=C diff --git a/testing/scripts/file-analysis-test.bro b/testing/scripts/file-analysis-test.bro index 8b85ae2bd5..d84fadae5c 100644 --- a/testing/scripts/file-analysis-test.bro +++ b/testing/scripts/file-analysis-test.bro @@ -1,7 +1,11 @@ +@load base/files/extract +@load base/files/hash + +redef FileExtract::prefix = "./"; global test_file_analysis_source: string = "" &redef; -global test_file_analyzers: set[FileAnalysis::AnalyzerArgs]; +global test_file_analyzers: set[Files::Tag]; global test_get_file_name: function(f: fa_file): string = function(f: fa_file): string { return ""; } &redef; @@ -42,15 +46,15 @@ event file_new(f: fa_file) f$source == test_file_analysis_source ) { for ( tag in test_file_analyzers ) - FileAnalysis::add_analyzer(f, tag); + Files::add_analyzer(f, tag); local filename: string = test_get_file_name(f); if ( filename != "" ) - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=filename]); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_DATA_EVENT, - $chunk_event=file_chunk, - $stream_event=file_stream]); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, + [$extract_filename=filename]); + Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT, + [$chunk_event=file_chunk, + $stream_event=file_stream]); } if ( f?$bof_buffer ) @@ -66,7 +70,7 @@ event file_new(f: fa_file) } } -event file_over_new_connection(f: fa_file, c: connection) +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) { print "FILE_OVER_NEW_CONNECTION"; } @@ -106,7 +110,7 @@ event file_state_remove(f: fa_file) event bro_init() { - add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_MD5]]; - add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_SHA1]]; - add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_SHA256]]; + add test_file_analyzers[Files::ANALYZER_MD5]; + add test_file_analyzers[Files::ANALYZER_SHA1]; + add test_file_analyzers[Files::ANALYZER_SHA256]; }