diff --git a/doc/frameworks/file_analysis_02.bro b/doc/frameworks/file_analysis_02.bro index 3ef32d865c..141b11fca6 100644 --- a/doc/frameworks/file_analysis_02.bro +++ b/doc/frameworks/file_analysis_02.bro @@ -1,7 +1,7 @@ -event file_new(f: fa_file) +event file_mime_type(f: fa_file, mime_type: string) { print "new file", f$id; - if ( f?$mime_type && f$mime_type == "text/plain" ) + if ( mime_type == "text/plain" ) Files::add_analyzer(f, Files::ANALYZER_MD5); } diff --git a/doc/httpmonitor/file_extraction.bro b/doc/httpmonitor/file_extraction.bro index b2318c595e..3860cb361e 100644 --- a/doc/httpmonitor/file_extraction.bro +++ b/doc/httpmonitor/file_extraction.bro @@ -7,18 +7,15 @@ global mime_to_ext: table[string] of string = { ["text/html"] = "html", }; -event file_new(f: fa_file) +event file_mime_type(f: fa_file, mime_type: string) { if ( f$source != "HTTP" ) return; - if ( ! f?$mime_type ) + if ( mime_type !in mime_to_ext ) return; - if ( f$mime_type !in mime_to_ext ) - return; - - local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]); + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[mime_type]); print fmt("Extracting file %s", fname); Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); - } + } \ No newline at end of file diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index d18f87ac38..fb1c1b67a1 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -103,9 +103,9 @@ In the ``file_hash`` event handler, there is an ``if`` statement that is used to check for the correct type of hash, in this case a SHA1 hash. It also checks for a mime type we've defined as being of interest as defined in the constant ``match_file_types``. -The comparison is made against the expression ``f$mime_type``, which uses +The comparison is made against the expression ``f$info$mime_type``, which uses the ``$`` dereference operator to check the value ``mime_type`` -inside the variable ``f``. If the entire expression evaluates to true, +inside the variable ``f$info``. If the entire expression evaluates to true, then a helper function is called to do the rest of the work. In that function, a local variable is defined to hold a string comprised of the SHA1 hash concatenated with ``.malware.hash.cymru.com``; this diff --git a/scripts/base/frameworks/files/magic/__load__.bro b/scripts/base/frameworks/files/magic/__load__.bro index 4a2de0926d..c6ee799a53 100644 --- a/scripts/base/frameworks/files/magic/__load__.bro +++ b/scripts/base/frameworks/files/magic/__load__.bro @@ -1,2 +1,3 @@ @load-sigs ./general +@load-sigs ./msoffice @load-sigs ./libmagic diff --git a/scripts/base/frameworks/files/magic/general.sig b/scripts/base/frameworks/files/magic/general.sig index a11e4a05e4..500c4f7be0 100644 --- a/scripts/base/frameworks/files/magic/general.sig +++ b/scripts/base/frameworks/files/magic/general.sig @@ -1,16 +1,137 @@ # General purpose file magic signatures. signature file-plaintext { - file-magic /([[:print:][:space:]]{10})/ + file-magic /^([[:print:][:space:]]{10})/ file-mime "text/plain", -20 } signature file-tar { - file-magic /([[:print:]\x00]){100}(([[:digit:]\x00\x20]){8}){3}/ - file-mime "application/x-tar", 150 + file-magic /^[[:print:]\x00]{100}([[:digit:]\x20]{7}\x00){3}([[:digit:]\x20]{11}\x00){2}([[:digit:]\x00\x20]{7}[\x20\x00])[0-7\x00]/ + file-mime "application/x-tar", 100 } +signature file-zip { + file-mime "application/zip", 10 + file-magic /^PK\x03\x04.{2}/ +} + +signature file-jar { + file-mime "application/java-archive", 100 + file-magic /^PK\x03\x04.{1,200}\x14\x00..META-INF\/MANIFEST\.MF/ +} + +signature file-java-applet { + file-magic /^\xca\xfe\xba\xbe...[\x2e-\x34]/ + file-mime "application/x-java-applet", 71 +} + +# Shockwave flash signature file-swf { - file-magic /(F|C|Z)WS/ + file-magic /^(F|C|Z)WS/ file-mime "application/x-shockwave-flash", 60 -} \ No newline at end of file +} + +# Microsoft Outlook's Transport Neutral Encapsulation Format +signature file-tnef { + file-magic /^\x78\x9f\x3e\x22/ + file-mime "application/vnd.ms-tnef", 100 +} + +# Mac OS X DMG files +signature file-dmg { + file-magic /^(\x78\x01\x73\x0D\x62\x62\x60|\x78\xDA\x63\x60\x18\x05|\x78\x01\x63\x60\x18\x05|\x78\xDA\x73\x0D|\x78[\x01\xDA]\xED[\xD0-\xD9])/ + file-mime "application/x-dmg", 100 +} + +# Mac OS X Mach-O executable +signature file-mach-o { + file-magic /^[\xce\xcf]\xfa\xed\xfe/ + file-mime "application/x-mach-o-executable", 100 +} + +# Mac OS X Universal Mach-O executable +signature file-mach-o-universal { + file-magic /^\xca\xfe\xba\xbe..\x00[\x01-\x14]/ + file-mime "application/x-mach-o-executable", 100 +} + +# XAR (eXtensible ARchive) format. +# Mac OS X uses this for the .pkg format. +signature file-xar { + file-magic /^xar\!/ + file-mime "application/x-xar", 100 +} + +signature file-pkcs7 { + file-magic /^MIME-Version:.*protocol=\"application\/pkcs7-signature\"/ + file-mime "application/pkcs7-signature", 100 +} + +# Concatenated X.509 certificates in textual format. +signature file-pem { + file-magic /^-----BEGIN CERTIFICATE-----/ + file-mime "application/x-pem" +} + +# Java Web Start file. +signature file-jnlp { + file-magic /^\2080 string,=Foglio di lavoro Microsoft Exce (len=31), ["%s"], swap_endian=0 -signature file-magic-auto0 { - file-mime "application/vnd.ms-excel", 340 - file-magic /(.{2080})(Foglio di lavoro Microsoft Exce)/ -} - # >2 string,=---BEGIN PGP PUBLIC KEY BLOCK- (len=30), ["PGP public key block"], swap_endian=0 signature file-magic-auto1 { file-mime "application/pgp-keys", 330 file-magic /(.{2})(\x2d\x2d\x2dBEGIN PGP PUBLIC KEY BLOCK\x2d)/ } -# >2080 string,=Microsoft Excel 5.0 Worksheet (len=29), ["%s"], swap_endian=0 -signature file-magic-auto2 { - file-mime "application/vnd.ms-excel", 320 - file-magic /(.{2080})(Microsoft Excel 5\x2e0 Worksheet)/ -} - # >11 string,=must be converted with BinHex (len=29), ["BinHex binary text"], swap_endian=0 signature file-magic-auto3 { file-mime "application/mac-binhex40", 320 file-magic /(.{11})(must be converted with BinHex)/ } -# >2080 string,=Microsoft Word 6.0 Document (len=27), ["%s"], swap_endian=0 -signature file-magic-auto4 { - file-mime "application/msword", 300 - file-magic /(.{2080})(Microsoft Word 6\x2e0 Document)/ -} - -# >2080 string,=Documento Microsoft Word 6 (len=26), ["Spanish Microsoft Word 6 document data"], swap_endian=0 -signature file-magic-auto5 { - file-mime "application/msword", 290 - file-magic /(.{2080})(Documento Microsoft Word 6)/ -} - # >0 string,=-----BEGIN PGP SIGNATURE- (len=25), ["PGP signature"], swap_endian=0 signature file-magic-auto6 { file-mime "application/pgp-signature", 280 @@ -92,36 +68,6 @@ signature file-magic-auto13 { file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fgawk)/ } -# >0 string/wt,=#! /usr/local/bin/bash (len=22), ["Bourne-Again shell script text executable"], swap_endian=0 -signature file-magic-auto14 { - file-mime "text/x-shellscript", 250 - file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fbash)/ -} - -# >0 string/wt,=#! /usr/local/bin/tcsh (len=22), ["Tenex C shell script text executable"], swap_endian=0 -signature file-magic-auto15 { - file-mime "text/x-shellscript", 250 - file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2ftcsh)/ -} - -# >0 string/wt,=#! /usr/local/bin/zsh (len=21), ["Paul Falstad's zsh script text executable"], swap_endian=0 -signature file-magic-auto16 { - file-mime "text/x-shellscript", 240 - file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fzsh)/ -} - -# >0 string/wt,=#! /usr/local/bin/ash (len=21), ["Neil Brown's ash script text executable"], swap_endian=0 -signature file-magic-auto17 { - file-mime "text/x-shellscript", 240 - file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fash)/ -} - -# >0 string/wt,=#! /usr/local/bin/ae (len=20), ["Neil Brown's ae script text executable"], swap_endian=0 -signature file-magic-auto18 { - file-mime "text/x-shellscript", 230 - file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fae)/ -} - # >0 string,=# PaCkAgE DaTaStReAm (len=20), ["pkg Datastream (SVR4)"], swap_endian=0 signature file-magic-auto19 { file-mime "application/x-svr4-package", 230 @@ -140,30 +86,12 @@ signature file-magic-auto21 { file-magic /(\x5bKDE Desktop Entry\x5d)/ } -# >512 string,=R\000o\000o\000t\000 \000E\000n\000t\000r\000y (len=19), ["Microsoft Word Document"], swap_endian=0 -signature file-magic-auto22 { - file-mime "application/msword", 220 - file-magic /(.{512})(R\x00o\x00o\x00t\x00 \x00E\x00n\x00t\x00r\x00y)/ -} - # >0 string,=!\n__________E (len=19), ["MIPS archive"], swap_endian=0 signature file-magic-auto23 { file-mime "application/x-archive", 220 file-magic /(\x21\x3carch\x3e\x0a\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5fE)/ } -# >0 string/wt,=#! /usr/local/tcsh (len=18), ["Tenex C shell script text executable"], swap_endian=0 -signature file-magic-auto24 { - file-mime "text/x-shellscript", 210 - file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2ftcsh)/ -} - -# >0 string/wt,=#! /usr/local/bash (len=18), ["Bourne-Again shell script text executable"], swap_endian=0 -signature file-magic-auto25 { - file-mime "text/x-shellscript", 210 - file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbash)/ -} - # >0 string/t,=# KDE Config File (len=17), ["KDE config file"], swap_endian=0 signature file-magic-auto26 { file-mime "application/x-kdelnk", 200 @@ -189,12 +117,6 @@ signature file-magic-auto29 { file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fnawk)/ } -# >0 string/wt,=#! /usr/bin/tcsh (len=16), ["Tenex C shell script text executable"], swap_endian=0 -signature file-magic-auto30 { - file-mime "text/x-shellscript", 190 - file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2ftcsh)/ -} - # >0 string/wt,=#! /usr/bin/gawk (len=16), ["GNU awk script text executable"], swap_endian=0 signature file-magic-auto31 { file-mime "text/x-gawk", 190 @@ -207,12 +129,6 @@ signature file-magic-auto32 { file-magic /(.{369})(MICROSOFT PIFEX\x00)/ } -# >0 string/wt,=#! /usr/bin/bash (len=16), ["Bourne-Again shell script text executable"], swap_endian=0 -signature file-magic-auto33 { - file-mime "text/x-shellscript", 190 - file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fbash)/ -} - # >0 string/w,=#VRML V1.0 ascii (len=16), ["VRML 1 file"], swap_endian=0 signature file-magic-auto34 { file-mime "model/vrml", 190 @@ -334,12 +250,6 @@ signature file-magic-auto51 { file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fawk)/ } -# >0 string/wt,=#! /usr/bin/zsh (len=15), ["Paul Falstad's zsh script text executable"], swap_endian=0 -signature file-magic-auto52 { - file-mime "text/x-shellscript", 180 - file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fzsh)/ -} - # >0 string,=MAS_UTrack_V00 (len=14), [""], swap_endian=0 # >>14 string,>/0 (len=2), ["ultratracker V1.%.1s module sound data"], swap_endian=0 signature file-magic-auto53 { @@ -457,12 +367,6 @@ signature file-magic-auto70 { file-magic /(\x3cmap ?version)/ } -# >0 string/wt,=#! /bin/tcsh (len=12), ["Tenex C shell script text executable"], swap_endian=0 -signature file-magic-auto71 { - file-mime "text/x-shellscript", 150 - file-magic /(\x23\x21 ?\x2fbin\x2ftcsh)/ -} - # >0 string/wt,=#! /bin/nawk (len=12), ["new awk script text executable"], swap_endian=0 signature file-magic-auto72 { file-mime "text/x-nawk", 150 @@ -475,12 +379,6 @@ signature file-magic-auto73 { file-magic /(\x23\x21 ?\x2fbin\x2fgawk)/ } -# >0 string/wt,=#! /bin/bash (len=12), ["Bourne-Again shell script text executable"], swap_endian=0 -signature file-magic-auto74 { - file-mime "text/x-shellscript", 150 - file-magic /(\x23\x21 ?\x2fbin\x2fbash)/ -} - # >0 string/wt,=#! /bin/awk (len=11), ["awk script text executable"], swap_endian=0 signature file-magic-auto75 { file-mime "text/x-awk", 140 @@ -505,24 +403,6 @@ signature file-magic-auto78 { file-magic /(d8\x3aannounce)/ } -# >0 string/wt,=#! /bin/csh (len=11), ["C shell script text executable"], swap_endian=0 -signature file-magic-auto79 { - file-mime "text/x-shellscript", 140 - file-magic /(\x23\x21 ?\x2fbin\x2fcsh)/ -} - -# >0 string/wt,=#! /bin/ksh (len=11), ["Korn shell script text executable"], swap_endian=0 -signature file-magic-auto80 { - file-mime "text/x-shellscript", 140 - file-magic /(\x23\x21 ?\x2fbin\x2fksh)/ -} - -# >0 string/wt,=#! /bin/zsh (len=11), ["Paul Falstad's zsh script text executable"], swap_endian=0 -signature file-magic-auto81 { - file-mime "text/x-shellscript", 140 - file-magic /(\x23\x21 ?\x2fbin\x2fzsh)/ -} - # >0 string/c,=BEGIN:VCARD (len=11), ["vCard visiting card"], swap_endian=0 signature file-magic-auto82 { file-mime "text/x-vcard", 140 @@ -545,12 +425,6 @@ signature file-magic-auto84 { file-magic /(Forward to)/ } -# >0 string/wt,=#! /bin/sh (len=10), ["POSIX shell script text executable"], swap_endian=0 -signature file-magic-auto85 { - file-mime "text/x-shellscript", 130 - file-magic /(\x23\x21 ?\x2fbin\x2fsh)/ -} - # >0 string,=II*\000\020\000\000\000CR (len=10), ["Canon CR2 raw image data"], swap_endian=0 signature file-magic-auto86 { file-mime "image/x-canon-cr2", 130 @@ -585,12 +459,6 @@ signature file-magic-auto90 { file-magic /(\x3cBookFile)/ } -# >2112 string,=MSWordDoc (len=9), ["Microsoft Word document data"], swap_endian=0 -signature file-magic-auto91 { - file-mime "application/msword", 120 - file-magic /(.{2112})(MSWordDoc)/ -} - # >0 string/t,=N#! rnews (len=9), ["mailed, batched news text"], swap_endian=0 signature file-magic-auto92 { file-mime "message/rfc822", 120 @@ -656,12 +524,6 @@ signature file-magic-auto100 { file-magic /(MSCF\x00\x00\x00\x00)/ } -# >0 string/b,=\320\317\021\340\241\261\032\341 (len=8), ["Microsoft Office Document"], swap_endian=0 -signature file-magic-auto101 { - file-mime "application/msword", 110 - file-magic /(\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1)/ -} - # >21 string/c,=!SCREAM! (len=8), ["Screamtracker 2 module sound data"], swap_endian=0 signature file-magic-auto102 { file-mime "audio/x-mod", 110 @@ -754,10 +616,10 @@ signature file-magic-auto116 { } # >257 string,=ustar \000 (len=8), ["GNU tar archive"], swap_endian=0 -signature file-magic-auto117 { - file-mime "application/x-tar", 110 - file-magic /(.{257})(ustar \x00)/ -} +#signature file-magic-auto117 { +# file-mime "application/x-tar", 110 +# file-magic /(.{257})(ustar \x00)/ +#} # >0 string,=0 string/b,=\t\004\006\000\000\000\020\000 (len=8), ["Microsoft Excel Worksheet"], swap_endian=0 -signature file-magic-auto120 { - file-mime "application/vnd.ms-excel", 110 - file-magic /(\x09\x04\x06\x00\x00\x00\x10\x00)/ -} - # >0 string/b,=WordPro\000 (len=8), ["Lotus WordPro"], swap_endian=0 signature file-magic-auto121 { file-mime "application/vnd.lotus-wordpro", 110 @@ -844,10 +700,10 @@ signature file-magic-auto130 { } # >257 string,=ustar\000 (len=6), ["POSIX tar archive"], swap_endian=0 -signature file-magic-auto131 { - file-mime "application/x-tar", 90 - file-magic /(.{257})(ustar\x00)/ -} +#signature file-magic-auto131 { +# file-mime "application/x-tar", 90 +# file-magic /(.{257})(ustar\x00)/ +#} # >0 string,=AC1.40 (len=6), ["DWG AutoDesk AutoCAD Release 1.40"], swap_endian=0 signature file-magic-auto132 { @@ -994,12 +850,6 @@ signature file-magic-auto155 { file-magic /(\x23 xmcd)/ } -# >0 string/b,=\333\245-\000\000\000 (len=6), ["Microsoft Office Document"], swap_endian=0 -signature file-magic-auto156 { - file-mime "application/msword", 90 - file-magic /(\xdb\xa5\x2d\x00\x00\x00)/ -} - # >2 string,=MMXPR3 (len=6), ["Motorola Quark Express Document (English)"], swap_endian=0 signature file-magic-auto157 { file-mime "application/x-quark-xpress-3", 90 @@ -1046,36 +896,6 @@ signature file-magic-auto162 { file-magic /(\x3c\x3fxml)(.{15})(.*)( xmlns\x3d)(['"]http:\x2f\x2fwww.opengis.net\x2fkml)/ } -# >0 string,=PK\003\004 (len=4), [""], swap_endian=0 -# >>30 regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0 -# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0 -# >>>>&26 search/1000,=PK\003\004 (len=4), [""], swap_endian=0 -# >>>>>&26 string,=word/ (len=5), ["Microsoft Word 2007+"], swap_endian=0 -signature file-magic-auto163 { - file-mime "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 80 - file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(word\x2f)/ -} - -# >0 string,=PK\003\004 (len=4), [""], swap_endian=0 -# >>30 regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0 -# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0 -# >>>>&26 search/1000,=PK\003\004 (len=4), [""], swap_endian=0 -# >>>>>&26 string,=ppt/ (len=4), ["Microsoft PowerPoint 2007+"], swap_endian=0 -signature file-magic-auto164 { - file-mime "application/vnd.openxmlformats-officedocument.presentationml.presentation", 70 - file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(ppt\x2f)/ -} - -# >0 string,=PK\003\004 (len=4), [""], swap_endian=0 -# >>30 regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0 -# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0 -# >>>>&26 search/1000,=PK\003\004 (len=4), [""], swap_endian=0 -# >>>>>&26 string,=xl/ (len=3), ["Microsoft Excel 2007+"], swap_endian=0 -signature file-magic-auto165 { - file-mime "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 60 - file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(xl\x2f)/ -} - # >60 string,=RINEX (len=5), [""], swap_endian=0 # >>80 search/256,=XXRINEXB (len=8), ["RINEX Data, GEO SBAS Broadcast"], swap_endian=0 # >>>5 string,x, [", version %6.6s"], swap_endian=0 @@ -1229,30 +1049,12 @@ signature file-magic-auto187 { file-magic /(\x00\x01\x00\x00\x00)/ } -# >0 string/b,=PO^Q` (len=5), ["Microsoft Word 6.0 Document"], swap_endian=0 -signature file-magic-auto188 { - file-mime "application/msword", 80 - file-magic /(PO\x5eQ\x60)/ -} - # >0 string,=%PDF- (len=5), ["PDF document"], swap_endian=0 signature file-magic-auto189 { file-mime "application/pdf", 80 file-magic /(\x25PDF\x2d)/ } -# >2114 string,=Biff5 (len=5), ["Microsoft Excel 5.0 Worksheet"], swap_endian=0 -signature file-magic-auto190 { - file-mime "application/vnd.ms-excel", 80 - file-magic /(.{2114})(Biff5)/ -} - -# >2121 string,=Biff5 (len=5), ["Microsoft Excel 5.0 Worksheet"], swap_endian=0 -signature file-magic-auto191 { - file-mime "application/vnd.ms-excel", 80 - file-magic /(.{2121})(Biff5)/ -} - # >0 string/t,=Path: (len=5), ["news text"], swap_endian=0 signature file-magic-auto192 { file-mime "message/news", 80 @@ -1383,12 +1185,6 @@ signature file-magic-auto211 { file-magic /(\x00\x00\x00\x01)([\x07\x27\x47\x67\x87\xa7\xc7\xe7])/ } -# >0 belong&,=-889275714 (0xcafebabe), [""], swap_endian=0 -signature file-magic-auto212 { - file-mime "application/x-java-applet", 71 - file-magic /(\xca\xfe\xba\xbe)/ -} - # >0 belong&ffffffffffffff00,=256 (0x00000100), [""], swap_endian=0 # >>3 byte&,=0xba, ["MPEG sequence"], swap_endian=0 signature file-magic-auto213 { @@ -1706,46 +1502,6 @@ signature file-magic-auto245 { file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetypeapplication\x2f)(epub\x2bzip)/ } -# Seems redundant with other zip signature below. -# >0 string,=PK\003\004 (len=4), [""], swap_endian=0 -# >>26 string,=\b\000\000\000mimetypeapplication/ (len=24), [""], swap_endian=0 -# >>>50 string,!epub+zip (len=8), [""], swap_endian=0 -# >>>>50 string,!vnd.oasis.opendocument. (len=23), [""], swap_endian=0 -# >>>>>50 string,!vnd.sun.xml. (len=12), [""], swap_endian=0 -# >>>>>>50 string,!vnd.kde. (len=8), [""], swap_endian=0 -# >>>>>>>38 regex,=[!-OQ-~]+ (len=9), ["Zip data (MIME type "%s"?)"], swap_endian=0 -#signature file-magic-auto246 { -# file-mime "application/zip", 39 -# file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetypeapplication\x2f)/ -#} - -# >0 string,=PK\003\004 (len=4), [""], swap_endian=0 -# >>26 string,=\b\000\000\000mimetype (len=12), [""], swap_endian=0 -# >>>38 string,!application/ (len=12), [""], swap_endian=0 -# >>>>38 regex,=[!-OQ-~]+ (len=9), ["Zip data (MIME type "%s"?)"], swap_endian=0 -signature file-magic-auto247 { - file-mime "application/zip", 39 - file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetype)/ -} - -# The indirect offset makes this difficult to convert. -# The (.*) may be too generous. -# >0 string,=PK\003\004 (len=4), [""], swap_endian=0 -# >>26 (leshort,+30), leshort&,=-13570 (0xcafe), ["Java archive data (JAR)"], swap_endian=0 -signature file-magic-auto248 { - file-mime "application/java-archive", 50 - file-magic /(PK\x03\x04)(.*)(\xfe\xca)/ -} - -# The indeirect offset and string inequality make this difficult to convert. -# >0 string,=PK\003\004 (len=4), [""], swap_endian=0 -# >>26 (leshort,+30), leshort&,!-13570 (0xcafe), [""], swap_endian=0 -# >>>26 string,!\b\000\000\000mimetype (len=12), ["Zip archive data"], swap_endian=0 -signature file-magic-auto249 { - file-mime "application/zip", 10 - file-magic /(PK\x03\x04)(.{2})/ -} - # >0 belong&,=442 (0x000001ba), [""], swap_endian=0 # >>4 byte&,&0x40, [""], swap_endian=0 signature file-magic-auto250 { @@ -2065,18 +1821,6 @@ signature file-magic-auto299 { file-magic /(PDN3)/ } -# >0 ulelong&,=2712847316 (0xa1b2c3d4), ["tcpdump capture file (little-endian)"], swap_endian=0 -signature file-magic-auto300 { - file-mime "application/vnd.tcpdump.pcap", 70 - file-magic /(\xd4\xc3\xb2\xa1)/ -} - -# >0 ubelong&,=2712847316 (0xa1b2c3d4), ["tcpdump capture file (big-endian)"], swap_endian=0 -signature file-magic-auto301 { - file-mime "application/vnd.tcpdump.pcap", 70 - file-magic /(\xa1\xb2\xc3\xd4)/ -} - # >0 belong&,=-17957139 (0xfeedfeed), ["Java KeyStore"], swap_endian=0 signature file-magic-auto302 { file-mime "application/x-java-keystore", 70 @@ -2297,12 +2041,6 @@ signature file-magic-auto335 { file-magic /(SIT\x21)/ } -# >0 lelong&,=574529400 (0x223e9f78), ["Transport Neutral Encapsulation Format"], swap_endian=0 -signature file-magic-auto336 { - file-mime "application/vnd.ms-tnef", 70 - file-magic /(\x78\x9f\x3e\x22)/ -} - # >0 string,= (len=4), ["System V Release 1 ar archive"], swap_endian=0 signature file-magic-auto337 { file-mime "application/x-archive", 70 @@ -2433,48 +2171,6 @@ signature file-magic-auto357 { file-magic /(RIFF)(.{4})(AVI )/ } -# >0 belong&,=834535424 (0x31be0000), ["Microsoft Word Document"], swap_endian=0 -signature file-magic-auto358 { - file-mime "application/msword", 70 - file-magic /(\x31\xbe\x00\x00)/ -} - -# >0 string/b,=\3767\000# (len=4), ["Microsoft Office Document"], swap_endian=0 -signature file-magic-auto359 { - file-mime "application/msword", 70 - file-magic /(\xfe7\x00\x23)/ -} - -# >0 string/b,=\333\245-\000 (len=4), ["Microsoft WinWord 2.0 Document"], swap_endian=0 -signature file-magic-auto360 { - file-mime "application/msword", 70 - file-magic /(\xdb\xa5\x2d\x00)/ -} - -# >0 string/b,=\333\245-\000 (len=4), ["Microsoft WinWord 2.0 Document"], swap_endian=0 -signature file-magic-auto361 { - file-mime "application/msword", 70 - file-magic /(\xdb\xa5\x2d\x00)/ -} - -# >0 belong&,=6656 (0x00001a00), ["Lotus 1-2-3"], swap_endian=0 -signature file-magic-auto362 { - file-mime "application/x-123", 70 - file-magic /(\x00\x00\x1a\x00)/ -} - -# >0 belong&,=512 (0x00000200), ["Lotus 1-2-3"], swap_endian=0 -signature file-magic-auto363 { - file-mime "application/x-123", 70 - file-magic /(\x00\x00\x02\x00)/ -} - -# >0 string/b,=\000\000\001\000 (len=4), ["MS Windows icon resource"], swap_endian=0 -signature file-magic-auto364 { - file-mime "image/x-icon", 70 - file-magic /(\x00\x00\x01\x00)/ -} - # >0 lelong&,=268435536 (0x10000050), ["Psion Series 5"], swap_endian=0 # >>4 lelong&,=268435565 (0x1000006d), ["database"], swap_endian=0 # >>>8 lelong&,=268435588 (0x10000084), ["Agenda file"], swap_endian=0 @@ -2737,12 +2433,6 @@ signature file-magic-auto403 { file-magic /(SBI)/ } -# >0 string/b,=\224\246. (len=3), ["Microsoft Word Document"], swap_endian=0 -signature file-magic-auto404 { - file-mime "application/msword", 60 - file-magic /(\x94\xa6\x2e)/ -} - # >0 string,=\004%! (len=3), ["PostScript document text"], swap_endian=0 signature file-magic-auto405 { file-mime "application/postscript", 60 @@ -2763,17 +2453,11 @@ signature file-magic-auto407 { file-magic /(.*)([ \x09]*(class|module)[ \x09][A-Z])((modul|includ)e [A-Z]|def [a-z])(^[ \x09]*end([ \x09]*[;#].*)?$)/ } -# >512 string/b,=\354\245\301 (len=3), ["Microsoft Word Document"], swap_endian=0 -signature file-magic-auto408 { - file-mime "application/msword", 60 - file-magic /(.{512})(\xec\xa5\xc1)/ -} - # >0 regex/20,=^\.[A-Za-z0-9][A-Za-z0-9][ \t] (len=29), ["troff or preprocessor input text"], swap_endian=0 -signature file-magic-auto411 { - file-mime "text/troff", 59 - file-magic /(^\.[A-Za-z0-9][A-Za-z0-9][ \x09])/ -} +#signature file-magic-auto411 { +# file-mime "text/troff", 59 +# file-magic /(^\.[A-Za-z0-9][A-Za-z0-9][ \x09])/ +#} # >0 search/4096,=\documentclass (len=14), ["LaTeX 2e document text"], swap_endian=0 signature file-magic-auto412 { @@ -2806,10 +2490,10 @@ signature file-magic-auto416 { } # >0 regex/20,=^\.[A-Za-z0-9][A-Za-z0-9]$ (len=26), ["troff or preprocessor input text"], swap_endian=0 -signature file-magic-auto417 { - file-mime "text/troff", 56 - file-magic /(^\.[A-Za-z0-9][A-Za-z0-9]$)/ -} +#signature file-magic-auto417 { +# file-mime "text/troff", 56 +# file-magic /(^\.[A-Za-z0-9][A-Za-z0-9]$)/ +#} # >0 search/w/1,=#! /usr/bin/php (len=15), ["PHP script text executable"], swap_endian=0 signature file-magic-auto418 { @@ -2829,30 +2513,12 @@ signature file-magic-auto420 { file-magic /(.*)(eval \x22exec \x2fusr\x2fbin\x2fperl)/ } -# >0 search/w/1,=#! /usr/local/bin/python (len=24), ["Python script text executable"], swap_endian=0 -signature file-magic-auto421 { - file-mime "text/x-python", 54 - file-magic /(.*)(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fpython)/ -} - # >0 search/1,=Common subdirectories: (len=23), ["diff output text"], swap_endian=0 signature file-magic-auto422 { file-mime "text/x-diff", 53 file-magic /(.*)(Common subdirectories\x3a )/ } -# >0 search/1,=#! /usr/bin/env python (len=22), ["Python script text executable"], swap_endian=0 -signature file-magic-auto423 { - file-mime "text/x-python", 52 - file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv python)/ -} - -# >0 search/w/1,=#! /usr/local/bin/ruby (len=22), ["Ruby script text executable"], swap_endian=0 -signature file-magic-auto424 { - file-mime "text/x-ruby", 52 - file-magic /(.*)(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fruby)/ -} - # >0 search/w/1,=#! /usr/local/bin/wish (len=22), ["Tcl/Tk script text executable"], swap_endian=0 signature file-magic-auto425 { file-mime "text/x-tcl", 52 @@ -2871,12 +2537,6 @@ signature file-magic-auto427 { file-magic /(\xff\xd8)/ } -# >0 search/1,=#!/usr/bin/env python (len=21), ["Python script text executable"], swap_endian=0 -signature file-magic-auto428 { - file-mime "text/x-python", 51 - file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv python)/ -} - # >0 search/1,=#!/usr/bin/env nodejs (len=21), ["Node.js script text executable"], swap_endian=0 signature file-magic-auto429 { file-mime "application/javascript", 51 @@ -3189,12 +2849,6 @@ signature file-magic-auto474 { file-magic /(\x25\x21)/ } -# >0 search/1,=#! /usr/bin/env ruby (len=20), ["Ruby script text executable"], swap_endian=0 -signature file-magic-auto475 { - file-mime "text/x-ruby", 50 - file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv ruby)/ -} - # >0 regex/1,=(^[0-9]{5})[acdn][w] (len=20), ["MARC21 Classification"], swap_endian=0 signature file-magic-auto476 { file-mime "application/marc", 50 @@ -3228,10 +2882,10 @@ signature file-magic-auto480 { } # >0 string,=\n( (len=2), ["Emacs v18 byte-compiled Lisp data"], swap_endian=0 -signature file-magic-auto481 { - file-mime "application/x-elc", 50 - file-magic /(\x0a\x28)/ -} +#signature file-magic-auto481 { +# file-mime "application/x-elc", 50 +# file-magic /(\x0a\x28)/ +#} # >0 string,=\021\t (len=2), ["Award BIOS Logo, 136 x 126"], swap_endian=0 signature file-magic-auto482 { @@ -3305,17 +2959,17 @@ signature file-magic-auto493 { file-magic /(\xf7\x02)/ } -# >2 string,=\000\021 (len=2), ["TeX font metric data"], swap_endian=0 -signature file-magic-auto494 { - file-mime "application/x-tex-tfm", 50 - file-magic /(.{2})(\x00\x11)/ -} - -# >2 string,=\000\022 (len=2), ["TeX font metric data"], swap_endian=0 -signature file-magic-auto495 { - file-mime "application/x-tex-tfm", 50 - file-magic /(.{2})(\x00\x12)/ -} +## >2 string,=\000\021 (len=2), ["TeX font metric data"], swap_endian=0 +#signature file-magic-auto494 { +# file-mime "application/x-tex-tfm", 50 +# file-magic /(.{2})(\x00\x11)/ +#} +# +## >2 string,=\000\022 (len=2), ["TeX font metric data"], swap_endian=0 +#signature file-magic-auto495 { +# file-mime "application/x-tex-tfm", 50 +# file-magic /(.{2})(\x00\x12)/ +#} # >0 beshort&,=-31486 (0x8502), ["GPG encrypted data"], swap_endian=0 signature file-magic-auto496 { @@ -3470,12 +3124,6 @@ signature file-magic-auto514 { file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv lua)/ } -# >0 search/1,=#!/usr/bin/env ruby (len=19), ["Ruby script text executable"], swap_endian=0 -signature file-magic-auto515 { - file-mime "text/x-ruby", 49 - file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv ruby)/ -} - # >0 search/1,=#! /usr/bin/env tcl (len=19), ["Tcl script text executable"], swap_endian=0 signature file-magic-auto516 { file-mime "text/x-tcl", 49 @@ -3493,12 +3141,6 @@ signature file-magic-auto519 { file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv lua)/ } -# >0 search/w/1,=#! /usr/bin/python (len=18), ["Python script text executable"], swap_endian=0 -signature file-magic-auto520 { - file-mime "text/x-python", 48 - file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fpython)/ -} - # >0 search/w/1,=#!/usr/bin/nodejs (len=17), ["Node.js script text executable"], swap_endian=0 signature file-magic-auto521 { file-mime "application/javascript", 47 @@ -3506,10 +3148,10 @@ signature file-magic-auto521 { } # >0 regex,=^class[ \t\n]+ (len=12), ["C++ source text"], swap_endian=0 -signature file-magic-auto522 { - file-mime "text/x-c++", 47 - file-magic /(.*)(class[ \x09\x0a]+[[:alnum:]_]+)(.*)(\x7b)(.*)(public:)/ -} +#signature file-magic-auto522 { +# file-mime "text/x-c++", 47 +# file-magic /(.*)(class[ \x09\x0a]+[[:alnum:]_]+)(.*)(\x7b)(.*)(public:)/ +#} # >0 search/1,=This is Info file (len=17), ["GNU Info text"], swap_endian=0 signature file-magic-auto528 { @@ -3658,12 +3300,6 @@ signature file-magic-auto545 { file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fwish)/ } -# >0 search/w/1,=#! /usr/bin/ruby (len=16), ["Ruby script text executable"], swap_endian=0 -signature file-magic-auto546 { - file-mime "text/x-ruby", 46 - file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fruby)/ -} - # >0 search/w/1,=#! /usr/bin/lua (len=15), ["Lua script text executable"], swap_endian=0 signature file-magic-auto547 { file-mime "text/x-lua", 45 @@ -3727,10 +3363,10 @@ signature file-magic-auto556 { } # >0 regex,=^extern[ \t\n]+ (len=13), ["C source text"], swap_endian=0 -signature file-magic-auto557 { - file-mime "text/x-c", 43 - file-magic /(.*)(extern[ \x09\x0a]+)/ -} +#signature file-magic-auto557 { +# file-mime "text/x-c", 43 +# file-magic /(.*)(extern[ \x09\x0a]+)/ +#} # >0 search/4096,=% -*-latex-*- (len=13), ["LaTeX document text"], swap_endian=0 signature file-magic-auto558 { @@ -3746,10 +3382,10 @@ signature file-magic-auto558 { #} # >0 regex,=^struct[ \t\n]+ (len=13), ["C source text"], swap_endian=0 -signature file-magic-auto560 { - file-mime "text/x-c", 43 - file-magic /(.*)(struct[ \x09\x0a]+)/ -} +#signature file-magic-auto560 { +# file-mime "text/x-c", 43 +# file-magic /(.*)(struct[ \x09\x0a]+)/ +#} # >0 search/w/1,=#!/bin/nodejs (len=13), ["Node.js script text executable"], swap_endian=0 signature file-magic-auto561 { @@ -3802,10 +3438,10 @@ signature file-magic-auto567 { } # >0 regex,=^char[ \t\n]+ (len=11), ["C source text"], swap_endian=0 -signature file-magic-auto568 { - file-mime "text/x-c", 41 - file-magic /(.*)(char[ \x09\x0a]+)/ -} +#signature file-magic-auto568 { +# file-mime "text/x-c", 41 +# file-magic /(.*)(char[ \x09\x0a]+)/ +#} # >0 search/1,=#! (len=2), [""], swap_endian=0 # >>0 regex,=^#!.*/bin/perl$ (len=15), ["Perl script text executable"], swap_endian=0 @@ -3887,23 +3523,11 @@ signature file-magic-auto578 { file-magic /(^dnl )/ } -# >0 regex,=^all: (len=5), ["makefile script text"], swap_endian=0 -signature file-magic-auto579 { - file-mime "text/x-makefile", 40 - file-magic /(^all:)/ -} - -# >0 regex,=^.PRECIOUS (len=10), ["makefile script text"], swap_endian=0 -signature file-magic-auto580 { - file-mime "text/x-makefile", 40 - file-magic /(^.PRECIOUS)/ -} - # >0 search/8192,=main( (len=5), ["C source text"], swap_endian=0 -signature file-magic-auto581 { - file-mime "text/x-c", 40 - file-magic /(.*)(main\x28)/ -} +#signature file-magic-auto581 { +# file-mime "text/x-c", 40 +# file-magic /(.*)(main\x28)/ +#} # Not specific enough. # >0 search/1,=\" (len=2), ["troff or preprocessor input text"], swap_endian=0 @@ -3932,22 +3556,22 @@ signature file-magic-auto584 { #} # >0 regex,=^#include (len=9), ["C source text"], swap_endian=0 -signature file-magic-auto586 { - file-mime "text/x-c", 39 - file-magic /(.*)(#include)/ -} +#signature file-magic-auto586 { +# file-mime "text/x-c", 39 +# file-magic /(.*)(#include)/ +#} # >0 search/1,=.\" (len=3), ["troff or preprocessor input text"], swap_endian=0 -signature file-magic-auto587 { - file-mime "text/troff", 39 - file-magic /(.*)(\x2e\x5c\x22)/ -} +#signature file-magic-auto587 { +# file-mime "text/troff", 39 +# file-magic /(.*)(\x2e\x5c\x22)/ +#} # >0 search/1,='\" (len=3), ["troff or preprocessor input text"], swap_endian=0 -signature file-magic-auto588 { - file-mime "text/troff", 39 - file-magic /(.*)(\x27\x5c\x22)/ -} +#signature file-magic-auto588 { +# file-mime "text/troff", 39 +# file-magic /(.*)(\x27\x5c\x22)/ +#} # >0 search/1,=0 search/1,=''' (len=3), ["troff or preprocessor input text"], swap_endian=0 -signature file-magic-auto593 { - file-mime "text/troff", 39 - file-magic /(.*)(\x27\x27\x27)/ -} +#signature file-magic-auto593 { +# file-mime "text/troff", 39 +# file-magic /(.*)(\x27\x27\x27)/ +#} # >0 search/4096,=try: (len=4), [""], swap_endian=0 # >>&0 regex,=^\s*except.*: (len=13), ["Python script text executable"], swap_endian=0 @@ -3999,12 +3623,6 @@ signature file-magic-auto596 { file-magic /(.*)(\x22LIBHDR\x22)/ } -# >0 regex,=^SUBDIRS (len=8), ["automake makefile script text"], swap_endian=0 -signature file-magic-auto597 { - file-mime "text/x-makefile", 38 - file-magic /(.*)(SUBDIRS)/ -} - # >0 search/4096,=(defvar (len=8), ["Lisp/Scheme program text"], swap_endian=0 signature file-magic-auto598 { file-mime "text/x-lisp", 38 @@ -4031,19 +3649,6 @@ signature file-magic-auto600 { # file-magic /(.*)(\x2a\x2a\x2a )/ #} -# >0 search/1,='.\" (len=4), ["troff or preprocessor input text"], swap_endian=0 -signature file-magic-auto602 { - file-mime "text/troff", 38 - file-magic /(.*)(\x27\x2e\x5c\x22)/ -} - -# LDFLAGS appears in other contexts, e.g. shell script. -# >0 regex,=^LDFLAGS (len=8), ["makefile script text"], swap_endian=0 -#signature file-magic-auto603 { -# file-mime "text/x-makefile", 38 -# file-magic /(.*)(LDFLAGS)/ -#} - # >0 search/8192,="libhdr" (len=8), ["BCPL source text"], swap_endian=0 signature file-magic-auto604 { file-mime "text/x-bcpl", 38 @@ -4057,12 +3662,6 @@ signature file-magic-auto604 { # file-magic /(^record)/ #} -# >0 regex,=^CFLAGS (len=7), ["makefile script text"], swap_endian=0 -signature file-magic-auto606 { - file-mime "text/x-makefile", 37 - file-magic /(.*)(CFLAGS)/ -} - # >0 search/4096,=(defun (len=7), ["Lisp/Scheme program text"], swap_endian=0 signature file-magic-auto607 { file-mime "text/x-lisp", 37 diff --git a/scripts/base/frameworks/files/magic/msoffice.sig b/scripts/base/frameworks/files/magic/msoffice.sig new file mode 100644 index 0000000000..111ec77004 --- /dev/null +++ b/scripts/base/frameworks/files/magic/msoffice.sig @@ -0,0 +1,28 @@ + +# This signature is non-specific and terrible but after +# searching for a long time there doesn't seem to be a +# better option. +signature file-msword { + file-magic /^\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1/ + file-mime "application/msword", 50 +} + +signature file-ooxml { + file-magic /^PK\x03\x04\x14\x00\x06\x00/ + file-mime "application/vnd.openxmlformats-officedocument", 50 +} + +signature file-docx { + file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|word\x2f).*PK\x03\x04.{26}word\x2f/ + file-mime "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 80 +} + +signature file-xlsx { + file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|xl\2f).*PK\x03\x04.{26}xl\x2f/ + file-mime "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 80 +} + +signature file-pptx { + file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|ppt\x2f).*PK\x03\x04.{26}ppt\x2f/ + file-mime "application/vnd.openxmlformats-officedocument.presentationml.presentation", 80 +} diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index d680c467b6..f1f381a141 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -100,8 +100,9 @@ export { ## during the process of analysis e.g. due to dropped packets. missing_bytes: count &log &default=0; - ## The number of not all-in-sequence bytes in the file stream that - ## were delivered to file analyzers due to reassembly buffer overflow. + ## The number of bytes in the file stream that were not delivered to + ## stream file analyzers. This could be overlapping bytes or + ## bytes that couldn't be reassembled. overflow_bytes: count &log &default=0; ## Whether the file analysis timed out at least once for the file. @@ -124,6 +125,37 @@ export { ## generate two handles that would hash to the same file id. const salt = "I recommend changing this." &redef; + ## Decide if you want to automatically attached analyzers to + ## files based on the detected mime type of the file. + const analyze_by_mime_type_automatically = T &redef; + + ## The default setting for if the file reassembler is enabled for + ## each file. + const enable_reassembler = T &redef; + + ## The default allow per-file reassembly buffer size. + const reassembly_buffer_size = 1048576 &redef; + + ## Allows the file reassembler to be used if it's necessary because the + ## file is transferred out of order. + ## + ## f: the file. + global enable_reassembly: function(f: fa_file); + + ## Disables the file reassembler on this file. If the file is not + ## transferred out of order this will have no effect. + ## + ## f: the file. + global disable_reassembly: function(f: fa_file); + + ## Set the maximum size the reassembly buffer is allowed to grow + ## for the given file. + ## + ## f: the file. + ## + ## max: Maximum allowed size of the reassembly buffer. + global set_reassembly_buffer_size: function(f: fa_file, max: count); + ## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is ## used to determine the length of inactivity that is allowed for a file ## before internal state related to it is cleaned up. When used within @@ -153,15 +185,6 @@ export { tag: Files::Tag, args: AnalyzerArgs &default=AnalyzerArgs()): bool; - ## Adds all analyzers associated with a give MIME type to the analysis of - ## a file. Note that analyzers added via MIME types cannot take further - ## arguments. - ## - ## f: the file. - ## - ## mtype: the MIME type; it will be compared case-insensitive. - global add_analyzers_for_mime_type: function(f: fa_file, mtype: string); - ## Removes an analyzer from the analysis of a given file. ## ## f: the file. @@ -284,6 +307,7 @@ global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table() # Store the MIME type to analyzer mappings. global mime_types: table[Analyzer::Tag] of set[string]; +global mime_type_to_analyzers: table[string] of set[Analyzer::Tag]; global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table(); @@ -313,8 +337,6 @@ function set_info(f: fa_file) f$info$overflow_bytes = f$overflow_bytes; if ( f?$is_orig ) f$info$is_orig = f$is_orig; - if ( f?$mime_type ) - f$info$mime_type = f$mime_type; } function set_timeout_interval(f: fa_file, t: interval): bool @@ -322,6 +344,21 @@ function set_timeout_interval(f: fa_file, t: interval): bool return __set_timeout_interval(f$id, t); } +function enable_reassembly(f: fa_file) + { + __enable_reassembly(f$id); + } + +function disable_reassembly(f: fa_file) + { + __disable_reassembly(f$id); + } + +function set_reassembly_buffer_size(f: fa_file, max: count) + { + __set_reassembly_buffer(f$id, max); + } + function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool { add f$info$analyzers[Files::analyzer_name(tag)]; @@ -337,15 +374,6 @@ function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool return T; } -function add_analyzers_for_mime_type(f: fa_file, mtype: string) - { - local dummy_args: AnalyzerArgs; - local analyzers = __add_analyzers_for_mime_type(f$id, mtype, dummy_args); - - for ( tag in analyzers ) - add f$info$analyzers[Files::analyzer_name(tag)]; - } - function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs)) { analyzer_add_callbacks[tag] = callback; @@ -366,42 +394,6 @@ function analyzer_name(tag: Files::Tag): string return __analyzer_name(tag); } -event file_new(f: fa_file) &priority=10 - { - set_info(f); - - if ( f?$mime_type ) - add_analyzers_for_mime_type(f, f$mime_type); - } - -event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10 - { - set_info(f); - add f$info$conn_uids[c$uid]; - local cid = c$id; - add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h]; - if( |Site::local_nets| > 0 ) - f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h); - - add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h]; - } - -event file_timeout(f: fa_file) &priority=10 - { - set_info(f); - f$info$timedout = T; - } - -event file_state_remove(f: fa_file) &priority=10 - { - set_info(f); - } - -event file_state_remove(f: fa_file) &priority=-10 - { - Log::write(Files::LOG, f$info); - } - function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool { local result = (tag !in registered_protocols); @@ -424,13 +416,18 @@ function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool { - if ( ! __register_for_mime_type(tag, mt) ) - return F; - if ( tag !in mime_types ) + { mime_types[tag] = set(); - + } add mime_types[tag][mt]; + + if ( mt !in mime_type_to_analyzers ) + { + mime_type_to_analyzers[mt] = set(); + } + add mime_type_to_analyzers[mt][tag]; + return T; } @@ -462,3 +459,62 @@ event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priorit local handler = registered_protocols[tag]; set_file_handle(handler$get_file_handle(c, is_orig)); } + +event file_new(f: fa_file) &priority=10 + { + set_info(f); + + if ( enable_reassembler ) + { + Files::enable_reassembly(f); + Files::set_reassembly_buffer_size(f, reassembly_buffer_size); + } + } + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10 + { + set_info(f); + + add f$info$conn_uids[c$uid]; + local cid = c$id; + add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h]; + if( |Site::local_nets| > 0 ) + f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h); + + add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h]; + } + +event file_mime_type(f: fa_file, mime_type: string) &priority=10 + { + set_info(f); + + f$info$mime_type = mime_type; + + + if ( analyze_by_mime_type_automatically && + mime_type in mime_type_to_analyzers ) + { + local analyzers = mime_type_to_analyzers[mime_type]; + for ( a in analyzers ) + { + add f$info$analyzers[Files::analyzer_name(a)]; + Files::add_analyzer(f, a); + } + } + } + +event file_timeout(f: fa_file) &priority=10 + { + set_info(f); + f$info$timedout = T; + } + +event file_state_remove(f: fa_file) &priority=10 + { + set_info(f); + } + +event file_state_remove(f: fa_file) &priority=-10 + { + Log::write(Files::LOG, f$info); + } diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index 5b31dd964e..b5f305a7fc 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -289,8 +289,8 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 if ( ! info?$fuid ) info$fuid = s$f$id; - if ( ! info?$file_mime_type && s$f?$mime_type ) - info$file_mime_type = s$f$mime_type; + if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type ) + info$file_mime_type = s$f$info$mime_type; if ( ! info?$file_desc ) info$file_desc = Files::describe(s$f); diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 4790245de0..d7d9bd61c9 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -531,8 +531,8 @@ function create_file_info(f: fa_file): Notice::FileInfo local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id, $desc = Files::describe(f)); - if ( f?$mime_type ) - fi$mime = f$mime_type; + if ( f?$info && f$info?$mime_type ) + fi$mime = f$info$mime_type; if ( f?$conns && |f$conns| == 1 ) for ( id in f$conns ) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index efce524fc5..b112d3ea0f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -353,9 +353,9 @@ type connection: record { ## gives up and discards any internal state related to the file. const default_file_timeout_interval: interval = 2 mins &redef; -## Default amount of bytes that file analysis will buffer before raising -## :bro:see:`file_new`. -const default_file_bof_buffer_size: count = 1024 &redef; +## Default amount of bytes that file analysis will buffer to provide +## data back in time to attached analyzers +const default_file_bof_buffer_size: count = 4096 &redef; ## A file that Bro is analyzing. This is Bro's type for describing the basic ## internal metadata collected about a "file", which is essentially just a @@ -394,8 +394,9 @@ type fa_file: record { ## during the process of analysis e.g. due to dropped packets. missing_bytes: count &default=0; - ## The number of not all-in-sequence bytes in the file stream that - ## were delivered to file analyzers due to reassembly buffer overflow. + ## The number of bytes in the file stream that were not delivered to + ## stream file analyzers. This could be overlapping bytes or + ## bytes that couldn't be reassembled. overflow_bytes: count &default=0; ## The amount of time between receiving new data for this file that @@ -409,16 +410,6 @@ type fa_file: record { ## The content of the beginning of a file up to *bof_buffer_size* bytes. ## This is also the buffer that's used for file/mime type detection. bof_buffer: string &optional; - - ## The mime type of the strongest file magic signature matches against - ## the data chunk in *bof_buffer*, or in the cases where no buffering - ## of the beginning of file occurs, an initial guess of the mime type - ## based on the first data seen. - mime_type: string &optional; - - ## All mime types that matched file magic signatures against the data - ## chunk in *bof_buffer*, in order of their strength value. - mime_types: mime_matches &optional; } &redef; ## Fields of a SYN packet. diff --git a/scripts/base/protocols/ftp/files.bro b/scripts/base/protocols/ftp/files.bro index b507ca32a7..617b57348b 100644 --- a/scripts/base/protocols/ftp/files.bro +++ b/scripts/base/protocols/ftp/files.bro @@ -17,6 +17,10 @@ export { ## Describe the file being transferred. global describe_file: function(f: fa_file): string; + + redef record fa_file += { + ftp: FTP::Info &optional; + }; } function get_file_handle(c: connection, is_orig: bool): string @@ -48,7 +52,6 @@ event bro_init() &priority=5 $describe = FTP::describe_file]); } - event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 { if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) @@ -56,6 +59,14 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p]; ftp$fuid = f$id; - if ( f?$mime_type ) - ftp$mime_type = f$mime_type; + + f$ftp = ftp; + } + +event file_mime_type(f: fa_file, mime_type: string) &priority=5 + { + if ( ! f?$ftp ) + return; + + f$ftp$mime_type = mime_type; } diff --git a/scripts/base/protocols/http/entities.bro b/scripts/base/protocols/http/entities.bro index ff5c915801..9fcf7f24f7 100644 --- a/scripts/base/protocols/http/entities.bro +++ b/scripts/base/protocols/http/entities.bro @@ -35,6 +35,10 @@ export { ## body. resp_mime_depth: count &default=0; }; + + redef record fa_file += { + http: HTTP::Info &optional; + }; } event http_begin_entity(c: connection, is_orig: bool) &priority=10 @@ -67,6 +71,8 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori { if ( f$source == "HTTP" && c?$http ) { + f$http = c$http; + if ( c$http?$current_entity && c$http$current_entity?$filename ) f$info$filename = c$http$current_entity$filename; @@ -76,14 +82,6 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori c$http$orig_fuids = string_vec(f$id); else c$http$orig_fuids[|c$http$orig_fuids|] = f$id; - - if ( f?$mime_type ) - { - if ( ! c$http?$orig_mime_types ) - c$http$orig_mime_types = string_vec(f$mime_type); - else - c$http$orig_mime_types[|c$http$orig_mime_types|] = f$mime_type; - } } else { @@ -91,17 +89,29 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori c$http$resp_fuids = string_vec(f$id); else c$http$resp_fuids[|c$http$resp_fuids|] = f$id; - - if ( f?$mime_type ) - { - if ( ! c$http?$resp_mime_types ) - c$http$resp_mime_types = string_vec(f$mime_type); - else - c$http$resp_mime_types[|c$http$resp_mime_types|] = f$mime_type; - } } } + } +event file_mime_type(f: fa_file, mime_type: string) &priority=5 + { + if ( ! f?$http || ! f?$is_orig ) + return; + + if ( f$is_orig ) + { + if ( ! f$http?$orig_mime_types ) + f$http$orig_mime_types = string_vec(mime_type); + else + f$http$orig_mime_types[|f$http$orig_mime_types|] = mime_type; + } + else + { + if ( ! f$http?$resp_mime_types ) + f$http$resp_mime_types = string_vec(mime_type); + else + f$http$resp_mime_types[|f$http$resp_mime_types|] = mime_type; + } } event http_end_entity(c: connection, is_orig: bool) &priority=5 diff --git a/scripts/base/protocols/irc/files.bro b/scripts/base/protocols/irc/files.bro index 7e077c8331..518775abb4 100644 --- a/scripts/base/protocols/irc/files.bro +++ b/scripts/base/protocols/irc/files.bro @@ -12,6 +12,10 @@ export { ## Default file handle provider for IRC. global get_file_handle: function(c: connection, is_orig: bool): string; + + redef record fa_file += { + irc: IRC::Info &optional; + }; } function get_file_handle(c: connection, is_orig: bool): string @@ -34,6 +38,12 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori irc$fuid = f$id; if ( irc?$dcc_file_name ) f$info$filename = irc$dcc_file_name; - if ( f?$mime_type ) - irc$dcc_mime_type = f$mime_type; + + f$irc = irc; } + +event file_mime_type(f: fa_file, mime_type: string) &priority=5 + { + if ( f?$irc ) + f$irc$dcc_mime_type = mime_type; + } \ No newline at end of file diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro index 3000d2d83e..d0b8a852e6 100644 --- a/scripts/policy/frameworks/files/detect-MHR.bro +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -66,6 +66,7 @@ function do_mhr_lookup(hash: string, fi: Notice::FileInfo) event file_hash(f: fa_file, kind: string, hash: string) { - if ( kind == "sha1" && f?$mime_type && match_file_types in f$mime_type ) + if ( kind == "sha1" && f?$info && f$info?$mime_type && + match_file_types in f$info$mime_type ) do_mhr_lookup(hash, Notice::create_file_info(f)); } diff --git a/src/Frag.cc b/src/Frag.cc index d0389c264a..8ada148750 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -28,7 +28,7 @@ void FragTimer::Dispatch(double t, int /* is_expire */) FragReassembler::FragReassembler(NetSessions* arg_s, const IP_Hdr* ip, const u_char* pkt, HashKey* k, double t) - : Reassembler(0, REASSEM_IP) + : Reassembler(0) { s = arg_s; key = k; diff --git a/src/Reassem.cc b/src/Reassem.cc index 1ad0cb2717..8bf965427b 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -31,7 +31,7 @@ DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq, uint64 Reassembler::total_size = 0; -Reassembler::Reassembler(uint64 init_seq, ReassemblerType arg_type) +Reassembler::Reassembler(uint64 init_seq) { blocks = last_block = 0; trim_seq = last_reassem_seq = init_seq; diff --git a/src/Reassem.h b/src/Reassem.h index 7b77a628d8..39617f7816 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -22,11 +22,10 @@ public: }; -enum ReassemblerType { REASSEM_IP, REASSEM_TCP }; class Reassembler : public BroObj { public: - Reassembler(uint64 init_seq, ReassemblerType arg_type); + Reassembler(uint64 init_seq); virtual ~Reassembler(); void NewBlock(double t, uint64 seq, uint64 len, const u_char* data); diff --git a/src/SerialTypes.h b/src/SerialTypes.h index 81ccbc030e..d2f227838c 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -87,6 +87,7 @@ SERIAL_TCP_CONTENTS(TCP_NVT, 3) #define SERIAL_REASSEMBLER(name, val) SERIAL_CONST(name, val, REASSEMBLER) SERIAL_REASSEMBLER(REASSEMBLER, 1) SERIAL_REASSEMBLER(TCP_REASSEMBLER, 2) +SERIAL_REASSEMBLER(FILE_REASSEMBLER, 3) #define SERIAL_VAL(name, val) SERIAL_CONST(name, val, VAL) SERIAL_VAL(VAL, 1) diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index e00e32ef1b..16bb9cc56d 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -28,7 +28,7 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, TCP_Analyzer* arg_tcp_analyzer, TCP_Reassembler::Type arg_type, TCP_Endpoint* arg_endp) - : Reassembler(1, REASSEM_TCP) + : Reassembler(1) { dst_analyzer = arg_dst_analyzer; tcp_analyzer = arg_tcp_analyzer; diff --git a/src/event.bif b/src/event.bif index 4006888eab..dd941b6736 100644 --- a/src/event.bif +++ b/src/event.bif @@ -905,7 +905,8 @@ event get_file_handle%(tag: Analyzer::Tag, c: connection, is_orig: bool%); ## ## f: The file. ## -## .. bro:see:: file_over_new_connection file_timeout file_gap file_state_remove +## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type +## file_state_remove event file_new%(f: fa_file%); ## Indicates that a file has been seen being transferred over a connection @@ -917,16 +918,39 @@ event file_new%(f: fa_file%); ## ## is_orig: true if the originator of *c* is the one sending the file. ## -## .. bro:see:: file_new file_timeout file_gap file_state_remove +## .. bro:see:: file_new file_timeout file_gap file_mime_type +## file_state_remove event file_over_new_connection%(f: fa_file, c: connection, is_orig: bool%); +## Provide the most likely matching MIME type for this file. The analysis +## can be augmented at this time via :bro:see:`Files::add_analyzer`. +## +## f: The file. +## +## mime_type: The mime type that was discovered. +## +## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type +## file_mime_types file_state_remove +event file_mime_type%(f: fa_file, mime_type: string%); + +## Provide all matching MIME types for this file. The analysis can be +## augmented at this time via :bro:see:`Files::add_analyzer`. +## +## f: The file. +## +## mime_types: The mime types that were discovered. +## +## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type +## file_mime_types file_state_remove +event file_mime_types%(f: fa_file, mime_types: mime_matches%); + ## Indicates that file analysis has timed out because no activity was seen ## for the file in a while. ## ## f: The file. ## -## .. bro:see:: file_new file_over_new_connection file_gap file_state_remove -## default_file_timeout_interval Files::set_timeout_interval +## .. bro:see:: file_new file_over_new_connection file_gap file_mime_type +## file_mime_types file_state_remove default_file_timeout_interval ## Files::set_timeout_interval event file_timeout%(f: fa_file%); @@ -938,14 +962,34 @@ event file_timeout%(f: fa_file%); ## ## len: The number of missing bytes. ## -## .. bro:see:: file_new file_over_new_connection file_timeout file_state_remove +## .. bro:see:: file_new file_over_new_connection file_timeout file_mime_type +## file_mime_types file_state_remove file_reassembly_overflow event file_gap%(f: fa_file, offset: count, len: count%); +## Indicates that the file had an overflow of the reassembly buffer. +## This is a specialization of the :bro:id:`file_gap` event. +## +## f: The file. +## +## offset: The byte offset from the start of the file at which the reassembly +## couldn't continue due to running out of reassembly buffer space. +## +## skipped: The number of bytes of the file skipped over to flush some +## file data and get back under the reassembly buffer size limit. +## This value will also be represented as a gap. +## +## .. bro:see:: file_new file_over_new_connection file_timeout file_mime_type +## file_mime_types file_state_remove file_gap Files::enable_reassembler +## Files::reassembly_buffer_size Files::enable_reassembly +## Files::disable_reassembly Files::set_reassembly_buffer_size +event file_reassembly_overflow%(f: fa_file, offset: count, skipped: count%); + ## This event is generated each time file analysis is ending for a given file. ## ## f: The file. ## ## .. bro:see:: file_new file_over_new_connection file_timeout file_gap +## file_mime_type file_mime_types event file_state_remove%(f: fa_file%); ## Generated when an internal DNS lookup produces the same result as last time. diff --git a/src/file_analysis/AnalyzerSet.cc b/src/file_analysis/AnalyzerSet.cc index b3f11b6816..2657a5b709 100644 --- a/src/file_analysis/AnalyzerSet.cc +++ b/src/file_analysis/AnalyzerSet.cc @@ -72,10 +72,10 @@ bool AnalyzerSet::Add(file_analysis::Tag tag, RecordVal* args) return true; } -bool AnalyzerSet::QueueAdd(file_analysis::Tag tag, RecordVal* args) +bool AnalyzerSet::QueueAdd(file_analysis::Tag tag, RecordVal* args, file_analysis::Analyzer* a) { HashKey* key = GetKey(tag, args); - file_analysis::Analyzer* a = InstantiateAnalyzer(tag, args); + a = InstantiateAnalyzer(tag, args); if ( ! a ) { diff --git a/src/file_analysis/AnalyzerSet.h b/src/file_analysis/AnalyzerSet.h index 38eddb8967..839425980c 100644 --- a/src/file_analysis/AnalyzerSet.h +++ b/src/file_analysis/AnalyzerSet.h @@ -57,9 +57,10 @@ public: * Queue the attachment of an analyzer to #file. * @param tag the analyzer tag of the file analyzer to add. * @param args an \c AnalyzerArgs value which specifies an analyzer. + * @param a an analyzer pointer to return the instantiated analyzer to the caller. * @return true if analyzer was able to be instantiated, else false. */ - bool QueueAdd(file_analysis::Tag tag, RecordVal* args); + bool QueueAdd(file_analysis::Tag tag, RecordVal* args, file_analysis::Analyzer* a); /** * Remove an analyzer from #file immediately. diff --git a/src/file_analysis/CMakeLists.txt b/src/file_analysis/CMakeLists.txt index 846fc4bf15..34dc8d5387 100644 --- a/src/file_analysis/CMakeLists.txt +++ b/src/file_analysis/CMakeLists.txt @@ -11,6 +11,7 @@ set(file_analysis_SRCS Manager.cc File.cc FileTimer.cc + FileReassembler.cc Analyzer.cc AnalyzerSet.cc Component.cc diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 4509fc7d42..e4e9b6dc9d 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -53,8 +53,6 @@ int File::overflow_bytes_idx = -1; int File::timeout_interval_idx = -1; int File::bof_buffer_size_idx = -1; int File::bof_buffer_idx = -1; -int File::mime_type_idx = -1; -int File::mime_types_idx = -1; void File::StaticInit() { @@ -74,15 +72,14 @@ void File::StaticInit() timeout_interval_idx = Idx("timeout_interval"); bof_buffer_size_idx = Idx("bof_buffer_size"); bof_buffer_idx = Idx("bof_buffer"); - mime_type_idx = Idx("mime_type"); - mime_types_idx = Idx("mime_types"); } File::File(const string& file_id, Connection* conn, analyzer::Tag tag, bool is_orig) - : id(file_id), val(0), postpone_timeout(false), first_chunk(true), - missed_bof(false), need_reassembly(false), done(false), - did_file_new_event(false), analyzers(this) + : id(file_id), val(0), file_reassembler(0), stream_offset(0), + reassembly_max_buffer(0), did_mime_type(false), + reassembly_enabled(false), postpone_timeout(false), done(false), + analyzers(this) { StaticInit(); @@ -96,7 +93,6 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag, // add source, connection, is_orig fields SetSource(analyzer_mgr->GetComponentName(tag)); val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL)); - UpdateConnectionFields(conn, is_orig); } UpdateLastActivityTime(); @@ -107,10 +103,9 @@ File::~File() DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Destroying File object", id.c_str()); Unref(val); - while ( ! fonc_queue.empty() ) + if ( file_reassembler ) { - delete_vals(fonc_queue.front().second); - fonc_queue.pop(); + delete file_reassembler; } } @@ -150,11 +145,7 @@ void File::UpdateConnectionFields(Connection* conn, bool is_orig) vl->append(conn_val->Ref()); vl->append(new Val(is_orig, TYPE_BOOL)); - if ( did_file_new_event ) - FileEvent(file_over_new_connection, vl); - else - fonc_queue.push(pair( - file_over_new_connection, vl)); + FileEvent(file_over_new_connection, vl); } } @@ -242,7 +233,7 @@ bool File::IsComplete() const if ( ! total ) return false; - if ( LookupFieldDefaultCount(seen_bytes_idx) >= total->AsCount() ) + if ( stream_offset >= total->AsCount() ) return true; return false; @@ -258,7 +249,23 @@ bool File::AddAnalyzer(file_analysis::Tag tag, RecordVal* args) DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing addition of %s analyzer", id.c_str(), file_mgr->GetComponentName(tag).c_str()); - return done ? false : analyzers.QueueAdd(tag, args); + if ( done ) + return false; + + file_analysis::Analyzer *a = 0; + bool success = analyzers.QueueAdd(tag, args, a); + if ( success && a ) + { + // Catch up this analyzer with the BOF buffer + for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i ) + { + if ( ! a->DeliverStream(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len()) ) + { + analyzers.QueueRemove(a->Tag(), a->Args()); + } + } + } + return success; } bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args) @@ -269,9 +276,61 @@ bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args) return done ? false : analyzers.QueueRemove(tag, args); } +void File::EnableReassembly() + { + reassembly_enabled = true; + } + +void File::DisableReassembly() + { + reassembly_enabled = false; + if ( file_reassembler ) + { + delete file_reassembler; + file_reassembler = NULL; + } + } + +void File::SetReassemblyBuffer(uint64 max) + { + reassembly_max_buffer = max; + } + +bool File::DetectMIME() + { + RuleMatcher::MIME_Matches matches; + + BroString *bs = concatenate(bof_buffer.chunks); + const u_char* data = bs->Bytes(); + uint64 len = bs->Len(); + len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx)); + file_mgr->DetectMIME(data, len, &matches); + + if ( matches.empty() ) + return false; + + if ( FileEventAvailable(file_mime_type) ) + { + val_list* vl = new val_list(); + vl->append(val->Ref()); + vl->append(new StringVal(*(matches.begin()->second.begin()))); + FileEvent(file_mime_type, vl); + } + + if ( FileEventAvailable(file_mime_types) ) + { + val_list* vl = new val_list(); + vl->append(val->Ref()); + vl->append(file_analysis::GenMIMEMatchesVal(matches)); + FileEvent(file_mime_types, vl); + } + + return true; + } + bool File::BufferBOF(const u_char* data, uint64 len) { - if ( bof_buffer.full || bof_buffer.replayed ) + if ( bof_buffer.full ) return false; uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx); @@ -282,101 +341,91 @@ bool File::BufferBOF(const u_char* data, uint64 len) if ( bof_buffer.size >= desired_size ) { bof_buffer.full = true; - ReplayBOF(); } return true; } -bool File::DetectMIME(const u_char* data, uint64 len) +void File::DeliverStream(const u_char* data, uint64 len) { - RuleMatcher::MIME_Matches matches; - len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx)); - file_mgr->DetectMIME(data, len, &matches); + // Buffer enough data for the BOF buffer + BufferBOF(data, len); - if ( matches.empty() ) - return false; - - val->Assign(mime_type_idx, - new StringVal(*(matches.begin()->second.begin()))); - val->Assign(mime_types_idx, file_analysis::GenMIMEMatchesVal(matches)); - - return true; - } - -void File::ReplayBOF() - { - if ( bof_buffer.replayed ) - return; - - bof_buffer.replayed = true; - - if ( bof_buffer.chunks.empty() ) + // TODO: mime matching size needs defined. + if ( ! did_mime_type && + bof_buffer.size >= 1024 && + LookupFieldDefaultCount(missing_bytes_idx) == 0 ) { - // Since we missed the beginning, try file type detect on next data in. - missed_bof = true; - return; + did_mime_type = true; + DetectMIME(); + + // TODO: this needs to be done elsewhere. For now it's here. + BroString* bs = concatenate(bof_buffer.chunks); + val->Assign(bof_buffer_idx, new StringVal(bs)); } - BroString* bs = concatenate(bof_buffer.chunks); - val->Assign(bof_buffer_idx, new StringVal(bs)); - - DetectMIME(bs->Bytes(), bs->Len()); - FileEvent(file_new); - - for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i ) - DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len()); - } - -void File::DataIn(const u_char* data, uint64 len, uint64 offset) - { - analyzers.DrainModifications(); - - if ( first_chunk ) - { - // TODO: this should all really be delayed until we attempt reassembly - DetectMIME(data, len); - FileEvent(file_new); - first_chunk = false; - } - - DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in at offset" PRIu64 "; %s [%s]", - id.c_str(), len, offset, + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in at offset %" PRIu64 "; %s [%s]", + id.c_str(), len, stream_offset, IsComplete() ? "complete" : "incomplete", fmt_bytes((const char*) data, min((uint64)40, len)), len > 40 ? "..." : ""); file_analysis::Analyzer* a = 0; IterCookie* c = analyzers.InitForIteration(); - while ( (a = analyzers.NextEntry(c)) ) { - if ( ! a->DeliverChunk(data, len, offset) ) + if ( !a->DeliverStream(data, len) ) + { analyzers.QueueRemove(a->Tag(), a->Args()); + } } - analyzers.DrainModifications(); - - // TODO: check reassembly requirement based on buffer size in record - if ( need_reassembly ) - reporter->InternalError("file_analyzer::File TODO: reassembly not yet supported"); - - // TODO: reassembly overflow stuff, increment overflow count, eval trigger - + stream_offset += len; IncrementByteCount(len, seen_bytes_idx); } -void File::DataIn(const u_char* data, uint64 len) +void File::DeliverChunk(const u_char* data, uint64 len, uint64 offset) { - analyzers.DrainModifications(); - - if ( BufferBOF(data, len) ) - return; - - if ( missed_bof ) + // Potentially handle reassembly and deliver to the stream analyzers. + if ( file_reassembler ) { - DetectMIME(data, len); - FileEvent(file_new); - missed_bof = false; + if ( reassembly_max_buffer > 0 && + reassembly_max_buffer < file_reassembler->TotalSize() ) + { + uint64 first_offset = file_reassembler->GetFirstBlockOffset(); + int gap_bytes = file_reassembler->TrimToSeq(first_offset); + + if ( FileEventAvailable(file_reassembly_overflow) ) + { + val_list* vl = new val_list(); + vl->append(val->Ref()); + vl->append(new Val(stream_offset, TYPE_COUNT)); + vl->append(new Val(gap_bytes, TYPE_COUNT)); + FileEvent(file_reassembly_overflow, vl); + } + + Gap(stream_offset, gap_bytes); + } + + // Forward data to the reassembler. + file_reassembler->NewBlock(network_time, offset, len, data); + } + else if ( stream_offset == offset ) + { + // This is the normal case where a file is transferred linearly. + // Nothing special should be done here. + DeliverStream(data, len); + } + else if ( reassembly_enabled ) + { + // This is data that doesn't match the offset and the reassembler + // needs to be enabled. + file_reassembler = new FileReassembler(this, stream_offset); + file_reassembler->NewBlock(network_time, offset, len, data); + } + else + { + // We can't reassemble so we throw out the data for streaming. + IncrementByteCount(len, overflow_bytes_idx); } DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in; %s [%s]", @@ -386,24 +435,37 @@ void File::DataIn(const u_char* data, uint64 len) file_analysis::Analyzer* a = 0; IterCookie* c = analyzers.InitForIteration(); - while ( (a = analyzers.NextEntry(c)) ) { - if ( ! a->DeliverStream(data, len) ) + if ( !a->DeliverChunk(data, len, offset) ) { analyzers.QueueRemove(a->Tag(), a->Args()); - continue; } - - uint64 offset = LookupFieldDefaultCount(seen_bytes_idx) + - LookupFieldDefaultCount(missing_bytes_idx); - - if ( ! a->DeliverChunk(data, len, offset) ) - analyzers.QueueRemove(a->Tag(), a->Args()); } + if ( IsComplete() ) + { + // If the file is complete we can automatically go and close out the file from here. + EndOfFile(); + } + } + + +void File::DataIn(const u_char* data, uint64 len, uint64 offset) + { + analyzers.DrainModifications(); + DeliverChunk(data, len, offset); + analyzers.DrainModifications(); + } + +void File::DataIn(const u_char* data, uint64 len) + { + analyzers.DrainModifications(); + + uint64 offset = LookupFieldDefaultCount(seen_bytes_idx) + + LookupFieldDefaultCount(missing_bytes_idx); + DeliverChunk(data, len, offset); analyzers.DrainModifications(); - IncrementByteCount(len, seen_bytes_idx); } void File::EndOfFile() @@ -413,10 +475,19 @@ void File::EndOfFile() if ( done ) return; - analyzers.DrainModifications(); + if ( ! did_mime_type ) + { + DetectMIME(); - // Send along anything that's been buffered, but never flushed. - ReplayBOF(); + // TODO: this also needs to be done elsewhere. + if ( bof_buffer.size > 0 ) + { + BroString* bs = concatenate(bof_buffer.chunks); + val->Assign(bof_buffer_idx, new StringVal(bs)); + } + } + + analyzers.DrainModifications(); done = true; @@ -441,10 +512,6 @@ void File::Gap(uint64 offset, uint64 len) analyzers.DrainModifications(); - // If we were buffering the beginning of the file, a gap means we've got - // as much contiguous stuff at the beginning as possible, so work with that. - ReplayBOF(); - file_analysis::Analyzer* a = 0; IterCookie* c = analyzers.InitForIteration(); @@ -464,6 +531,8 @@ void File::Gap(uint64 offset, uint64 len) } analyzers.DrainModifications(); + + stream_offset += len; IncrementByteCount(len, missing_bytes_idx); } @@ -482,30 +551,13 @@ void File::FileEvent(EventHandlerPtr h) FileEvent(h, vl); } -static void flush_file_event_queue(queue >& q) - { - while ( ! q.empty() ) - { - pair p = q.front(); - mgr.QueueEvent(p.first, p.second); - q.pop(); - } - } - void File::FileEvent(EventHandlerPtr h, val_list* vl) { - if ( h == file_state_remove ) - flush_file_event_queue(fonc_queue); - mgr.QueueEvent(h, vl); - if ( h == file_new ) - { - did_file_new_event = true; - flush_file_event_queue(fonc_queue); - } - - if ( h == file_new || h == file_timeout || h == file_extraction_limit ) + if ( h == file_new || h == file_over_new_connection || + h == file_mime_type || + h == file_timeout || h == file_extraction_limit ) { // immediate feedback is required for these events. mgr.Drain(); diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index 86f60caf9f..bfd38a263c 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -8,6 +8,7 @@ #include #include +#include "FileReassembler.h" #include "Conn.h" #include "Val.h" #include "Tag.h" @@ -16,6 +17,8 @@ namespace file_analysis { +class FileReassembler; + /** * Wrapper class around \c fa_file record values from script layer. */ @@ -166,6 +169,7 @@ public: protected: friend class Manager; + friend class FileReassembler; /** * Constructor; only file_analysis::Manager should be creating these. @@ -227,12 +231,37 @@ protected: /** * Does mime type detection via file magic signatures and assigns * strongest matching mime type (if available) to \c mime_type - * field in #val. - * @param data pointer to a chunk of file data. - * @param len number of bytes in the data chunk. + * field in #val. It uses the data in the BOF buffer * @return whether a mime type match was found. */ - bool DetectMIME(const u_char* data, uint64 len); + bool DetectMIME(); + + /** + * Enables reassembly on the file. + */ + void EnableReassembly(); + + /** + * Disables reassembly on the file. If there is an existing reassembler + * for the file, this will cause it to be deleted and won't allow a new + * one to be created until reassembly is reenabled. + */ + void DisableReassembly(); + + /** + * Set a maximum allowed bytes of memory for file reassembly for this file. + */ + void SetReassemblyBuffer(uint64 max); + + /** + * Perform stream-wise delivery for analyzers that need it. + */ + void DeliverStream(const u_char* data, uint64 len); + + /** + * Perform chunk-wise delivery for analyzers that need it. + */ + void DeliverChunk(const u_char* data, uint64 len, uint64 offset); /** * Lookup a record field index/offset by name. @@ -246,25 +275,24 @@ protected: */ static void StaticInit(); -private: +protected: string id; /**< A pretty hash that likely identifies file */ RecordVal* val; /**< \c fa_file from script layer. */ + FileReassembler *file_reassembler; /**< A reassembler for the file if it's needed. */ + uint64 stream_offset; /**< The offset of the file which has been forwarded. */ + uint64 reassembly_max_buffer; /**< Maximum allowed buffer for reassembly. */ + bool did_mime_type; /**< Whether the mime type ident has already been attempted. */ + bool reassembly_enabled; /**< Whether file stream reassembly is needed. */ bool postpone_timeout; /**< Whether postponing timeout is requested. */ - bool first_chunk; /**< Track first non-linear chunk. */ - bool missed_bof; /**< Flags that we missed start of file. */ - bool need_reassembly; /**< Whether file stream reassembly is needed. */ bool done; /**< If this object is about to be deleted. */ - bool did_file_new_event; /**< Whether the file_new event has been done. */ - AnalyzerSet analyzers; /**< A set of attached file analyzer. */ - queue > fonc_queue; + AnalyzerSet analyzers; /**< A set of attached file analyzers. */ struct BOF_Buffer { - BOF_Buffer() : full(false), replayed(false), size(0) {} + BOF_Buffer() : full(false), size(0) {} ~BOF_Buffer() { for ( size_t i = 0; i < chunks.size(); ++i ) delete chunks[i]; } bool full; - bool replayed; uint64 size; BroString::CVec chunks; } bof_buffer; /**< Beginning of file buffer. */ diff --git a/src/file_analysis/FileReassembler.cc b/src/file_analysis/FileReassembler.cc new file mode 100644 index 0000000000..71e4c30bca --- /dev/null +++ b/src/file_analysis/FileReassembler.cc @@ -0,0 +1,65 @@ + +#include "FileReassembler.h" +#include "File.h" + + +namespace file_analysis { + +class File; + +FileReassembler::FileReassembler(File *f, uint64 starting_offset) + : Reassembler(starting_offset), the_file(f) + { + } + +FileReassembler::~FileReassembler() + { + } + +void FileReassembler::BlockInserted(DataBlock* start_block) + { + if ( start_block->seq > last_reassem_seq || + start_block->upper <= last_reassem_seq ) + return; + + for ( DataBlock* b = start_block; + b && b->seq <= last_reassem_seq; b = b->next ) + { + if ( b->seq == last_reassem_seq ) + { // New stuff. + uint64 len = b->Size(); + uint64 seq = last_reassem_seq; + last_reassem_seq += len; + the_file->DeliverStream(b->block, len); + } + } + + // Throw out forwarded data + TrimToSeq(last_reassem_seq); + } + +void FileReassembler::Undelivered(uint64 up_to_seq) + { + // Not doing anything here yet. + } + +void FileReassembler::Overlap(const u_char* b1, const u_char* b2, uint64 n) + { + // Not doing anything here yet. + } + +IMPLEMENT_SERIAL(FileReassembler, SER_FILE_REASSEMBLER); + +bool FileReassembler::DoSerialize(SerialInfo* info) const + { + reporter->InternalError("FileReassembler::DoSerialize not implemented"); + return false; // Cannot be reached. + } + +bool FileReassembler::DoUnserialize(UnserialInfo* info) + { + reporter->InternalError("FileReassembler::DoUnserialize not implemented"); + return false; // Cannot be reached. + } + +} // end file_analysis diff --git a/src/file_analysis/FileReassembler.h b/src/file_analysis/FileReassembler.h new file mode 100644 index 0000000000..c29563efc8 --- /dev/null +++ b/src/file_analysis/FileReassembler.h @@ -0,0 +1,47 @@ +#ifndef FILE_ANALYSIS_FILEREASSEMBLER_H +#define FILE_ANALYSIS_FILEREASSEMBLER_H + +#include "Reassem.h" +#include "File.h" + +class BroFile; +class Connection; + +namespace file_analysis { + +class File; + +//const int STOP_ON_GAP = 1; +//const int PUNT_ON_PARTIAL = 1; + +class FileReassembler : public Reassembler { +public: + + FileReassembler(File* f, uint64 starting_offset); + virtual ~FileReassembler(); + + void Done(); + uint64 GetFirstBlockOffset() { return blocks->seq; } + + // Checks if we have delivered all contents that we can possibly + // deliver for this endpoint. + void CheckEOF(); + +protected: + FileReassembler() { } + + DECLARE_SERIAL(FileReassembler); + + void Undelivered(uint64 up_to_seq); + void BlockInserted(DataBlock* b); + void Overlap(const u_char* b1, const u_char* b2, uint64 n); + + unsigned int had_gap:1; + unsigned int did_EOF:1; + unsigned int skip_deliveries:1; + File* the_file; +}; + +} // namespace analyzer::* + +#endif diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index 59c0fa0023..191bd1e1e4 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -232,6 +232,39 @@ bool Manager::SetTimeoutInterval(const string& file_id, double interval) const return true; } +bool Manager::EnableReassembly(const string& file_id) + { + File* file = LookupFile(file_id); + + if ( ! file ) + return false; + + file->EnableReassembly(); + return true; + } + +bool Manager::DisableReassembly(const string& file_id) + { + File* file = LookupFile(file_id); + + if ( ! file ) + return false; + + file->DisableReassembly(); + return true; + } + +bool Manager::SetReassemblyBuffer(const string& file_id, uint64 max) + { + File* file = LookupFile(file_id); + + if ( ! file ) + return false; + + file->SetReassemblyBuffer(max); + return true; + } + bool Manager::SetExtractionLimit(const string& file_id, RecordVal* args, uint64 n) const { @@ -254,28 +287,6 @@ bool Manager::AddAnalyzer(const string& file_id, file_analysis::Tag tag, return file->AddAnalyzer(tag, args); } -TableVal* Manager::AddAnalyzersForMIMEType(const string& file_id, const string& mtype, - RecordVal* args) - { - if ( ! tag_set_type ) - tag_set_type = internal_type("files_tag_set")->AsTableType(); - - TableVal* sval = new TableVal(tag_set_type); - TagSet* l = LookupMIMEType(mtype, false); - - if ( ! l ) - return sval; - - for ( TagSet::const_iterator i = l->begin(); i != l->end(); i++ ) - { - file_analysis::Tag tag = *i; - if ( AddAnalyzer(file_id, tag, args) ) - sval->Assign(tag.AsEnumVal(), 0); - } - - return sval; - } - bool Manager::RemoveAnalyzer(const string& file_id, file_analysis::Tag tag, RecordVal* args) const { @@ -304,6 +315,12 @@ File* Manager::GetFile(const string& file_id, Connection* conn, id_map.Insert(file_id.c_str(), rval); rval->ScheduleInactivityTimer(); + // Generate file_new here so the manager knows about the file. + rval->FileEvent(file_new); + // Same for file_over_new_connection which is generated by + // updating the connection fields. + rval->UpdateConnectionFields(conn, is_orig); + if ( IsIgnored(file_id) ) return 0; } @@ -461,63 +478,6 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const return a; } -Manager::TagSet* Manager::LookupMIMEType(const string& mtype, bool add_if_not_found) - { - MIMEMap::const_iterator i = mime_types.find(to_upper(mtype)); - - if ( i != mime_types.end() ) - return i->second; - - if ( ! add_if_not_found ) - return 0; - - TagSet* l = new TagSet; - mime_types.insert(std::make_pair(to_upper(mtype), l)); - return l; - } - -bool Manager::RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype) - { - Component* p = Lookup(tag); - - if ( ! p ) - return false; - - return RegisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString()); - } - -bool Manager::RegisterAnalyzerForMIMEType(Tag tag, const string& mtype) - { - TagSet* l = LookupMIMEType(mtype, true); - - DBG_LOG(DBG_FILE_ANALYSIS, "Register analyzer %s for MIME type %s", - GetComponentName(tag).c_str(), mtype.c_str()); - - l->insert(tag); - return true; - } - -bool Manager::UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype) - { - Component* p = Lookup(tag); - - if ( ! p ) - return false; - - return UnregisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString()); - } - -bool Manager::UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype) - { - TagSet* l = LookupMIMEType(mtype, true); - - DBG_LOG(DBG_FILE_ANALYSIS, "Unregister analyzer %s for MIME type %s", - GetComponentName(tag).c_str(), mtype.c_str()); - - l->erase(tag); - return true; - } - RuleMatcher::MIME_Matches* Manager::DetectMIME(const u_char* data, uint64 len, RuleMatcher::MIME_Matches* rval) const { diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 5a088e86cc..2e8efefcb0 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -213,6 +213,21 @@ public: */ bool SetTimeoutInterval(const string& file_id, double interval) const; + /** + * Enable the reassembler for a file. + */ + bool EnableReassembly(const string& file_id); + + /** + * Disable the reassembler for a file. + */ + bool DisableReassembly(const string& file_id); + + /** + * Set the reassembly for a file in bytes. + */ + bool SetReassemblyBuffer(const string& file_id, uint64 max); + /** * Sets a limit on the maximum size allowed for extracting the file * to local disk; @@ -238,18 +253,6 @@ public: bool AddAnalyzer(const string& file_id, file_analysis::Tag tag, RecordVal* args) const; - /** - * Queue attachment of an all analyzers associated with a given MIME - * type to the file identifier. - * - * @param file_id the file identifier/hash. - * @param mtype the MIME type; comparisions will be performanced case-insensitive. - * @param args a \c AnalyzerArgs value which describes a file analyzer. - * @return A ref'ed \c set[Tag] with all added analyzers. - */ - TableVal* AddAnalyzersForMIMEType(const string& file_id, const string& mtype, - RecordVal* args); - /** * Queue removal of an analyzer for a given file identifier. * @param file_id the file identifier/hash. @@ -277,62 +280,6 @@ public: Analyzer* InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const; /** - * Registers a MIME type for an analyzer. Once registered, files of - * that MIME type will automatically get a corresponding analyzer - * assigned. - * - * @param tag The analyzer's tag as an enum of script type \c - * Files::Tag. - * - * @param mtype The MIME type. It will be matched case-insenistive. - * - * @return True if successful. - */ - bool RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype); - - /** - * Registers a MIME type for an analyzer. Once registered, files of - * that MIME type will automatically get a corresponding analyzer - * assigned. - * - * @param tag The analyzer's tag as an enum of script type \c - * Files::Tag. - * - * @param mtype The MIME type. It will be matched case-insenistive. - * - * @return True if successful. - */ - bool RegisterAnalyzerForMIMEType(Tag tag, const string& mtype); - - /** - * Unregisters a MIME type for an analyzer. - * - * @param tag The analyzer's tag as an enum of script type \c - * Files::Tag. - * - * @param mtype The MIME type. It will be matched case-insenistive. - * - * @return True if successful (incl. when the type wasn't actually - * registered for the analyzer). - * - */ - bool UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype); - - /** - * Unregisters a MIME type for an analyzer. - * - * @param tag The analyzer's tag as an enum of script type \c - * Files::Tag. - * - * @param mtype The MIME type. It will be matched case-insenistive. - * - * @return True if successful (incl. when the type wasn't actually - * registered for the analyzer). - * - */ - bool UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype); - - /** * Returns a set of all matching MIME magic signatures for a given * chunk of data. * @param data A chunk of bytes to match magic MIME signatures against. diff --git a/src/file_analysis/analyzer/extract/Extract.cc b/src/file_analysis/analyzer/extract/Extract.cc index 1a3917cd0e..8b3ed4cdad 100644 --- a/src/file_analysis/analyzer/extract/Extract.cc +++ b/src/file_analysis/analyzer/extract/Extract.cc @@ -12,9 +12,9 @@ using namespace file_analysis; Extract::Extract(RecordVal* args, File* file, const string& arg_filename, uint64 arg_limit) : file_analysis::Analyzer(file_mgr->GetComponentTag("EXTRACT"), args, file), - filename(arg_filename), limit(arg_limit) + filename(arg_filename), limit(arg_limit), depth(0) { - fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_APPEND, 0666); if ( fd < 0 ) { @@ -53,7 +53,7 @@ file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file) limit->AsCount()); } -static bool check_limit_exceeded(uint64 lim, uint64 off, uint64 len, uint64* n) +static bool check_limit_exceeded(uint64 lim, uint64 depth, uint64 len, uint64* n) { if ( lim == 0 ) { @@ -61,29 +61,32 @@ static bool check_limit_exceeded(uint64 lim, uint64 off, uint64 len, uint64* n) return false; } - if ( off >= lim ) + if ( depth >= lim ) { *n = 0; return true; } - - *n = lim - off; - - if ( len > *n ) + else if ( depth + len > lim ) + { + printf("exceeded the maximum extraction lenght depth: %llu len: %llu lim: %llu\n", depth, len, lim); + *n = lim - depth; return true; + } else + { *n = len; + } return false; } -bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset) +bool Extract::DeliverStream(const u_char* data, uint64 len) { if ( ! fd ) return false; uint64 towrite = 0; - bool limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite); + bool limit_exceeded = check_limit_exceeded(limit, depth, len, &towrite); if ( limit_exceeded && file_extraction_limit ) { @@ -92,16 +95,24 @@ bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset) vl->append(f->GetVal()->Ref()); vl->append(Args()->Ref()); vl->append(new Val(limit, TYPE_COUNT)); - vl->append(new Val(offset, TYPE_COUNT)); vl->append(new Val(len, TYPE_COUNT)); f->FileEvent(file_extraction_limit, vl); - // Limit may have been modified by BIF, re-check it. - limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite); + // Limit may have been modified by a BIF, re-check it. + limit_exceeded = check_limit_exceeded(limit, depth, len, &towrite); } if ( towrite > 0 ) - safe_pwrite(fd, data, towrite, offset); + { + safe_pwrite(fd, (const u_char *) data, towrite, depth); + depth += towrite; + } return ( ! limit_exceeded ); } + +bool Extract::Undelivered(uint64 offset, uint64 len) + { + depth += len; + return true; + } diff --git a/src/file_analysis/analyzer/extract/Extract.h b/src/file_analysis/analyzer/extract/Extract.h index 00c4dbe2b7..cb57950d4c 100644 --- a/src/file_analysis/analyzer/extract/Extract.h +++ b/src/file_analysis/analyzer/extract/Extract.h @@ -28,11 +28,18 @@ public: * Write a chunk of file data to the local extraction file. * @param data pointer to a chunk of file data. * @param len number of bytes in the data chunk. - * @param offset number of bytes from start of file at which chunk starts. * @return false if there was no extraction file open and the data couldn't * be written, else true. */ - virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset); + virtual bool DeliverStream(const u_char* data, uint64 len); + + /** + * Report undelivered bytes. + * @param offset distance into the file where the gap occurred. + * @param len number of bytes undelivered. + * @return true + */ + virtual bool Undelivered(uint64 offset, uint64 len); /** * Create a new instance of an Extract analyzer. @@ -67,6 +74,7 @@ private: string filename; int fd; uint64 limit; + uint64 depth; }; } // namespace file_analysis diff --git a/src/file_analysis/analyzer/extract/events.bif b/src/file_analysis/analyzer/extract/events.bif index 1c08736416..f5ebb6816b 100644 --- a/src/file_analysis/analyzer/extract/events.bif +++ b/src/file_analysis/analyzer/extract/events.bif @@ -11,9 +11,7 @@ ## ## limit: The limit, in bytes, the extracted file is about to breach. ## -## offset: The offset at which a file chunk is about to be written. -## ## len: The length of the file chunk about to be written. ## ## .. bro:see:: Files::add_analyzer Files::ANALYZER_EXTRACT -event file_extraction_limit%(f: fa_file, args: any, limit: count, offset: count, len: count%); +event file_extraction_limit%(f: fa_file, args: any, limit: count, len: count%); diff --git a/src/file_analysis/file_analysis.bif b/src/file_analysis/file_analysis.bif index 43efd8ba0f..4e4b4c6cdb 100644 --- a/src/file_analysis/file_analysis.bif +++ b/src/file_analysis/file_analysis.bif @@ -15,6 +15,27 @@ function Files::__set_timeout_interval%(file_id: string, t: interval%): bool return new Val(result, TYPE_BOOL); %} +## :bro:see:`Files::enable_reassembly`. +function Files::__enable_reassembly%(file_id: string%): bool + %{ + bool result = file_mgr->EnableReassembly(file_id->CheckString()); + return new Val(result, TYPE_BOOL); + %} + +## :bro:see:`Files::disable_reassembly`. +function Files::__disable_reassembly%(file_id: string%): bool + %{ + bool result = file_mgr->DisableReassembly(file_id->CheckString()); + return new Val(result, TYPE_BOOL); + %} + +## :bro:see:`Files::set_reassembly_buffer`. +function Files::__set_reassembly_buffer%(file_id: string, max: count%): bool + %{ + bool result = file_mgr->SetReassemblyBuffer(file_id->CheckString(), max); + return new Val(result, TYPE_BOOL); + %} + ## :bro:see:`Files::add_analyzer`. function Files::__add_analyzer%(file_id: string, tag: Files::Tag, args: any%): bool %{ @@ -26,16 +47,6 @@ function Files::__add_analyzer%(file_id: string, tag: Files::Tag, args: any%): b return new Val(result, TYPE_BOOL); %} -## :bro:see:`Files::add_analyzers_for_mime_type`. -function Files::__add_analyzers_for_mime_type%(file_id: string, mtype: string, args: any%): files_tag_set - %{ - using BifType::Record::Files::AnalyzerArgs; - RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); - Val* analyzers = file_mgr->AddAnalyzersForMIMEType(file_id->CheckString(), mtype->CheckString(), rv); - Unref(rv); - return analyzers; - %} - ## :bro:see:`Files::remove_analyzer`. function Files::__remove_analyzer%(file_id: string, tag: Files::Tag, args: any%): bool %{ @@ -60,13 +71,6 @@ function Files::__analyzer_name%(tag: Files::Tag%) : string return new StringVal(file_mgr->GetComponentName(tag)); %} -## :bro:see:`Files::register_for_mime_type`. -function Files::__register_for_mime_type%(id: Analyzer::Tag, mt: string%) : bool - %{ - bool result = file_mgr->RegisterAnalyzerForMIMEType(id->AsEnumVal(), mt); - return new Val(result, TYPE_BOOL); - %} - module GLOBAL; ## For use within a :bro:see:`get_file_handle` handler to set a unique diff --git a/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1 b/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1 index 5c7da193c6..a3bec06fc1 100644 --- a/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1 +++ b/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1 @@ -8,7 +8,7 @@ Extracting file HTTP-FiIpIB2hRQSDBOSJRg.html Extracting file HTTP-FMG4bMmVV64eOsCb.txt Extracting file HTTP-FnaT2a3UDd093opCB9.txt + Extracting file HTTP-FfQGqj4Fhh3pH7nVQj.txt Extracting file HTTP-FsvATF146kf1Emc21j.txt - Extracting file HTTP-FkMQHg2nBr44fc5h63.txt [...] diff --git a/testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1 b/testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1 index 3d6b9dffad..e62ab5a373 100644 --- a/testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1 +++ b/testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1 @@ -16,15 +16,15 @@ #empty_field (empty) #unset_field - #path mime_metrics - #open 2014-04-21-21-34-08 + #open 2014-10-08-03-56-52 #fields ts ts_delta mtype uniq_hosts hits bytes #types time interval string count count count - 1389719059.311698 300.000000 text/html 1 3 47335 + 1389719059.311698 300.000000 text/html 1 7 68469 1389719059.311698 300.000000 image/jpeg 1 1 186859 1389719059.311698 300.000000 application/pgp-signature 1 1 836 - 1389719059.311698 300.000000 text/plain 1 13 119717 + 1389719059.311698 300.000000 text/plain 1 10 101763 1389719059.311698 300.000000 image/gif 1 1 172 1389719059.311698 300.000000 image/png 1 9 82176 1389719059.311698 300.000000 image/x-icon 1 2 2300 - #close 2014-04-21-21-34-08 + #close 2014-10-08-03-56-52 diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit/1.out b/testing/btest/Baseline/scripts.base.files.extract.limit/1.out index f767bfcccd..6cb00f1954 100644 --- a/testing/btest/Baseline/scripts.base.files.extract.limit/1.out +++ b/testing/btest/Baseline/scripts.base.files.extract.limit/1.out @@ -1 +1 @@ -file_extraction_limit, 3000, 2896, 1448 +file_extraction_limit, 3000, 1448 diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit/2.out b/testing/btest/Baseline/scripts.base.files.extract.limit/2.out index bdf1f9d171..ecfd786e38 100644 --- a/testing/btest/Baseline/scripts.base.files.extract.limit/2.out +++ b/testing/btest/Baseline/scripts.base.files.extract.limit/2.out @@ -1,3 +1,3 @@ -file_extraction_limit, 3000, 2896, 1448 +file_extraction_limit, 3000, 1448 T -file_extraction_limit, 6000, 5792, 1448 +file_extraction_limit, 6000, 1448 diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit/3.out b/testing/btest/Baseline/scripts.base.files.extract.limit/3.out index b6da9537b7..7f9b69f0ce 100644 --- a/testing/btest/Baseline/scripts.base.files.extract.limit/3.out +++ b/testing/btest/Baseline/scripts.base.files.extract.limit/3.out @@ -1,2 +1,2 @@ -file_extraction_limit, 7000, 5792, 1448 +file_extraction_limit, 7000, 1448 T diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out index d1cc77944c..5e70c0645c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out @@ -1,9 +1,5 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -^J0.26 | 201 -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION file_stream, file #0, 1146, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J file_chunk, file #0, 1146, 0, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J @@ -16,6 +12,10 @@ file_chunk, file #0, 663, 4042, thread library when necessary (e.g.^J PF_RIN FILE_STATE_REMOVE file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] +FILE_BOF_BUFFER +^J0.26 | 201 +MIME_TYPE +text/plain total bytes: 4705 source: HTTP MD5: 397168fd09991a0e712254df7bc639ac diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.register_mime_type/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.register_mime_type/files.log index b836d14e47..dcb1c18c97 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.register_mime_type/files.log +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.register_mime_type/files.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path files -#open 2014-09-08-21-50-32 +#open 2014-10-08-03-58-17 #fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted #types time string set[addr] set[addr] set[string] string count set[string] string string interval bool bool count count count count bool string string string string string -1362692527.009512 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 MD5 text/plain - 0.000263 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac - - - -#close 2014-09-08-21-50-32 +1362692527.009765 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 MD5 text/plain - 0.000010 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac - - - +#close 2014-10-08-03-58-17 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out index eb62690f91..4b2bf1e210 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out @@ -1,12 +1,12 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -^J0.26 | 201 -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] +FILE_BOF_BUFFER +^J0.26 | 201 +MIME_TYPE +text/plain total bytes: 4705 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout index 1e68c761de..e1e0eb2da4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout @@ -1,20 +1,25 @@ FILE_NEW file #0, 0, 0 -MIME_TYPE -application/x-dosexec FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] +FILE_BOF_BUFFER +MZ\x90\0^C\0\0\0^D\0\0 +MIME_TYPE +application/x-dosexec total bytes: 1022920 source: HTTP +MD5: fc13fee1d44ef737a3133f1298b21d28 +SHA1: 7d99803eaf3b6e8dfa3581348bc694089579d25a +SHA256: dcb87a62a2b5d449abc138776000fd1b14edc690e9da6ea325b8f352ab033202 FILE_NEW file #1, 0, 0 FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_TIMEOUT FILE_STATE_REMOVE -file #1, 206024, 0 +file #1, 0, 0 [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] total bytes: 1022920 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out index 13cfe5de58..72bc73e726 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out @@ -1,7 +1,2 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -^J0.26 | 201 -MIME_TYPE -text/plain -FILE_OVER_NEW_CONNECTION diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out index ef818618b3..a7ba6981c7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out @@ -1,13 +1,13 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -The Nationa -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 16557, 0 [orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp] +FILE_BOF_BUFFER +The Nationa +MIME_TYPE +text/plain source: FTP_DATA MD5: 7192a8075196267203adb3dfaa5c908d SHA1: 44586aed07cfe19cad25076af98f535585cd5797 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out index d42db4b90a..0ed8262afc 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out @@ -1,13 +1,13 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -{^J "origin -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 197, 0 [orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp] +FILE_BOF_BUFFER +{^J "origin +MIME_TYPE +text/plain source: HTTP MD5: 5baba7eea57bc8a42a92c817ed566d72 SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out index 219aad4eff..cc04790c70 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out @@ -1,13 +1,13 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -^J0.26 | 201 -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] +FILE_BOF_BUFFER +^J0.26 | 201 +MIME_TYPE +text/plain total bytes: 4705 source: HTTP MD5: 397168fd09991a0e712254df7bc639ac diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out index 0bf8d6a0c9..6499401f8d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out @@ -1,49 +1,49 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -test FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 4, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +FILE_BOF_BUFFER +test^M^J source: HTTP MD5: 098f6bcd4621d373cade4e832627b4f6 SHA1: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3 SHA256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 FILE_NEW file #1, 0, 0 -FILE_BOF_BUFFER -test2 FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 5, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +FILE_BOF_BUFFER +test2^M^J source: HTTP MD5: ad0234829205b9033196ba818f7a872b SHA1: 109f4b3c50d7b0df729d299bc6f8e9ef9066971f SHA256: 60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752 FILE_NEW file #2, 0, 0 -FILE_BOF_BUFFER -test3 FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #2, 5, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +FILE_BOF_BUFFER +test3^M^J source: HTTP MD5: 8ad8757baa8564dc136c1e07507f4a98 SHA1: 3ebfa301dc59196f18593c45e519287a23297589 SHA256: fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13 FILE_NEW file #3, 0, 0 -FILE_BOF_BUFFER -{^J "data": -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #3, 465, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +FILE_BOF_BUFFER +{^J "data": +MIME_TYPE +text/plain total bytes: 465 source: HTTP MD5: 226244811006caf4ac904344841168dd diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out index 077fb5282c..5f2e28889e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out @@ -1,12 +1,17 @@ FILE_NEW file #0, 0, 0 -MIME_TYPE -application/pdf FILE_OVER_NEW_CONNECTION FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 555523, 0 [orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp] [orig_h=10.101.84.70, orig_p=10977/tcp, resp_h=129.174.93.161, resp_p=80/tcp] +FILE_BOF_BUFFER +%PDF-1.4^J%\xd0 +MIME_TYPE +application/pdf total bytes: 555523 source: HTTP +MD5: 5a484ada9c816c0e8b6d2d3978e3f503 +SHA1: 54e7d39e99eb9d40d6251c0361a1090a0d278571 +SHA256: 61c0718bd534ab55716eba161e91bb49155562ddc7c08f0c20f6359d7b808b66 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out index f5698aba23..2b3d76e59d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out @@ -1,19 +1,24 @@ FILE_NEW file #0, 0, 0 -MIME_TYPE -application/x-dosexec FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] +FILE_BOF_BUFFER +MZ\x90\0^C\0\0\0^D\0\0 +MIME_TYPE +application/x-dosexec total bytes: 1022920 source: HTTP +MD5: fc13fee1d44ef737a3133f1298b21d28 +SHA1: 7d99803eaf3b6e8dfa3581348bc694089579d25a +SHA256: dcb87a62a2b5d449abc138776000fd1b14edc690e9da6ea325b8f352ab033202 FILE_NEW file #1, 0, 0 FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_STATE_REMOVE -file #1, 206024, 0 +file #1, 0, 0 [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] total bytes: 1022920 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out index 5b213f429a..34cffd7f1e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out @@ -3,8 +3,15 @@ file #0, 0, 0 FILE_OVER_NEW_CONNECTION FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE -file #0, 498702, 0 +file #0, 498668, 0 [orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp] [orig_h=10.45.179.94, orig_p=19953/tcp, resp_h=129.174.93.170, resp_p=80/tcp] +FILE_BOF_BUFFER +%PDF-1.4^M%\xe2 +MIME_TYPE +application/pdf total bytes: 498668 source: HTTP +MD5: 94046a5fb1c5802d0f1e6d704cf3e10e +SHA1: 250aa71dd1594363bc7083d25cfd0240e441b119 +SHA256: 5c3bc213c9eff85f98feceac8810b955f8415564e50e3889b447e847c50c5ba7 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out index b85485cd1a..e0880d128c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out @@ -1,41 +1,41 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -/*^J******** -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 2675, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +FILE_BOF_BUFFER +/*^J******** +MIME_TYPE +text/plain source: HTTP MD5: b932c3310ce47e158d1a5a42e0b01279 SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706 SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42 FILE_NEW file #1, 0, 0 -FILE_BOF_BUFFER -//-- Google -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 21421, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +FILE_BOF_BUFFER +//-- Google +MIME_TYPE +text/plain source: HTTP MD5: e732f7bf1d7cb4eedcb1661697d7bc8c SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983 SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da FILE_NEW file #2, 0, 0 -FILE_BOF_BUFFER -GIF89a^D\0^D\0\xb3 -MIME_TYPE -image/gif FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #2, 94, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +FILE_BOF_BUFFER +GIF89a^D\0^D\0\xb3 +MIME_TYPE +image/gif total bytes: 94 source: HTTP MD5: d903de7e30db1691d3130ba5eae6b9a7 @@ -43,14 +43,14 @@ SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315 FILE_NEW file #3, 0, 0 -FILE_BOF_BUFFER -\x89PNG^M^J^Z^J\0\0\0 -MIME_TYPE -image/png FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #3, 2349, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +FILE_BOF_BUFFER +\x89PNG^M^J^Z^J\0\0\0 +MIME_TYPE +image/png total bytes: 2349 source: HTTP MD5: e0029eea80812e9a8e57b8d05d52938a @@ -58,14 +58,14 @@ SHA1: 560eab5a0177246827a94042dd103916d8765ac7 SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4 FILE_NEW file #4, 0, 0 -FILE_BOF_BUFFER -\x89PNG^M^J^Z^J\0\0\0 -MIME_TYPE -image/png FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #4, 27579, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +FILE_BOF_BUFFER +\x89PNG^M^J^Z^J\0\0\0 +MIME_TYPE +image/png total bytes: 27579 source: HTTP MD5: 30aa926344f58019d047e85ba049ca1e diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out index cedc396254..deddfbb640 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out @@ -1,13 +1,13 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -hello world -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 11, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] +FILE_BOF_BUFFER +hello world +MIME_TYPE +text/plain total bytes: 11 source: HTTP MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3 @@ -15,14 +15,14 @@ SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 FILE_NEW file #1, 0, 0 -FILE_BOF_BUFFER -{^J "origin -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 366, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] +FILE_BOF_BUFFER +{^J "origin +MIME_TYPE +text/plain total bytes: 366 source: HTTP MD5: c9337794df612aeaa901dcf9fa446bca diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout index afeb32b334..a5093d22c2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout @@ -1,11 +1,11 @@ FILE_NEW file #0, 0, 0 +FILE_STATE_REMOVE +file #0, 311, 0 FILE_BOF_BUFFER #separator MIME_TYPE text/plain -FILE_STATE_REMOVE -file #0, 311, 0 source: ../input.log MD5: bf4dfa6169b74146da5236e918743599 SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out index 082fb7e038..906225c051 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out @@ -1,18 +1,14 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -PK^C^D^T\0\0\0^H\0\xae -MIME_TYPE -application/zip FILE_OVER_NEW_CONNECTION FILE_NEW file #1, 0, 0 -FILE_BOF_BUFFER -\0\0^Ex\0\0^J\xf0\0\0^P FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 124, 0 [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] +FILE_BOF_BUFFER +\0\0^Ex\0\0^J\xf0\0\0^P source: IRC_DATA MD5: 35288fd50a74c7d675909ff83424d7a1 SHA1: 8a98f177cb47e6bf771bf57c2f7e94c4b5e79ffa @@ -20,6 +16,10 @@ SHA256: b24dde52b933a0d76e885ab418cb6d697b14a4e2fef45fce66e12ecc5a6a81aa FILE_STATE_REMOVE file #0, 42208, 0 [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] +FILE_BOF_BUFFER +PK^C^D^T\0\0\0^H\0\xae +MIME_TYPE +application/zip source: IRC_DATA MD5: 8c0803242f549c2780cb88b9a9215c65 SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log index daf862e3b9..7edaa67263 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path files -#open 2014-09-08-21-55-01 +#open 2014-10-08-03-59-03 #fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted #types time string set[addr] set[addr] set[string] string count set[string] string string interval bool bool count count count count bool string string string string string -1362692527.009512 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000263 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 FakNcS1Jfe01uljb3-file -#close 2014-09-08-21-55-01 +1362692527.009765 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000010 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 FakNcS1Jfe01uljb3-file +#close 2014-10-08-03-59-03 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out index 44c240c7ee..561f3c49f6 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out @@ -1,41 +1,41 @@ FILE_NEW file #0, 0, 0 -FILE_BOF_BUFFER -Hello^M^J^M^J ^M -MIME_TYPE -text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 77, 0 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] +FILE_BOF_BUFFER +Hello^M^J^M^J ^M +MIME_TYPE +text/plain source: SMTP MD5: 58aff3af22807bc5f4b6357c0038256c SHA1: c39dc8cd0f8d8b1f7fc8b362c41e69fdf20f668a SHA256: 8d057f3af311c20675eea767a9df5fa31ff3597c6d5d50fd0cdc34766c40204d FILE_NEW file #1, 0, 0 -FILE_BOF_BUFFER -, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] [2] is_orig: bool = F +1254722770.692743 file_new + [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns=, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=, ftp=, http=, irc=, u2_events=] + +1254722770.692743 file_over_new_connection + [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^J}, rx_hosts={^J^J}, conn_uids={^J^J}, source=SMTP, depth=0, analyzers={^J^J}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] + [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] + [2] is_orig: bool = F + 1254722770.692743 mime_end_entity - [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] + [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=], fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] 1254722770.692743 get_file_handle [0] tag: enum = Analyzer::ANALYZER_SMTP - [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] + [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] [2] is_orig: bool = T -1254722770.692743 file_new - [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=, u2_events=] - -1254722770.692743 file_over_new_connection - [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^J}, rx_hosts={^J^J}, conn_uids={^J^J}, source=SMTP, depth=0, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=] - [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] - [2] is_orig: bool = F +1254722770.692743 file_mime_type + [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] + [1] mime_type: string = text/plain 1254722770.692743 file_state_remove - [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=] + [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] 1254722770.692743 get_file_handle [0] tag: enum = Analyzer::ANALYZER_SMTP @@ -336,14 +340,18 @@ [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=], fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=4], socks=, ssh=, syslog=] [2] is_orig: bool = F -1254722770.692786 file_new - [0] f: fa_file = [id=Ft4M3f2yMvLlmwtbq9, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=], fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=4], socks=, ssh=, syslog=]^J}, last_active=1254722770.692786, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=^M^J^M^J^M^J^M^J^M^J