From 17d74c23dbb60bcf2917d3f1b8ce2d8dc46adbd2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 29 Jul 2011 14:55:53 -0500 Subject: [PATCH 01/10] SMTP script refactor. (addresses #509) - message header state tracking is now done by handling mime_one_header instead of parsing the data in the smtp_data event - changed the logging point to be when an smtp_reply is seen in response to the end of a DATA section - the smtp package now uses it's own mime script and logging stream for logging entities, extraction, etc. - fixes for mime file extraction: now logs the extracted file name, and the count of extracted files needed to be maintained in the State record --- policy/protocols/mime/file-extract.bro | 12 +- policy/protocols/smtp/__load__.bro | 4 +- policy/protocols/smtp/base/__load__.bro | 5 +- policy/protocols/smtp/base/main.bro | 195 +++++-------- policy/protocols/smtp/base/mime.bro | 224 ++++++++++++++ policy/protocols/smtp/base/software.bro | 6 +- .../policy.protocols.smtp.basic/smtp.log | 2 + ...item_10.10.1.4:1470-74.53.140.153:25_1.dat | 14 + ...item_10.10.1.4:1470-74.53.140.153:25_2.dat | 275 ++++++++++++++++++ .../smtp_mime.log | 4 + .../policy.protocols.smtp.mime/smtp_mime.log | 4 + testing/btest/Traces/smtp.trace | Bin 0 -> 27850 bytes .../btest/policy/protocols/smtp/basic.test | 4 + .../policy/protocols/smtp/mime-extract.test | 26 ++ testing/btest/policy/protocols/smtp/mime.test | 24 ++ 15 files changed, 657 insertions(+), 142 deletions(-) create mode 100644 policy/protocols/smtp/base/mime.bro create mode 100644 testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log create mode 100644 testing/btest/Traces/smtp.trace create mode 100644 testing/btest/policy/protocols/smtp/basic.test create mode 100644 testing/btest/policy/protocols/smtp/mime-extract.test create mode 100644 testing/btest/policy/protocols/smtp/mime.test diff --git a/policy/protocols/mime/file-extract.bro b/policy/protocols/mime/file-extract.bro index d6989ad809..81f31d6cd6 100644 --- a/policy/protocols/mime/file-extract.bro +++ b/policy/protocols/mime/file-extract.bro @@ -16,11 +16,13 @@ export { extract_file: bool &default=F; ## Store the file handle here for the file currently being extracted. - extraction_file: file &optional; - + extraction_file: file &log &optional; + }; + + redef record State += { ## Store a count of the number of files that have been transferred in ## this conversation to create unique file names on disk. - num_extracted_files: count &optional; + num_extracted_files: count &default=0; }; } @@ -34,7 +36,7 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=3 { if ( c$mime$extract_file && c$mime$content_len == 0 ) { - local suffix = fmt("%d.dat", ++c$mime$num_extracted_files); + local suffix = fmt("%d.dat", ++c$mime_state$num_extracted_files); local fname = generate_extraction_filename(extraction_prefix, c, suffix); c$mime$extraction_file = open(fname); enable_raw_output(c$mime$extraction_file); @@ -57,4 +59,4 @@ event mime_end_entity(c: connection) &priority=-3 if ( c$mime?$extraction_file ) close(c$mime$extraction_file); } - \ No newline at end of file + diff --git a/policy/protocols/smtp/__load__.bro b/policy/protocols/smtp/__load__.bro index 08161451ac..fd6a7eaeee 100644 --- a/policy/protocols/smtp/__load__.bro +++ b/policy/protocols/smtp/__load__.bro @@ -1,4 +1,4 @@ -@load protocols/smtp/base +@load ./base # This should be optional -@load protocols/smtp/detect-suspicious-orig +@load ./detect-suspicious-orig diff --git a/policy/protocols/smtp/base/__load__.bro b/policy/protocols/smtp/base/__load__.bro index 826efdef0d..efa30304e1 100644 --- a/policy/protocols/smtp/base/__load__.bro +++ b/policy/protocols/smtp/base/__load__.bro @@ -1,2 +1,3 @@ -@load protocols/smtp/base/main -@load protocols/smtp/base/software \ No newline at end of file +@load ./main +@load ./software +@load ./mime diff --git a/policy/protocols/smtp/base/main.bro b/policy/protocols/smtp/base/main.bro index 0f7a674481..53f93b4bcb 100644 --- a/policy/protocols/smtp/base/main.bro +++ b/policy/protocols/smtp/base/main.bro @@ -37,15 +37,10 @@ export { path: vector of addr &log &optional; user_agent: string &log &optional; - ## Indicate if this session is currently transmitting SMTP message - ## envelope headers. - in_headers: bool &default=F; ## Indicate if the "Received: from" headers should still be processed. process_received_from: bool &default=T; - ## Maintain the current header for cases where there is header wrapping. - current_header: string &default=""; - ## Indicate when the message is logged and no longer applicable. - done: bool &default=F; + ## Indicates if client activity has been seen, but not yet logged + has_client_activity: bool &default=F; }; type State: record { @@ -139,26 +134,23 @@ function set_smtp_session(c: connection) if ( ! c?$smtp_state ) c$smtp_state = []; - if ( ! c?$smtp || c$smtp$done ) - { + if ( ! c?$smtp ) c$smtp = new_smtp_log(c); - } } - function smtp_message(c: connection) { - Log::write(SMTP, c$smtp); - - c$smtp$done = T; - # Track the number of messages seen in this session. - ++c$smtp_state$messages_transferred; + if ( c$smtp$has_client_activity ) + Log::write(SMTP, c$smtp); } event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5 { set_smtp_session(c); local upper_command = to_upper(command); + + if ( upper_command != "QUIT" ) + c$smtp$has_client_activity = T; if ( upper_command == "HELO" || upper_command == "EHLO" ) { @@ -175,26 +167,11 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) & else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg ) { - # In case this is not the first message in a session we want to - # essentially write out a log, clear the session tracking, and begin - # new session tracking. - if ( c$smtp_state$messages_transferred > 0 ) - { - smtp_message(c); - set_smtp_session(c); - } - local partially_done = split1(arg, /:[[:blank:]]*/)[2]; c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1]; } - - else if ( upper_command == "DATA" ) - { - c$smtp$in_headers = T; - } } - event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, msg: string, cont_resp: bool) &priority=5 { @@ -202,10 +179,10 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, # This continually overwrites, but we want the last reply, # so this actually works fine. + c$smtp$last_reply = fmt("%d %s", code, msg); + if ( code != 421 && code >= 400 ) { - c$smtp$last_reply = fmt("%d %s", code, msg); - # Raise a notice when an SMTP error about a block list is discovered. if ( bl_error_messages in msg ) { @@ -226,145 +203,103 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, } } -event smtp_data(c: connection, is_orig: bool, data: string) &priority=5 +event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, + msg: string, cont_resp: bool) &priority=-5 { - # Is there something we should be handling from the server? - if ( ! is_orig ) return; - set_smtp_session(c); - - if ( ! c$smtp$in_headers ) + if ( cmd == "." ) { - if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data ) - { - if ( ! c$smtp?$files ) - c$smtp$files = set(); - data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, ""); - add c$smtp$files[data]; - } - return; + # Track the number of messages seen in this session. + ++c$smtp_state$messages_transferred; + smtp_message(c); + c$smtp = new_smtp_log(c); + } + } + +event mime_one_header(c: connection, h: mime_header_rec) &priority=5 + { + if ( ! c?$smtp ) return; + c$smtp$has_client_activity = T; + + if ( h$name == "CONTENT-DISPOSITION" && + /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) + { + if ( ! c$smtp?$files ) c$smtp$files = set(); + add c$smtp$files[sub(h$value, + /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "")]; } - if ( /^[[:blank:]]*$/ in data ) - c$smtp$in_headers = F; + else if ( h$name == "MESSAGE-ID" ) + c$smtp$msg_id = h$value; - # This is to reconstruct headers that tend to wrap around. - if ( /^[[:blank:]]/ in data ) - { - # Remove all but a single space at the beginning (this seems to follow - # the most common behavior). - data = sub(data, /^[[:blank:]]*/, " "); - if ( c$smtp$current_header == "MESSAGE-ID" ) - c$smtp$msg_id += data; - else if ( c$smtp$current_header == "RECEIVED" ) - c$smtp$first_received += data; - else if ( c$smtp$current_header == "IN-REPLY-TO" ) - c$smtp$in_reply_to += data; - else if ( c$smtp$current_header == "SUBJECCT" ) - c$smtp$subject += data; - else if ( c$smtp$current_header == "FROM" ) - c$smtp$from += data; - else if ( c$smtp$current_header == "REPLY-TO" ) - c$smtp$reply_to += data; - else if ( c$smtp$current_header == "USER-AGENT" ) - c$smtp$user_agent += data; - return; - } - # Once there isn't a line starting with a blank, we're not continuing a - # header anymore. - c$smtp$current_header = ""; - - local header_parts = split1(data, /:[[:blank:]]*/); - # TODO: do something in this case? This would definitely be odd. - # Header wrapping needs to be handled more elegantly. This will happen - # if the header value is wrapped immediately after the header key. - if ( |header_parts| != 2 ) - return; - - local header_key = to_upper(header_parts[1]); - c$smtp$current_header = header_key; - - local header_val = header_parts[2]; - - if ( header_key == "MESSAGE-ID" ) - c$smtp$msg_id = header_val; - - else if ( header_key == "RECEIVED" ) + else if ( h$name == "RECEIVED" ) { if ( c$smtp?$first_received ) c$smtp$second_received = c$smtp$first_received; - c$smtp$first_received = header_val; + c$smtp$first_received = h$value; } - - else if ( header_key == "IN-REPLY-TO" ) - c$smtp$in_reply_to = header_val; - - else if ( header_key == "DATE" ) - c$smtp$date = header_val; - - else if ( header_key == "FROM" ) - c$smtp$from = header_val; - - else if ( header_key == "TO" ) + + else if ( h$name == "IN-REPLY-TO" ) + c$smtp$in_reply_to = h$value; + + else if ( h$name == "SUBJECT" ) + c$smtp$subject = h$value; + + else if ( h$name == "FROM" ) + c$smtp$from = h$value; + + else if ( h$name == "REPLY-TO" ) + c$smtp$reply_to = h$value; + + else if ( h$name == "DATE" ) + c$smtp$date = h$value; + + else if ( h$name == "TO" ) { if ( ! c$smtp?$to ) c$smtp$to = set(); - add c$smtp$to[header_val]; + add c$smtp$to[h$value]; } - - else if ( header_key == "REPLY-TO" ) - c$smtp$reply_to = header_val; - - else if ( header_key == "SUBJECT" ) - c$smtp$subject = header_val; - else if ( header_key == "X-ORIGINATING-IP" ) + else if ( h$name == "X-ORIGINATING-IP" ) { - local addresses = find_ip_addresses(header_val); + local addresses = find_ip_addresses(h$name); if ( 1 in addresses ) c$smtp$x_originating_ip = to_addr(addresses[1]); } - else if ( header_key == "X-MAILER" || - header_key == "USER-AGENT" || - header_key == "X-USER-AGENT" ) - { - c$smtp$user_agent = header_val; - # Explicitly set the current header here because there are several - # headers bulked under this same key. - c$smtp$current_header = "USER-AGENT"; - } + else if ( h$name == "X-MAILER" || + h$name == "USER-AGENT" || + h$name == "X-USER-AGENT" ) + c$smtp$user_agent = h$value; } # This event handler builds the "Received From" path by reading the # headers in the mail -event smtp_data(c: connection, is_orig: bool, data: string) &priority=3 +event mime_one_header(c: connection, h: mime_header_rec) &priority=3 { # If we've decided that we're done watching the received headers for # whatever reason, we're done. Could be due to only watching until # local addresses are seen in the received from headers. - if ( c$smtp$current_header != "RECEIVED" || - ! c$smtp$process_received_from ) + if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from) return; - - local text_ip = find_address_in_smtp_header(data); + + local text_ip = find_address_in_smtp_header(h$value); if ( text_ip == "" ) return; local ip = to_addr(text_ip); - + if ( ! addr_matches_host(ip, mail_path_capture) && ! Site::is_private_addr(ip) ) { c$smtp$process_received_from = F; } - if ( c$smtp$path[|c$smtp$path|-1] != ip ) c$smtp$path[|c$smtp$path|] = ip; } - event connection_state_remove(c: connection) &priority=-5 { - if ( c?$smtp && ! c$smtp$done ) + if ( c?$smtp ) smtp_message(c); } diff --git a/policy/protocols/smtp/base/mime.bro b/policy/protocols/smtp/base/mime.bro new file mode 100644 index 0000000000..7ed7c67d51 --- /dev/null +++ b/policy/protocols/smtp/base/mime.bro @@ -0,0 +1,224 @@ +##! Analysis and logging for MIME entities found in SMTP sessions. + +@load utils/strings +@load utils/files +@load ./main + +module SMTP; + +export { + redef enum Notice::Type += { + ## Indicates that an MD5 sum was calculated for a MIME message. + MD5, + }; + + redef enum Log::ID += { SMTP_MIME }; + + type MimeInfo: record { + ## This is the timestamp of when the MIME content transfer began. + ts: time &log; + uid: string &log; + id: conn_id &log; + ## The filename seen in the Content-Disposition header. + filename: string &log &optional; + ## Track how many byte of the MIME encoded file have been seen. + content_len: count &log &default=0; + mime_type: string &log &optional; + ## The calculated MD5 sum for the MIME entity. + md5: string &log &optional; + ## Optionally calculate the file's MD5 sum. Must be set prior to the + ## first data chunk being see in an event. + calc_md5: bool &default=F; + ## This boolean value indicates if an MD5 sum is being calculated + ## for the current file transfer. + calculating_md5: bool &default=F; + ## Optionally write the file to disk. Must be set prior to first + ## data chunk being seen in an event. + extract_file: bool &default=F; + ## Store the file handle here for the file currently being extracted. + extraction_file: file &log &optional; + }; + + type MimeState: record { + ## Store a count of the number of files that have been transferred in + ## this conversation to create unique file names on disk. + num_extracted_files: count &default=0; + ## Track the number of MIME encoded files transferred during this session. + level: count &default=0; + }; + + ## Generate MD5 sums for these filetypes. + const generate_md5 = /application\/x-dosexec/ # Windows and DOS executables + | /application\/x-executable/ # *NIX executable binary + &redef; + + ## Pattern of file mime types to extract from MIME bodies. + const extract_file_types = /NO_DEFAULT/ &redef; + + ## The on-disk prefix for files to be extracted from MIME entity bodies. + const extraction_prefix = "smtp-mime-item" &redef; + + global log_mime: event(rec: MimeInfo); +} + +redef record connection += { + smtp_mime: MimeInfo &optional; + smtp_mime_state: MimeState &optional; +}; + +event bro_init() + { + Log::create_stream(SMTP_MIME, [$columns=MimeInfo, $ev=log_mime]); + } + +function new_mime_session(c: connection): MimeInfo + { + local info: MimeInfo; + + info$ts=network_time(); + info$uid=c$uid; + info$id=c$id; + return info; + } + +function set_session(c: connection, new_entity: bool) + { + if ( ! c?$smtp_mime_state ) + c$smtp_mime_state = []; + + if ( ! c?$smtp_mime || new_entity ) + c$smtp_mime = new_mime_session(c); + } + +event mime_begin_entity(c: connection) &priority=10 + { + if ( ! c?$smtp ) return; + set_session(c, T); + ++c$smtp_mime_state$level; + } + +# This has priority -10 because other handlers need to know the current +# content_len before it's updated by this handler. +event mime_segment_data(c: connection, length: count, data: string) &priority=-10 + { + if ( ! c?$smtp ) return; + c$smtp_mime$content_len = c$smtp_mime$content_len + length; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=7 + { + if ( ! c?$smtp ) return; + if ( c$smtp_mime$content_len == 0 ) + c$smtp_mime$mime_type = split1(identify_data(data, T), /;/)[1]; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=-5 + { + if ( ! c?$smtp_mime ) return; + + if ( c$smtp_mime$content_len == 0 ) + { + if ( generate_md5 in c$smtp_mime$mime_type ) + c$smtp_mime$calc_md5 = T; + + if ( c$smtp_mime$calc_md5 ) + { + c$smtp_mime$calculating_md5 = T; + md5_hash_init(c$id); + } + } + + if ( c$smtp_mime$calculating_md5 ) + md5_hash_update(c$id, data); +} + +## In the event of a content gap during the MIME transfer, detect the state for +## the MD5 sum calculation and stop calculating the MD5 since it would be +## incorrect anyway. +event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5 + { + if ( is_orig || ! c?$smtp_mime ) return; + + if ( c$smtp_mime$calculating_md5 ) + { + c$smtp_mime$calculating_md5 = F; + md5_hash_finish(c$id); + } + } + +event mime_end_entity(c: connection) &priority=-3 + { + # TODO: this check is only due to a bug in mime_end_entity that + # causes the event to be generated twice for the same real event. + if ( ! c?$smtp_mime ) + return; + + if ( c$smtp_mime$calculating_md5 ) + { + c$smtp_mime$md5 = md5_hash_finish(c$id); + + NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h), + $sub=c$smtp_mime$md5, $conn=c]); + } + } + +event mime_one_header(c: connection, h: mime_header_rec) + { + if ( ! c?$smtp ) return; + if ( h$name == "CONTENT-DISPOSITION" && + /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) + c$smtp_mime$filename = sub(h$value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, ""); + } + +event mime_end_entity(c: connection) &priority=-5 + { + if ( ! c?$smtp ) return; + # This check and the delete below are just to cope with a bug where + # mime_end_entity can be generated multiple times for the same event. + if ( ! c?$smtp_mime ) + return; + + # Don't log anything if there wasn't any content. + if ( c$smtp_mime$content_len > 0 ) + Log::write(SMTP_MIME, c$smtp_mime); + + delete c$smtp_mime; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=5 + { + if ( ! c?$smtp ) return; + if ( extract_file_types in c$smtp_mime$mime_type ) + c$smtp_mime$extract_file = T; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=3 + { + if ( ! c?$smtp ) return; + if ( c$smtp_mime$extract_file && c$smtp_mime$content_len == 0 ) + { + local suffix = fmt("%d.dat", ++c$smtp_mime_state$num_extracted_files); + local fname = generate_extraction_filename(extraction_prefix, c, suffix); + c$smtp_mime$extraction_file = open(fname); + enable_raw_output(c$smtp_mime$extraction_file); + } + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=-5 + { + if ( ! c?$smtp ) return; + if ( c$smtp_mime$extract_file && c$smtp_mime?$extraction_file ) + print c$smtp_mime$extraction_file, data; + } + +event mime_end_entity(c: connection) &priority=-3 + { + if ( ! c?$smtp ) return; + # TODO: this check is only due to a bug in mime_end_entity that + # causes the event to be generated twice for the same real event. + if ( ! c?$smtp_mime ) + return; + + if ( c$smtp_mime?$extraction_file ) + close(c$smtp_mime$extraction_file); + } diff --git a/policy/protocols/smtp/base/software.bro b/policy/protocols/smtp/base/software.bro index 1773a87201..32261bc264 100644 --- a/policy/protocols/smtp/base/software.bro +++ b/policy/protocols/smtp/base/software.bro @@ -45,10 +45,10 @@ export { | /ZimbraWebClient/ &redef; } -event smtp_data(c: connection, is_orig: bool, data: string) &priority=4 +event mime_one_header(c: connection, h: mime_header_rec) &priority=4 { - if ( c$smtp$current_header == "USER-AGENT" && - webmail_user_agents in c$smtp$user_agent ) + if ( ! c?$smtp ) return; + if ( h$name == "USER-AGENT" && webmail_user_agents in c$smtp$user_agent ) c$smtp$is_webmail = T; } diff --git a/testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log new file mode 100644 index 0000000000..290a3c3a5a --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log @@ -0,0 +1,2 @@ +# ts uid id.orig_h id.orig_p id.resp_h id.resp_p helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply files path user_agent +1254722768.219663 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un "NEWS.txt" 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat new file mode 100644 index 0000000000..e91ec57ed2 --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat @@ -0,0 +1,14 @@ +Hello + + + +I send u smtp pcap file + +Find the attachment + + + +GPS + + + diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat new file mode 100644 index 0000000000..09479e05cb --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat @@ -0,0 +1,275 @@ +Version 4.9.9.1 +* Many bug fixes +* Improved editor + +Version 4.9.9.0 +* Support for latest Mingw compiler system builds +* Bug fixes + +Version 4.9.8.9 +* New code tooltip display +* Improved Indent/Unindent and Remove Comment +* Improved automatic indent +* Added support for the "interface" keyword +* WebUpdate should now report installation problems from PackMan +* New splash screen and association icons +* Improved installer +* Many bug fixes + +Version 4.9.8.7 +* Added support for GCC > 3.2 +* Debug variables are now resent during next debug session +* Watched Variables not in correct context are now kept and updated when it is needed +* Added new compiler/linker options: 20 + - Strip executable + - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20 + k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2) + - Enable use of processor specific built-in function +s (mmmx, sse, sse2, pni, 3dnow) +* "Default" button in Compiler Options is back +* Error messages parsing improved +* Bug fixes + +Version 4.9.8.5 +* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value") +* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features. +* Many bug fixes + +Version 4.9.8.4 +* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup +* Improved code completion cache +* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP +* Big speed up in function parameters listing while editing +* Bug fixes + +Version 4.9.8.3 +* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20 + include files can now be generated. +* Improved WebUpdate module +* Many bug fixes + +Version +4.9.8.2 +* New debug feature for DLLs: attach to a running process +* New project option: Use custom Makefile. 20 +* New WebUpdater module. +* Allow user to specify an alternate configuration file in Environment Options 20 + (still can be overriden by using "-c" command line parameter). +* Lots of bug fixes. + +Version 4.9.8.1 +* When creating a DLL, the created static lib respects now the project-defined output directory + +Version 4.9.8.0 +* Changed position of compiler/linker parameters in Project Options. +* Improved help file +* Bug fixes + +Version 4.9.7.9 +* Resource errors are now reported in the Resource sheet +* Many bug fixes + +Version 4.9.7.8 +* Made whole bottom report control floating instead of only debug output. +* Many bug fixes + +Version 4.9.7.7 +* Printing settings are now saved +* New environment options : "watch variable under mouse" and "Report watch errors" +* Bug fixes + +Version 4.9.7.6 +* Debug variable browser +* Added possibility to include in a Template the Project's directorie +s (include, libs and ressources) +* Changed tint of Class browser pictures colors to match the New Look style +* Bug fixes + +Version 4.9.7.5 +* Bug fixes + +Version 4.9.7.4 +* When compiling with debugging symbols, an extra definition is passed to the + compiler: -D__DEBUG__ +* Each project creates a _private.h file containing version + information definitions +* When compiling the current file only, no dependency checks are performed +* ~300% Speed-up in class parser +* Added "External programs" in Tools/Environment Options (for units "Open with") +* Added "Open with" in project units context menu +* Added "Classes" toolbar +* Fixed pre-compilation dependency checks to work correctly +* Added new file menu entry: Save Project As +* Bug-fix for double quotes in devcpp.cfg file read by vUpdate +* Other bug fixes + +Version 4.9.7.3 +* When adding debugging symbols on request, remove "-s" option from linker +* Compiling progress window +* Environment options : "Show progress window" and "Auto-cl +ose progress window" +* Bug fixes + +Version 4.9.7.2 +* Bug fixes + +Version 4.9.7.1 +* "Build priority" per-unit +* "Include file in linking process" per-unit +* New feature: compile current file only +* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744) +* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744) +* Necessary UI changes in Project Options +* Added display of project filename, project output and a summary of the project files in Project Options General tab. +* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0 +* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings) +* Resource files are treated as ordinary files now +* Updates in "Project Options/Files" code +* MSVC import now creates the folders structure of the original VC project +* Bug fixes + +Version 4.9.7.0 +* Allow customizing of per-unit compile command in projects +* Added two new macros: and < +DATETIME> +* Added support for macros in the "default source code" (Tools/Editor Options/Code) +* Separated layout info from project file. It is now kept in a different file + (the same filename as the project's but with extension ".layout"). If you + have your project under CVS control, you ''ll know why this had to happen... +* Compiler settings per-project +* Compiler set per-project +* Implemented new compiler settings framework +* "Compile as C++" per-unit +* "Include file in compilation process" per-unit +* Project version info (creates the relevant VERSIONINFO struct in the private + resource) +* Support XP Themes (creates the CommonControls 6.0 manifest file and includes + it in the private resource) +* Added CVS "login" and "logout" commands +* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows. +* Added "Add Library" button in Project Options +* Bug fixes + +Version 4.9.6.9 +* Implemented search in help files for the word at cursor (context sensitive + help) +* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2) +* Added "Files" tab in CVS form to allow selection of more than one file for + the requested CVS action + 20 +Version 4.9.6.8 +* support for DLL application hosting, for debugging and executing DLLs under Dev-C++. +* New class browser option: "Show inherited members" +* Added support for the '::' member access operator in code-completion +* Added *working* function arguments hint +* Added bracket highlighting. When the caret is on a bracket, that bracket and + its counterpart are highlighted +* Nested folders in project view + +Version 4.9.6.7 +* XP Theme support +* Added CVS commands "Add" and "Remove" +* Added configuration option for "Templates Directory" in "Environment Options" +* Code-completion updates +* Bug fixes + +Version 4.9.6.6 +* Editor colors are initialized properly on Dev-C++ first-run +* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVari +able wizards +* Added file's date/time stamp in File/Properties window +* Current windows listing in Window menu +* Bug fixes + +Version 4.9.6.5 +* CVS support +* Window list (in Window menu) +* bug fixes + +version 4.9.6.4 +* added ENTER key for opening file in project browser, DEL to delete from the project. +* bug fixes + +version 4.9.6.3 +* Bug fixes + +version 4.9.6.2 +* Bug fixes + +version 4.9.6.1 +* New "Abort compilation" button +* Bug fixes +* Now checks for vRoach existance when sending a crash report + +Version 4.9.5.5 +* New option in Editor Options: Show editor hints. User can disable the hints + displayed in the editor when the mouse moves over a word. Since this was the + cause of many errors (although it should be fixed by now), we are giving the + user the option to disable this feature. +* New option in Editor Options (code-completion): Use code-completion cache. + Well, it adds caching to code-completion. Depending on the cache size, + the program may take a bit longer to start-up, bu +t provides very fast + code-completion and the user has all the commands (belonging to the files + he added in the cache) at his fingertips. If, for example, the user adds + "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of + wxWindows! You get the picture... +* Removed "Only show classes from current file" option in class browser settings. + It used to be a checkbox, allowing only two states (on or off), but there is + a third relevant option now: "Project classes" so it didn't fit the purpose... + The user can define this in the class browser's context menu under "View mode". +* Fixed the dreaded "Clock skew detected" compiler warning! +* Fixed many class browser bugs, including some that had to do with class folders. + +Version 4.9.5.4 +* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data) + +Version 4.9.5.3 +* Added ExceptionsAnalyzer. If the devcpp.map file is in + the devcpp.exe directory + then we even get a stack trace in the bug report! +* Added new WebUpdate module (inactive temporarily). +* Added new code for code-completion caching of files (disabled - work in progress). + +Version 4.9.5.2 +* Added new option in class-browser: Use colors + (available when right-clicking the class-browser + and selecting "View mode"). +* Dev-C++ now traps access violation of your programs (and of itself too ;) + +Version 4.9.5.1 +* Implemented the "File/Export/Project to HTML" function. +* Added "Tip of the day" system. +* When running a source file in explorer, don't spawn new instance. + Instead open the file in an already launched Dev-C++. +* Class-parser speed-up (50% to 85% improvement timed!!!) +* Many code-completion updates. Now takes into account context, + class inheritance and visibility (shows items only from files + #included directly or indirectly)! +* Caching of result set of code-completion for speed-up. +* New option "Execution/Parameters" (and "Debug/Param +eters"). + +Version 4.9.5.0 (5.0 beta 5): +* CPU Window (still in development) +* ToDo list +* Backtrace in debugging +* Run to cursor +* Folders in Project and Class Browser +* Send custom commands to GDB +* Makefile can now be customized. +* Modified the behaviour of the -c param : 20 + -c +* Saving of custom syntax parameter group +* Possibility of changing compilers and tools filename. +* Many bug fixes + + +Version 4.9.4.1 (5.0 beta 4.1): + +* back to gcc 2.95.3 +* Profiling support +* new update/packages checker (vUpdate) +* Lots of bugfixes + + diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log new file mode 100644 index 0000000000..9a3d5ac0f9 --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log @@ -0,0 +1,4 @@ +# ts uid id.orig_h id.orig_p id.resp_h id.resp_p filename content_len mime_type md5 extraction_file +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 79 FAKE_MIME - smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 1918 FAKE_MIME - - +1254722770.692804 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 "NEWS.txt" 10823 FAKE_MIME - smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log b/testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log new file mode 100644 index 0000000000..cd2d543b51 --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log @@ -0,0 +1,4 @@ +# ts uid id.orig_h id.orig_p id.resp_h id.resp_p filename content_len mime_type md5 extraction_file +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 1918 FAKE_MIME - - +1254722770.692804 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 "NEWS.txt" 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - diff --git a/testing/btest/Traces/smtp.trace b/testing/btest/Traces/smtp.trace new file mode 100644 index 0000000000000000000000000000000000000000..931b43b3b878dc559b7423df26363bfe75c6a512 GIT binary patch literal 27850 zcmeHw3y@padEO-{>aq6Pq^EIdM;)Do#Arz^u)w}Zf{zdjU?0>zpj|8}$Fsy#Grk7^ ze09Qq&xUc98NAQ>?m^{uN*%g#?yCn4BRRLQtY%Xh z+{2%^-Z%wFeDzRrc<9@tWa8;gN@grj^0TP}?|l9X5Aw0!`SPLXiQ1D-TyOk)eCDeM z*3&~T3aneVg@sj#yc4T3x$~;bS;e9Qyw!UmV`HJn6kZeKp~&R1dqI3Ww>p1!QcaFc zj2%B74(qQ{^Qq**{L=i=to|HZOV6p)LTr9XEiBK@FL@89V=L+O!m4~ybIApdG#~yK z*BjHA312;wo;ZAqQ2GzLDE-+_zmj=1b?Eiq|K);V$avmJB+$#>88qeqjISO7MTcGz z0)4EDK(B(>e@M2?LX(Y;PpI>EE)}z-$wKDB+LT_zTVEbD(g4X<52iDRj|is!ybIGm z{PZhX1DJj$@EeA4_xSm(?Ci`|Zg!?|$J}%wH+Qa*8DF`)P%7332fhg03*Xq+5BKi~ z?jP;MJ!=5>7m54XNsas3?9Aotcy1$Gx&ycq?kB(52De*HM-PVt_uuKl{YBvZ4&eT= z^wWlsojYGQOEdNziR9K|?^t(Bm!N)M0 z$*@|!T|=9G>}|%qK$))|yq6Cj7rg&rC*J9I1MlO&yYb}}!&r%@(ki_StgGhz7ju?V zgjO80%V)RDg5yAoYSjM(3;gwOUSmMv%BzRzr-oh@)Vr8IdmO08i24g}*Qm!p_zLvn z-~eGh_rf(Tt<}vmp~I(T0#9{KAchIdU;_4;7lBPI9n-Krb7;VTWtUeEe)kQ1SMaNL zk#+|7%@V)U889}?K(bu3s;XqUuDNcFD65>a%j>FP*9s~S3MlZ7sfu>lQkgAf*IcWZ z*P~zeYXio2_Z#x+UW)x+KfKJDeE%=HW-^PJJdir@dgLn~FpQb1Q;Mpgf)3AAHF4Ps!GRkxPQ9Ent6Qf7LHs#=amm`OrEteZjL6r-#c(7Zo0yyq zhr%Z&r%sH|gad;EcLo<>L9HrgVP~t3>*Q-{IiI()mRhdYijH$hMaDROx=^*uTyVae zvo@nDmaWXJTcs1z8~5LTcQRqEL~bt}w{I(~ZJl?*F=`qg-vzV0UdPHuvGT#XJ2u#%`8nv)J*Eqikd0k zc0tX}#8YaneAhX3Zuyw2Zm*Z7)a_ff3u<8|JFgb*U#p#roXoCI7hS$FWhbza|y#V2BSc|YYk=5cb>aSLYT;zs1;v6JtM z$718hW+qPrBgbJ@Cnn>^g2yK(!ogTL)V{?nKh4tx5Y<41cve%UY7vWz-%`T=-t~I8YVl#Kd)?zV` zSE#7zXXm2ojiEL;z;%SHg@0y?hkW5wc3F}0RNkqU%o^OP!2$Bp8|0?ens@W4S%eBG z!%%EkSK-tMIdyVdPTc{Uou25%rUr2strShWOcb&Ov+7#4(}B7hG~KLiLlk_DP1F0# zTh(B)oOP(IqUu=2u5o!-fve{{$&@-@NK8@%CEb8SPHICP;_t3*dfy%~m@Gh~=Es-GK$;hq?cjEy>__8GT5 z-DosIjq#9ET^~)ajIJc(K~ixd9Dr{^l0sqP-KoW7I;Q5*=~OUz$J+dP0%-CYw0S_q zmzUDXr8F-Ho65S9#6FvjCz^apfF-MDs?A9E3e**Gor;<0-ecRkZkKC>R7&j~-Ps0HP9B-Y(np>s)no6c^6s@pUk7X2B~r zywCekvWr{MK-?^4sX%~~NqW~`WH$F+iy=$M*NlWH8V2{nbE_>_B5MqrHFzRR0F6$JIbNWg{i2)b6)&IdSKGYa44(ui_F77jg! z1+Vfe8Qpb)vILYx1jV(j=!p{|&`!~*Ml;2_=u7fMFlS{QtTP<~*m;3r=>axuhUtip zd&Zb?5EwS-8e*pX$Ev_dDqejw86wG`TZkVy)(N^s=L8dUow8Y^2v)!fO#=2O79eG1 z+6|~qr@={cej!pVUZ{X4O*lQorS(+Umr;XHG#!;vRL%iNcMj zAU={wxN0Mc=}}T4jI{t(&d!z0O@0vxk4**Cs6gh^r-t4`mpOkgMx&TsWrU)+<!AX0%4R13}Zc7#6dL4peAqyMjHWVX9m?)!yu#;u%u19$J_2E5VHws95S=%YUyqVPMm13ZfL(6kuk zT6%A3gXHbc;H1!qZ0ptR6o9ljT@u!GI%#6z0!zsYtD)Lvt-puX>(lrqHen-`2n{Kr zRLdI^tpsnaXOO`GFYrQ&^924vg&c)-D{mp?xDE$z(?Sq)RLz$vRc8bCloISzIq}Zt z5HrC<)hm+6ghve-sab9faaMV~0b^aNFz)iRo|(E`%n9J>)+n7IkB3eWmU0VDy_iF)x1p+*%-t@#HM5A>V+9n5 z29LAksyu>ol`^xJKw5uVoQGRbZnkP!NDBaGCql)kIGv>TAX}h*?-F$r52k9n zNzKON>Wmr>jq!5A;u>$5aA<%ff~2bD%>1%_y}h zn8&G*sBZN17`RQTAY7s64E=>h- z7ycio=D*Ni^RI~J|9+?Dx3a$G|JK8XF@jL5P~>ie{u!)g>?*t>@6$x{-DX8*x-$7O}G%k%90UcBkE(#@-~^y%;C`0LIS^H?B$o9$lPxk38Q!0)RxzrV zR&y0{K`wdI_S6z@>7z*|d0k65r5Xy-wKH}R9w_%;N(g)Nc!w(+CapNOXw0Tq_+o&W zh!DVmURT3ad%XZ2$7anXozrVj9$F7knvvY2AOsdQe1RR1*(bDQC9I8L{I=WFj9rDM zOxq=>tL1XuMgmyt0PP!&ko;9wWzDjPXBL@NY$O4pj8$t;chrR#AP{9xd<~VUIt@If z*OU5_D!6EgR?1sutzNZUq&%*!NGAHyPV>PQhQSJEi**c?vnl^hbxTB!R2BUVblXtK zLj0)HRR))lYJ*1))`dpW6v+!kph&B=dW8ry?T^6f2E;pBI=NvNi$W+ePaPJ+X_SkO zN&OD^Df1FCLS=ubd#IG*>bj(76Lm-r7SPA0rC|7A&f3UUDj|FsnvTV9UrTZHX?vYA zfgvzpn>3J95xK&MxNsER8kwh2VA_O+6rbeboo(PauP-;Tg+V5()W|BZ4{fZHv(2Kj z4!kwByD3DT#sp!QgwH`)W>0gTit?9^`J5J?VQJQNxNYTxRho;?qBUfv&Xmg4`A94d zt+_TO9uEqSBo-FjD3+h8Bl3i)s`WB#`8w?<6%OI^QG8@R+A{=EwFWWB)?HY0xGa|t z#uu%SOiAC~L=&1r@RV7^7$u5GajZ55_Vae#q?Ife)-+k(u&Yj)?gVM+X(^9{!&qAo zmof^v!SkqAZTJ>U^K0o01hdrqw?8&uEbTSq)wRp;%-*xfcVRy#s zy!`!NN*a0(;DKv@u;U)UP2btw19&s<0g$^s0PcDLyGVTX>>_>X@T&L~KkD==(#>6@ z)#xq^cNb}!4K9YdySqq|rrq5|`hRN|i5h?QV}1RJcZ)wU(()(vf3E3IU}p*b z#Ev^lU-_loouxPP&JwxncR${jyMA7{>$Vnm-E1|v3)yRM*N)liEfj3+X0LZn%K!hK zy(V`>AMeXuHwt$hZE@EZ&NaCU88UF!jv2C}ukL2Z-pm=YKLvMv|HFG)wSnD*pTk4n zlX|{Sb=C7d@W8Woy_Pxoq@L`{L_{ z(N$HlTdSdVYc*QM86(J=?$&B-Ul+1ltFc?FAr&J3$7(gG@qeMe#$OYS|HDp=fAQnK z#{cQdhS6Q|vFq)=dA;4v`qSN7jon%eX?BpBo8zI~T8;nPwHnm;UpsnTzj)yli5Gse zH(t2-@zjCfeV_Uvs zeeBkK?ACoqE#7Y32P+qvRX)1JX1DGG^(p_G>prOQr*7-3@h^zR@2cIn|0ll2|EnJv z#?`7v0$kh@>yl6oi6S=nZ@X(krMCA1%PytliUl^NLh;Rr z)CBouSf~wRX)HQCP}0is7yxE{D=WEB_TtsxphE*)GpKh${e1-`_|4L(-eILQI9@=_ zJZ|@klZh*tl~uuheP?q@QlORcihaB~8<}OZ0;ymXb^BT0`l(~OhIqwtoq9Eknk!Zj zw%56`Vi!oExQDZA-iK~YOR9pq3!ytHdMtEYS3aTYtx<4L)TMiA{8C_4Xw|AtQRPv6 zCY4_-NJWfQn?x|A7+>0343P?FxK%JNtUjr`p;#eNwguju;AuG&k#@(e3r zs21pVvSc)QaU*oZS~bDym{< zrTP)9xywgTkAqd*AdYQT|&xxVB?RWNb!Vj?-cHhb|RQBAUh&M%YG z5)BfmQ{Gn>S^t0LBKog3K<7|FcW{zV(J4yYFbYc7bdk4$TjUHA@L{#e3d*3apwwlo zEcDd{@0xuA$xUglEpn3UD6Mn@l-@L&1KiQ}0zJQmc$jtHb+q?MfoEVD$jAcdLR=4M z>stcz$)}$paaaM5vOk=F;ju9_{)LD8#tYwR5#{uAs_8OGna^Wawr z%L1%E@xq;AJ-T-4D^G~^IP`iXc)&1JK)BojJ=iIcF{MjjhVAm0kR{z|w%3)=yCmQ& z^fp`qUrF(PQQ9jsaaf2iD@p>OtG!m;LJK>zj4z=oCJX@~2*U74L1)gXvua$rR7971 zHM7|~jT5f0xt^tw8@i2xbX^7u@tm7x(@D3kl~Pz?J2NMTD)fjDE3vEAd+X>s8NnOr z{|E#@5v}`l69za>w-a#CcvC2aouEco5s!jlk)`dm=V}4Qum8V8=M13h2Fd_>6skcoDiIFMdnMR@#VRLCsRlAEqwTz#s?Mvz zjrt0uSB439)uLslp=7nXuY3G<3{BQ}@AnWTzbcpVKI~dBkXT=@QRe2U$%$hV6QZ#@ zNAQgTd&1+FXEw-5vqp#(ZPX~PR208m6 zZAB1msF(y>CDt(l%H*NgE7EiU60>dV7>}$#2{w<(KsT!6!YMiqW27 z78RatEdxjHBUeH?T@BduK+7M_cp$59eqfjpO< z&dM;+aUnm#3=||Zx(SXRK=^p*0@8jFOs!R>|AFIjGy;Jm7)!br(CbTh1sWf`uQtU| zd$95It5_j%xkq?i-Fhoxs!QaaMuGi3;O_$L4)`4tM}sn$J`@u9({_OUiM~J0wfE5D znzi2`e9?9dK?!>NFlk5(8Nip|7O*^fKxkkI3a24R*h3;fpLQzc9u=;E>2@HD*{jA= zo*a2@8KI;IE$&{ROT$9~ZQ=u>fpls*a`$+~6{s(wZe?wm2R!XBFxx#ZqJ4e*w#MKO- zrNxF~qMqe@gh|aPSR~J%JcdLMkM4GAxdqzpqQ#H>Gon1r%#mgr71}g_7yNBkyj)Rt zV2x(5rk~bLQW~WIey9Z;+5pAY-niyt;mJ~8eKVy)(srJ4Ewc)*9auJ_THUTl3bEys zol$VE*>^YWyQ4rIO`2@r&!&WCGAAD<$J)*}d_Hck|8=+}}% zqXC-?$4x5-t_U6=lW_8$Ge^e*kWA?>#5GBcx3?cp=|H@_dlzX91qp|N)3Q@=r1@$@ z`={gvY2+hk>Gm$_d^REx1+7oK_AgNobe1|B>$FdaXj=S1yA1D@=Taa}C}m)r#M{$E z0b11fH~+i0_S;{0RpNzD_QnhM|0H!Fao~gBWxVkFuN?^xRsyj4#0z(e_2{aOdtdX+ zm>XWh37<5qjEzX=TcgoiJ;Xs}>KJ+iaQSc@%Sh890fg=!6dNYi=O}d=2zRvE{fXdi zy-r;SRg37mbt{A>d&#Qd+x0?`&69X6q{N*k23hzIVo$&sCPH<&nRzL&J{H7(8q1{> z2)fj802o3+c&56?xCz)#e<0P8Yq|=U1+=GlbM%9{lT=fB56;(dejj)XeQjIB`n!sH zpqVLQl+m`>zW1cL&JR35$AF*Ra8<(Zkd)Xp(C%VkuAUYVn6%1mlt*(RGu@*4cJyeb zbRs6HO}v-npnE2V+~Cle%b*L0fdz(GgWi6dqx2RA(R$aKcFx({LJy!mwuOdGFD>kTVAuS-}-^%jU_s1w>84&TOgjezD#I^;T;Llv^I;xO%GZGv)U8F^L- zGI&}Tr;d(P*qwqG_H|3%WJs2R2zbH*DtWx?NdnRh47v5m6QWk!5L-WS1{6#O;x%be zC&@rQY8^0`;`cl!0=y1@yy!hIxbd?jJn{bqxQqRoK$ONd#B&od2XPkxu%Ufbg3Ovo zk&5Bs`50a@c^GY@I0RxH1{}V&J`Mt|1sO$B9B@C6pwAwT5JV90*6j^a1sD({0BRNN zO`fdKoDlp~3DBwWFZS2??-7mvO6QL9Lr?ngLV3+Fs(KrwAEoJDbSRu7UXN|(x15IT zE`c`{8n25uc?y#U|Fe0R=PF=+PPf?zLD4`OgeRbNxCDBOiU$^r5E8!v*E6jLClVez zdU|OM02?PA(7yvzuo#X%s^c63#tgW91AVqQV#s2!3I9eGstaFQc>)(|<1j`#BN_o6 z(7~t8SRBH`ocCgm6k6dWeAL42v6A3@gl-vQ3M6ZR8_p&HTh8aThhc3(j}?(IXbuZ_ zBoNUHc&aN@2#mmYLzgworH>M{PAoM)#E?f13wzVp9Bpj&+zp5hP7hV8v8h#+JEZPH z*aL8ylseJvg*f^=6!_80+<5?_dJYX}Cn~2sd)v&IVZNPS561LEBw74<99qrnG!_O- z4^gL9$T&DAL;O9`Q+7HS9CKp{qYcyf!(krzJmi9F2$-M}4U|yF1oAk_0{X%WTRhQ# zAi>*$Nf0Y8Xfa`cqo=?hIXhRrl@W)YIzX~)6p1*Qy<+I1S6@_yY+Z2}NkwppI%Wxr zls$F>bihOcx;T#<$dcQ1^op~^We))u=_kE}2yp65noerNw1)h9 zRV}4Qcw~84%=4WoeX2xYXCQmoBEPhfDE+3pF9lT6({1NyiJY& zxkH2f;)NebyztL@k+>N zyVI0h&ot$5$PPV!QWR7Ym5rEzQ|(?p@jhC!Z<%;u+@R+M`BsMeaa@^Ua|37YiCGtO zEvK#MY$XMvDX{+Ye1<&dMVGjxO;Ys=_)adfL0I1k6s`_*Hf+b+76Ik`tdi_30S&;! zxB8rB=r~Cno!o{?#ffSUzKLQ$eAuL%*GE@Y^cEE5TzYXK(2STwk!T79(m2fs;|WKb zNHOT&5g>O+q~srq0Y=pu+P+(fLjgfMtBO7Xhk6hh;|30k0hh|5QSfH~M5%dy{~2~z zM)9AAA7vJ)!g#hF+<2bsJw5p7IJvSQCbIzWNofK&v<}z!vwC%fe$Iy(4^vY0LlsQU_0t z)=dl==O)~AvzL3ydAf~J593;`3?QN+*m2Ks0sWExlEf)g0zP#MDm`8~>IEba;?=UbQT~S84GD#n@L=c;e%OS z(xRHjdu7$BHcQhoOp-*$4x~62SRaj(yM;~c%~n~ATCY0#?4VTJaZg-NB%H+2Yo)k2 zVeXpJm(yE4varIs)c?(>uT#BEgd%OLg-={B5`za1k;+JrDQ3$;V#PpSfESCdbE>?R zGODFv6uUh7BqDL5V6lh2?NM3awgR6fI@=sp$g>Bj@xK9$zX}V$SNBOEM)L5z)cCJ$ zb!z-auJ}93-wYbYc1Nh|BM(6w4l!HpYoPu@3{>~;B0z)P#&YVw-OpGr5VJpye^-)Y z&KLW_`b&ZJM6a=X-%nEq@`uvjAgq5p_wwAAfY)n-`}!Hq9}290+6(KUCsPOBTZ#V{ z!us~+{+9^r-TA(-UK3c4_rm(%YU;q>{iCt35Y`W$`CgE)PCeTf){g|%CwgH$awT=( zLu%+x2y5=;^-mJkw?E$()?W#%kN3iQ^hSa8uL*1LS0DX#!us-a`-~NQ=Bw*SqWg@* z(2oVy2eyTE=tJtEuNmI&lu&l=bYu~cuoHxvIi6S#u9XJ|IKo@-n+>J_oUa~c96o%X zFylYyWX4Bt1T%gFzue&Ge)|c-_zIvr{p4F(>)QLv9xQ?Kp>3fYdj042f!}Z*b$W1s zus-oyZ|R3oUSK`i1uGyJ9Rx-nHH<$4l%GDf$H-$!eDzT7&Y_&J{m}Y za;IiMxv_}a>DLdjkbv3(O0bMT0!C$UfTOL12Mpw=<#k2=7<+&574x9{O0Acs-f!U7b-j`UMt=*hE5^h4`^A$#e{k=E z-{5!n^lwoNlUz!kn@Pr(5{YCoF`Jl4%p|9i(}`H(ToQlcF+ErY0w2vG9pR!q~rO-+_I`+xM*fmhnFUQ%`SL literal 0 HcmV?d00001 diff --git a/testing/btest/policy/protocols/smtp/basic.test b/testing/btest/policy/protocols/smtp/basic.test new file mode 100644 index 0000000000..2717df0f49 --- /dev/null +++ b/testing/btest/policy/protocols/smtp/basic.test @@ -0,0 +1,4 @@ +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT +# @TEST-EXEC: btest-diff smtp.log + +@load protocols/smtp/base/main diff --git a/testing/btest/policy/protocols/smtp/mime-extract.test b/testing/btest/policy/protocols/smtp/mime-extract.test new file mode 100644 index 0000000000..d75ddbdc74 --- /dev/null +++ b/testing/btest/policy/protocols/smtp/mime-extract.test @@ -0,0 +1,26 @@ +# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT +# @TEST-EXEC: btest-diff smtp_mime.log +# @TEST-EXEC: btest-diff smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat +# @TEST-EXEC: btest-diff smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test" +# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat +# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat + +@load protocols/smtp/base/main +@load protocols/smtp/base/mime + +redef SMTP::extract_file_types=/text\/plain/; + +event bro_init() + { + Log::remove_default_filter(SMTP::SMTP_MIME); + Log::add_filter(SMTP::SMTP_MIME, [$name="normalized-mime-types", + $pred=function(rec: SMTP::MimeInfo): bool + { + if ( rec?$mime_type ) + rec$mime_type = "FAKE_MIME"; + return T; + } + ]); + } diff --git a/testing/btest/policy/protocols/smtp/mime.test b/testing/btest/policy/protocols/smtp/mime.test new file mode 100644 index 0000000000..41188d5eaa --- /dev/null +++ b/testing/btest/policy/protocols/smtp/mime.test @@ -0,0 +1,24 @@ +# Checks logging of mime types and md5 calculation. Mime type in the log +# is normalized to prevent sensitivity to libmagic version. + +# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT +# @TEST-EXEC: btest-diff smtp_mime.log + +@load protocols/smtp/base/main +@load protocols/smtp/base/mime + +redef SMTP::generate_md5=/text\/plain/; + +event bro_init() + { + Log::remove_default_filter(SMTP::SMTP_MIME); + Log::add_filter(SMTP::SMTP_MIME, [$name="normalized-mime-types", + $pred=function(rec: SMTP::MimeInfo): bool + { + if ( rec?$mime_type ) + rec$mime_type = "FAKE_MIME"; + return T; + } + ]); + } From 4b741293b16af992a59711d891243f835d028c1c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 29 Jul 2011 15:20:35 -0500 Subject: [PATCH 02/10] Make the doc.coverage test happy. --- doc/scripts/DocSourcesList.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index a26b9dcedd..8fa4c78c21 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -97,6 +97,7 @@ rest_target(${psd} protocols/mime/file-hash.bro) rest_target(${psd} protocols/mime/file-ident.bro) rest_target(${psd} protocols/rpc/base.bro) rest_target(${psd} protocols/smtp/base/main.bro) +rest_target(${psd} protocols/smtp/base/mime.bro) rest_target(${psd} protocols/smtp/base/software.bro) rest_target(${psd} protocols/smtp/detect-suspicious-orig.bro) rest_target(${psd} protocols/ssh/base.bro) From 8e7a76b5481fead9dd33682451e6cde8b4f23c27 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 10 Aug 2011 13:37:58 -0400 Subject: [PATCH 03/10] HTTP now uses the extract_filename_from_content_disposition function. --- scripts/base/protocols/http/main.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 172fb2bd1c..8cd80bde5f 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -214,7 +214,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr c$http$response_content_length = extract_count(value); else if ( name == "CONTENT-DISPOSITION" && /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) - c$http$filename = sub(value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, ""); + c$http$filename = extract_filename_from_content_disposition(value); } } From 6f50e362db1a5b8c456dde979921b958c0e43dc5 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:32:05 -0400 Subject: [PATCH 04/10] Updates for the build system and site local scripts for cluster. --- scripts/CMakeLists.txt | 21 +++++++++++++------ .../base/frameworks/cluster/nodes/manager.bro | 3 +++ .../base/frameworks/cluster/nodes/proxy.bro | 3 +++ .../base/frameworks/cluster/nodes/worker.bro | 3 +++ .../frameworks/cluster/setup-connections.bro | 1 + scripts/site/local-manager.bro | 7 +++++++ scripts/site/local-proxy.bro | 2 ++ scripts/site/local-worker.bro | 1 + 8 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 scripts/site/local-manager.bro create mode 100644 scripts/site/local-proxy.bro create mode 100644 scripts/site/local-worker.bro diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 5979e6befd..9933508b2d 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -1,18 +1,27 @@ include(InstallPackageConfigFile) install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING - PATTERN "all.bro" EXCLUDE - PATTERN "site/local.bro" EXCLUDE - PATTERN "bro.init" + PATTERN "site/local*" EXCLUDE PATTERN "*.bro" PATTERN "*.sig" - PATTERN "*.osf" + PATTERN "*.fp" ) -# Install as a config file since the local.bro script is meant to be +# Install all local* scripts as config files since they are meant to be # user modify-able. InstallPackageConfigFile( ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro ${POLICYDIR}/site local.bro) - +InstallPackageConfigFile( + ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro + ${POLICYDIR}/site + local-manager.bro) +InstallPackageConfigFile( + ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro + ${POLICYDIR}/site + local-proxy.bro) +InstallPackageConfigFile( + ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro + ${POLICYDIR}/site + local-worker.bro) diff --git a/scripts/base/frameworks/cluster/nodes/manager.bro b/scripts/base/frameworks/cluster/nodes/manager.bro index c9ce8c2d1a..b85c20ae77 100644 --- a/scripts/base/frameworks/cluster/nodes/manager.bro +++ b/scripts/base/frameworks/cluster/nodes/manager.bro @@ -10,6 +10,9 @@ @prefixes += cluster-manager +## Load the script for local site configuration for the manager node. +@load site/local-manager + ## Turn off remote logging since this is the manager and should only log here. redef Log::enable_remote_logging = F; diff --git a/scripts/base/frameworks/cluster/nodes/proxy.bro b/scripts/base/frameworks/cluster/nodes/proxy.bro index 377b087b36..60df27b452 100644 --- a/scripts/base/frameworks/cluster/nodes/proxy.bro +++ b/scripts/base/frameworks/cluster/nodes/proxy.bro @@ -1,6 +1,9 @@ @prefixes += cluster-proxy +## Load the script for local site configuration for proxy nodes. +@load site/local-proxy + ## The proxy only syncs state; does not forward events. redef forward_remote_events = F; redef forward_remote_state_changes = T; diff --git a/scripts/base/frameworks/cluster/nodes/worker.bro b/scripts/base/frameworks/cluster/nodes/worker.bro index cf8620c5d7..53d5d53872 100644 --- a/scripts/base/frameworks/cluster/nodes/worker.bro +++ b/scripts/base/frameworks/cluster/nodes/worker.bro @@ -1,6 +1,9 @@ @prefixes += cluster-worker +## Load the script for local site configuration for the worker nodes. +@load site/local-worker + ## Don't do any local logging. redef Log::enable_local_logging = F; diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 04d474e604..2a16ca1ea1 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -1,3 +1,4 @@ + module Cluster; event bro_init() &priority=9 diff --git a/scripts/site/local-manager.bro b/scripts/site/local-manager.bro new file mode 100644 index 0000000000..aa28bd79da --- /dev/null +++ b/scripts/site/local-manager.bro @@ -0,0 +1,7 @@ +##! Local site policy loaded only by the manager in a cluster. + +# If you are running a cluster you should define your Notice::policy here +# so that notice processing occurs on the manager. +redef Notice::policy += { + +}; diff --git a/scripts/site/local-proxy.bro b/scripts/site/local-proxy.bro new file mode 100644 index 0000000000..1b71cc1870 --- /dev/null +++ b/scripts/site/local-proxy.bro @@ -0,0 +1,2 @@ +##! Local site policy loaded only by the proxies if Bro is running as a cluster. + diff --git a/scripts/site/local-worker.bro b/scripts/site/local-worker.bro new file mode 100644 index 0000000000..b2a100e135 --- /dev/null +++ b/scripts/site/local-worker.bro @@ -0,0 +1 @@ +##! Local site policy loaded only by the workers if Bro is running as a cluster. \ No newline at end of file From 9a06cece67087cf74ef8efc48478580cf5747766 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:32:31 -0400 Subject: [PATCH 05/10] ConnSize analyzer is turned on by default now. --- scripts/base/init-bare.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 30674fafc7..f1468cea6b 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -509,7 +509,7 @@ const tunnel_port = 0/udp &redef; # packets and IP-level bytes transfered by each endpoint. If # true, these values are returned in the connection's endpoint # record val. -const use_conn_size_analyzer = F &redef; +const use_conn_size_analyzer = T &redef; const UDP_INACTIVE = 0; const UDP_ACTIVE = 1; # means we've seen something from this endpoint From 9c2273b7a7dfff8eda1f44ea6c25c5ebc6fab019 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:35:50 -0400 Subject: [PATCH 06/10] Updates for SSH scripts. --- scripts/base/protocols/ssh/main.bro | 197 ++++++------------ .../protocols/ssh/detect-bruteforcing.bro | 79 +++++++ scripts/policy/protocols/ssh/geo-data.bro | 39 ++++ .../protocols/ssh/interesting-hostnames.bro | 50 +++++ scripts/policy/protocols/ssh/software.bro | 8 +- scripts/site/local.bro | 5 + 6 files changed, 237 insertions(+), 141 deletions(-) create mode 100644 scripts/policy/protocols/ssh/detect-bruteforcing.bro create mode 100644 scripts/policy/protocols/ssh/geo-data.bro create mode 100644 scripts/policy/protocols/ssh/interesting-hostnames.bro diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index 7cc87b6684..5233c6da97 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -1,74 +1,58 @@ +##! Base SSH analysis script. The heuristic to blindly determine success or +##! failure for SSH connections is implemented here. At this time, it only +##! uses the size of the data being returned from the server to make the +##! heuristic determination about success of the connection. +##! Requires that :bro:id:`use_conn_size_analyzer` is set to T! The heuristic +##! is not attempted if the connection size analyzer isn't enabled. module SSH; export { redef enum Log::ID += { SSH }; - redef enum Notice::Type += { - Login, - Password_Guessing, - Login_By_Password_Guesser, - Login_From_Interesting_Hostname, - Bytecount_Inconsistency, - }; - type Info: record { ts: time &log; uid: string &log; id: conn_id &log; + ## Indicates if the login was heuristically guessed to be "success" + ## or "failure". status: string &log &optional; - direction: string &log &optional; - remote_location: geo_location &log &optional; + ## Direction of the connection. If the client was a local host + ## logging into an external host, this would be OUTBOUD. INBOUND + ## would be set for the opposite situation. + # TODO: handle local-local and remote-remote better. + direction: Direction &log &optional; + ## The software string given by the client. client: string &log &optional; + ## The software string given by the server. server: string &log &optional; + ## The amount of data returned from the server. This is currently + ## the only measure of the success heuristic and it is logged to + ## assist analysts looking at the logs to make their own determination + ## about the success on a case-by-case basis. resp_size: count &log &default=0; ## Indicate if the SSH session is done being watched. done: bool &default=F; }; - - const password_guesses_limit = 30 &redef; - # The size in bytes at which the SSH connection is presumed to be - # successful. + ## The size in bytes at which the SSH connection is presumed to be + ## successful. const authentication_data_size = 5500 &redef; - # The amount of time to remember presumed non-successful logins to build - # model of a password guesser. - const guessing_timeout = 30 mins &redef; - - # The set of countries for which you'd like to throw notices upon successful login - # requires Bro compiled with libGeoIP support - const watched_countries: set[string] = {"RO"} &redef; - - # Strange/bad host names to originate successful SSH logins - const interesting_hostnames = - /^d?ns[0-9]*\./ | - /^smtp[0-9]*\./ | - /^mail[0-9]*\./ | - /^pop[0-9]*\./ | - /^imap[0-9]*\./ | - /^www[0-9]*\./ | - /^ftp[0-9]*\./ &redef; - - # This is a table with orig subnet as the key, and subnet as the value. - const ignore_guessers: table[subnet] of subnet &redef; - - # If true, we tell the event engine to not look at further data - # packets after the initial SSH handshake. Helps with performance - # (especially with large file transfers) but precludes some - # kinds of analyses (e.g., tracking connection size). + ## If true, we tell the event engine to not look at further data + ## packets after the initial SSH handshake. Helps with performance + ## (especially with large file transfers) but precludes some + ## kinds of analyses (e.g., tracking connection size). const skip_processing_after_detection = F &redef; - # Keeps count of how many rejections a host has had - global password_rejections: table[addr] of TrackCount - &write_expire=guessing_timeout - &synchronized; - - # Keeps track of hosts identified as guessing passwords - # TODO: guessing_timeout doesn't work correctly here. If a user redefs - # the variable, it won't take effect. - global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized; + ## This event is generated when the heuristic thinks that a login + ## was successful. + global heuristic_successful_login: event(c: connection); + + ## This event is generated when the heuristic thinks that a login + ## failed. + global heuristic_failed_login: event(c: connection); global log_ssh: event(rec: Info); } @@ -106,116 +90,51 @@ function check_ssh_connection(c: connection, done: bool) # If this is still a live connection and the byte count has not # crossed the threshold, just return and let the resheduled check happen later. - if ( !done && c$resp$size < authentication_data_size ) + if ( !done && c$resp$num_bytes_ip < authentication_data_size ) return; # Make sure the server has sent back more than 50 bytes to filter out # hosts that are just port scanning. Nothing is ever logged if the server # doesn't send back at least 50 bytes. - if ( c$resp$size < 50 ) + if ( c$resp$num_bytes_ip < 50 ) return; - local status = "failure"; - local direction = Site::is_local_addr(c$id$orig_h) ? "to" : "from"; - local location: geo_location; - location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h); + c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND; + c$ssh$resp_size = c$resp$num_bytes_ip; - if ( done && c$resp$size < authentication_data_size ) + if ( c$resp$num_bytes_ip < authentication_data_size ) { - # presumed failure - if ( c$id$orig_h !in password_rejections ) - password_rejections[c$id$orig_h] = new_track_count(); - - # Track the number of rejections - if ( !(c$id$orig_h in ignore_guessers && - c$id$resp_h in ignore_guessers[c$id$orig_h]) ) - ++password_rejections[c$id$orig_h]$n; - - if ( default_check_threshold(password_rejections[c$id$orig_h]) ) - { - add password_guessers[c$id$orig_h]; - NOTICE([$note=Password_Guessing, - $conn=c, - $msg=fmt("SSH password guessing by %s", c$id$orig_h), - $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n), - $n=password_rejections[c$id$orig_h]$n]); - } - } - # TODO: This is to work around a quasi-bug in Bro which occasionally - # causes the byte count to be oversized. - # Watch for Gregors work that adds an actual counter of bytes transferred. - else if ( c$resp$size < 20000000 ) + c$ssh$status = "failure"; + event SSH::heuristic_failed_login(c); + } + else { # presumed successful login - status = "success"; - c$ssh$done = T; - - if ( c$id$orig_h in password_rejections && - password_rejections[c$id$orig_h]$n > password_guesses_limit && - c$id$orig_h !in password_guessers ) - { - add password_guessers[c$id$orig_h]; - NOTICE([$note=Login_By_Password_Guesser, - $conn=c, - $n=password_rejections[c$id$orig_h]$n, - $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h), - $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]); - } - - local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)", - direction, location$country_code, location$region, location$city, - location$latitude, location$longitude, - id_string(c$id), c$resp$size); - NOTICE([$note=Login, - $conn=c, - $msg=message, - $sub=location$country_code]); - - # Check to see if this login came from an interesting hostname - when ( local hostname = lookup_addr(c$id$orig_h) ) - { - if ( interesting_hostnames in hostname ) - { - NOTICE([$note=Login_From_Interesting_Hostname, - $conn=c, - $msg=fmt("Strange login from %s", hostname), - $sub=hostname]); - } - } - - if ( location$country_code in watched_countries ) - { - - } - + c$ssh$status = "success"; + event SSH::heuristic_successful_login(c); } - else if ( c$resp$size >= 200000000 ) - { - NOTICE([$note=Bytecount_Inconsistency, - $conn=c, - $msg="During byte counting in SSH analysis, an overly large value was seen.", - $sub=fmt("%d",c$resp$size)]); - } - - c$ssh$remote_location = location; - c$ssh$status = status; - c$ssh$direction = direction; - c$ssh$resp_size = c$resp$size; - - Log::write(SSH, c$ssh); # Set the "done" flag to prevent the watching event from rescheduling # after detection is done. - c$ssh$done; + c$ssh$done=T; - # Stop watching this connection, we don't care about it anymore. if ( skip_processing_after_detection ) { + # Stop watching this connection, we don't care about it anymore. skip_further_processing(c$id); set_record_packets(c$id, F); } } +event SSH::heuristic_successful_login(c: connection) &priority=-5 + { + Log::write(SSH, c$ssh); + } +event SSH::heuristic_failed_login(c: connection) &priority=-5 + { + Log::write(SSH, c$ssh); + } + event connection_state_remove(c: connection) &priority=-5 { if ( c?$ssh ) @@ -226,7 +145,7 @@ event ssh_watcher(c: connection) { local id = c$id; # don't go any further if this connection is gone already! - if ( !connection_exists(id) ) + if ( ! connection_exists(id) ) return; check_ssh_connection(c, F); @@ -244,5 +163,9 @@ event ssh_client_version(c: connection, version: string) &priority=5 { set_session(c); c$ssh$client = version; - schedule +15secs { ssh_watcher(c) }; + + # The heuristic detection for SSH relies on the ConnSize analyzer. + # Don't do the heuristics if it's disabled. + if ( use_conn_size_analyzer ) + schedule +15secs { ssh_watcher(c) }; } diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro new file mode 100644 index 0000000000..36e73bfa59 --- /dev/null +++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro @@ -0,0 +1,79 @@ + +module SSH; + +export { + redef enum Notice::Type += { + ## Indicates that a host has been identified as crossing the + ## :bro:id:`password_guesses_limit` threshold with heuristically + ## determined failed logins. + Password_Guessing, + ## Indicates that a host previously identified as a "password guesser" + ## has now had a heuristically successful login attempt. + Login_By_Password_Guesser, + }; + + ## The number of failed SSH connections before a host is designated as + ## guessing passwords. + const password_guesses_limit = 30 &redef; + + ## The amount of time to remember presumed non-successful logins to build + ## model of a password guesser. + const guessing_timeout = 30 mins &redef; + + ## This value can be used to exclude hosts or entire networks from being + ## tracked as potential "guessers". There are cases where the success + ## heuristic fails and this acts as the whitelist. The index represents + ## client subnets and the yield value represents server subnets. + const ignore_guessers: table[subnet] of subnet &redef; + + ## Keeps count of how many rejections a host has had. + global password_rejections: table[addr] of TrackCount + &write_expire=guessing_timeout + &synchronized; + + ## Keeps track of hosts identified as guessing passwords. + global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized; +} + +event SSH::heuristic_successful_login(c: connection) + { + local id = c$id; + + # TODO: this should be migrated to the metrics framework. + if ( id$orig_h in password_rejections && + password_rejections[id$orig_h]$n > password_guesses_limit && + id$orig_h !in password_guessers ) + { + add password_guessers[id$orig_h]; + NOTICE([$note=Login_By_Password_Guesser, + $conn=c, + $n=password_rejections[id$orig_h]$n, + $msg=fmt("Successful SSH login by password guesser %s", id$orig_h), + $sub=fmt("%d failed logins", password_rejections[id$orig_h]$n)]); + } + } + +event SSH::heuristic_failed_login(c: connection) + { + local id = c$id; + + # presumed failure + if ( id$orig_h !in password_rejections ) + password_rejections[id$orig_h] = new_track_count(); + + # Track the number of rejections + # TODO: this should be migrated to the metrics framework. + if ( ! (id$orig_h in ignore_guessers && + id$resp_h in ignore_guessers[id$orig_h]) ) + ++password_rejections[id$orig_h]$n; + + if ( default_check_threshold(password_rejections[id$orig_h]) ) + { + add password_guessers[id$orig_h]; + NOTICE([$note=Password_Guessing, + $conn=c, + $msg=fmt("SSH password guessing by %s", id$orig_h), + $sub=fmt("%d apparently failed logins", password_rejections[id$orig_h]$n), + $n=password_rejections[id$orig_h]$n]); + } + } \ No newline at end of file diff --git a/scripts/policy/protocols/ssh/geo-data.bro b/scripts/policy/protocols/ssh/geo-data.bro new file mode 100644 index 0000000000..97bd0a5803 --- /dev/null +++ b/scripts/policy/protocols/ssh/geo-data.bro @@ -0,0 +1,39 @@ +##! This implements all of the additional information and geodata detections +##! for SSH analysis. + +module SSH; + +export { + redef enum Notice::Type += { + ## If an SSH login is seen to or from a "watched" country based on the + ## :bro:id:`SSH::watched_countries` variable then this notice will + ## be generated. + Login_From_Watched_Country, + }; + + ## The set of countries for which you'd like to throw notices upon + ## successful login + const watched_countries: set[string] = {"RO"} &redef; + + redef record Info += { + ## Add geographic data related to the "remote" host of the connection. + remote_location: geo_location &log &optional; + }; +} + +event SSH::heuristic_successful_login(c: connection) &priority=5 + { + local location: geo_location; + location = (c$ssh$direction == OUTBOUND) ? + lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h); + + # Add the location data to the SSH record. + c$ssh$remote_location = location; + + if ( location$country_code in watched_countries ) + { + NOTICE([$note=Login_From_Watched_Country, + $conn=c, + $msg=fmt("SSH login from watched country: %s", location$country_code)]); + } + } diff --git a/scripts/policy/protocols/ssh/interesting-hostnames.bro b/scripts/policy/protocols/ssh/interesting-hostnames.bro new file mode 100644 index 0000000000..cf6ab7e40a --- /dev/null +++ b/scripts/policy/protocols/ssh/interesting-hostnames.bro @@ -0,0 +1,50 @@ + +module SSH; + +export { + redef enum Notice::Type += { + ## Generated if a login originates from a host matched by the + ## :bro:id:`interesting_hostnames` regular expression. + Login_From_Interesting_Hostname, + ## Generated if a login goes to a host matched by the + ## :bro:id:`interesting_hostnames` regular expression. + Login_To_Interesting_Hostname, + }; + + ## Strange/bad host names to see successful SSH logins from or to. + const interesting_hostnames = + /^d?ns[0-9]*\./ | + /^smtp[0-9]*\./ | + /^mail[0-9]*\./ | + /^pop[0-9]*\./ | + /^imap[0-9]*\./ | + /^www[0-9]*\./ | + /^ftp[0-9]*\./ &redef; +} + +event SSH::heuristic_successful_login(c: connection) + { + # Check to see if this login came from an interesting hostname. + when ( local orig_hostname = lookup_addr(c$id$orig_h) ) + { + if ( interesting_hostnames in orig_hostname ) + { + NOTICE([$note=Login_From_Interesting_Hostname, + $conn=c, + $msg=fmt("Interesting login from hostname: %s", orig_hostname), + $sub=orig_hostname]); + } + } + # Check to see if this login went to an interesting hostname. + when ( local resp_hostname = lookup_addr(c$id$orig_h) ) + { + if ( interesting_hostnames in resp_hostname ) + { + NOTICE([$note=Login_To_Interesting_Hostname, + $conn=c, + $msg=fmt("Interesting login to hostname: %s", resp_hostname), + $sub=resp_hostname]); + } + } + } + diff --git a/scripts/policy/protocols/ssh/software.bro b/scripts/policy/protocols/ssh/software.bro index d40ad513c8..1aa3bce1a2 100644 --- a/scripts/policy/protocols/ssh/software.bro +++ b/scripts/policy/protocols/ssh/software.bro @@ -3,8 +3,8 @@ module SSH; export { redef enum Software::Type += { - SSH_SERVER, - SSH_CLIENT, + SERVER, + CLIENT, }; } @@ -12,7 +12,7 @@ event ssh_client_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, ""); - local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT); + local si = Software::parse(cleaned_version, c$id$orig_h, CLIENT); Software::found(c$id, si); } @@ -20,6 +20,6 @@ event ssh_server_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, ""); - local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER); + local si = Software::parse(cleaned_version, c$id$resp_h, SERVER); Software::found(c$id, si); } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index cd0d6634b2..24a0aa144b 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -45,6 +45,11 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Load the script to enable SSL/TLS certificate validation. @load protocols/ssl/validate-certs +# If you have libGeoIP support built in, do some geographic detections and logging. +@load protocols/ssh/geo-data +@load protocols/ssh/detect-bruteforcing +@load protocols/ssh/interesting-hostnames + # Uncomment this redef if you want to extract SMTP MIME entities for # some file types. The numbers given indicate how many bytes to extract for # the various mime types. From d201215359ad0475bde319373a6af79a64f42033 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:37:57 -0400 Subject: [PATCH 07/10] Added the profiling script. --- scripts/policy/misc/profiling.bro | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts/policy/misc/profiling.bro diff --git a/scripts/policy/misc/profiling.bro b/scripts/policy/misc/profiling.bro new file mode 100644 index 0000000000..457675b1d6 --- /dev/null +++ b/scripts/policy/misc/profiling.bro @@ -0,0 +1,19 @@ +##! Turns on profiling of Bro resource consumption. + +module Profiling; + +redef profiling_file = open_log_file("prof"); + +export { + ## Cheap profiling every 15 seconds. + redef profiling_interval = 15 secs &redef; +} + +# Expensive profiling every 5 minutes. +redef expensive_profiling_multiple = 20; + +event bro_init() + { + set_buf(profiling_file, F); + } + From b45c175147847f417ebcadd16dca1597ce2daee1 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 08:26:20 -0400 Subject: [PATCH 08/10] Split out more SMTP analysis functionality. --- scripts/base/protocols/smtp/main.bro | 53 +----------------- scripts/policy/protocols/smtp/blocklists.bro | 58 ++++++++++++++++++++ 2 files changed, 61 insertions(+), 50 deletions(-) create mode 100644 scripts/policy/protocols/smtp/blocklists.bro diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index 2b0991d8f9..02b282894c 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -4,13 +4,6 @@ module SMTP; export { redef enum Log::ID += { SMTP }; - redef enum Notice::Type += { - ## Indicates that the server sent a reply mentioning an SMTP block list. - BL_Error_Message, - ## Indicates the client's address is seen in the block list error message. - BL_Blocked_Host, - }; - type Info: record { ts: time &log; uid: string &log; @@ -59,26 +52,7 @@ export { ## ALL_HOSTS - always capture the entire path. ## NO_HOSTS - never capture the path. const mail_path_capture = ALL_HOSTS &redef; - - # This matches content in SMTP error messages that indicate some - # block list doesn't like the connection/mail. - const bl_error_messages = - /spamhaus\.org\// - | /sophos\.com\/security\// - | /spamcop\.net\/bl/ - | /cbl\.abuseat\.org\// - | /sorbs\.net\// - | /bsn\.borderware\.com\// - | /mail-abuse\.com\// - | /b\.barracudacentral\.com\// - | /psbl\.surriel\.com\// - | /antispam\.imp\.ch\// - | /dyndns\.com\/.*spam/ - | /rbl\.knology\.net\// - | /intercept\.datapacket\.net\// - | /uceprotect\.net\// - | /hostkarma\.junkemailfilter\.com\// &redef; - + global log_smtp: event(rec: Info); ## Configure the default ports for SMTP analysis. @@ -181,27 +155,6 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, # This continually overwrites, but we want the last reply, # so this actually works fine. c$smtp$last_reply = fmt("%d %s", code, msg); - - if ( code != 421 && code >= 400 ) - { - # Raise a notice when an SMTP error about a block list is discovered. - if ( bl_error_messages in msg ) - { - local note = BL_Error_Message; - local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h); - - # Determine if the originator's IP address is in the message. - local ips = find_ip_addresses(msg); - local text_ip = ""; - if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h ) - { - note = BL_Blocked_Host; - message = fmt("%s is on an SMTP block list", c$id$orig_h); - } - - NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]); - } - } } event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, @@ -256,7 +209,7 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5 else if ( h$name == "X-ORIGINATING-IP" ) { - local addresses = find_ip_addresses(h$name); + local addresses = find_ip_addresses(h$value); if ( 1 in addresses ) c$smtp$x_originating_ip = to_addr(addresses[1]); } @@ -274,7 +227,7 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=3 # If we've decided that we're done watching the received headers for # whatever reason, we're done. Could be due to only watching until # local addresses are seen in the received from headers. - if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from) + if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from ) return; local text_ip = find_address_in_smtp_header(h$value); diff --git a/scripts/policy/protocols/smtp/blocklists.bro b/scripts/policy/protocols/smtp/blocklists.bro new file mode 100644 index 0000000000..a3e75318bb --- /dev/null +++ b/scripts/policy/protocols/smtp/blocklists.bro @@ -0,0 +1,58 @@ + +@load base/protocols/smtp + +module SMTP; + +export { + redef enum Notice::Type += { + ## Indicates that the server sent a reply mentioning an SMTP block list. + Blocklist_Error_Message, + ## Indicates the client's address is seen in the block list error message. + Blocklist_Blocked_Host, + }; + + # This matches content in SMTP error messages that indicate some + # block list doesn't like the connection/mail. + const blocklist_error_messages = + /spamhaus\.org\// + | /sophos\.com\/security\// + | /spamcop\.net\/bl/ + | /cbl\.abuseat\.org\// + | /sorbs\.net\// + | /bsn\.borderware\.com\// + | /mail-abuse\.com\// + | /b\.barracudacentral\.com\// + | /psbl\.surriel\.com\// + | /antispam\.imp\.ch\// + | /dyndns\.com\/.*spam/ + | /rbl\.knology\.net\// + | /intercept\.datapacket\.net\// + | /uceprotect\.net\// + | /hostkarma\.junkemailfilter\.com\// &redef; + +} + +event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, + msg: string, cont_resp: bool) &priority=3 + { + if ( code >= 400 && code != 421 ) + { + # Raise a notice when an SMTP error about a block list is discovered. + if ( blocklist_error_messages in msg ) + { + local note = Blocklist_Error_Message; + local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h); + + # Determine if the originator's IP address is in the message. + local ips = find_ip_addresses(msg); + local text_ip = ""; + if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h ) + { + note = Blocklist_Blocked_Host; + message = fmt("%s is on an SMTP block list", c$id$orig_h); + } + + NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]); + } + } + } From 423769c61d563b08669973024d1d81f3260524b1 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 08:26:40 -0400 Subject: [PATCH 09/10] Updates to local.bro --- scripts/policy/protocols/http/detect-MHR.bro | 2 +- scripts/site/local.bro | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 11e1d9f87e..fd54a62aeb 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -1,7 +1,7 @@ ##! This script takes MD5 sums of files transferred over HTTP and checks them with ##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). ##! By default, not all file transfers will have MD5 sums calculated. Read the -##! documentation for the protocols/http/file-hash.bro script to see how to +##! documentation for the base/protocols/http/file-hash.bro script to see how to ##! configure which transfers will have hashes calculated. export { diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 24a0aa144b..f894a30432 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -22,6 +22,7 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Load all of the scripts that detect software in various protocols. @load protocols/http/software +#@load protocols/http/detect-webapps @load protocols/ftp/software @load protocols/smtp/software @load protocols/ssh/software @@ -45,11 +46,19 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Load the script to enable SSL/TLS certificate validation. @load protocols/ssl/validate-certs -# If you have libGeoIP support built in, do some geographic detections and logging. +# If you have libGeoIP support built in, do some geographic detections and +# logging for SSH traffic. @load protocols/ssh/geo-data +# Detect hosts doing SSH bruteforce attacks. @load protocols/ssh/detect-bruteforcing +# Detect logins using "interesting" hostnames. @load protocols/ssh/interesting-hostnames +# Detect MD5 sums in Team Cymru's Malware Hash Registry. +@load protocols/http/detect-MHR +# Detect SQL injection attacks +@load protocols/http/detect-sqli + # Uncomment this redef if you want to extract SMTP MIME entities for # some file types. The numbers given indicate how many bytes to extract for # the various mime types. From 240ae9790bbb7e1ff5f07e5f428fbe8e8ffc67c8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 14:59:01 -0400 Subject: [PATCH 10/10] Small updates for notice framework. - New ACTION_ADD_GEODATA to add geodata to notices in an extension field named remote_location. - Loading extend-email/hostnames by default now that it only does anything when the ACTION_EMAIL action is applied (finally). --- scripts/base/frameworks/notice/__load__.bro | 7 +-- .../frameworks/notice/actions/add-geodata.bro | 47 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 scripts/base/frameworks/notice/actions/add-geodata.bro diff --git a/scripts/base/frameworks/notice/__load__.bro b/scripts/base/frameworks/notice/__load__.bro index bbc1fcae0d..2cc93ee933 100644 --- a/scripts/base/frameworks/notice/__load__.bro +++ b/scripts/base/frameworks/notice/__load__.bro @@ -6,7 +6,8 @@ @load ./actions/drop @load ./actions/email_admin @load ./actions/page +@load ./actions/add-geodata -# Load the script to add hostnames to emails by default. -# NOTE: this exposes a memleak in async DNS lookups. -#@load ./extend-email/hostnames +# There shouldn't be any defaul toverhead from loading these since they +# *should* only do anything when notices have the ACTION_EMAIL action applied. +@load ./extend-email/hostnames diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro new file mode 100644 index 0000000000..71e9c6b490 --- /dev/null +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -0,0 +1,47 @@ +##! This script adds geographic location data to notices for the "remote" +##! host in a connection. It does make the assumption that one of the +##! addresses in a connection is "local" and one is "remote" which is +##! probably a safe assumption to make in most cases. If both addresses +##! are remote, it will use the $src address. + +module Notice; + +export { + redef enum Action += { + ## Indicates that the notice should have geodata added for the + ## "remote" host. :bro:id:`Site::local_nets` must be defined + ## in order for this to work. + ACTION_ADD_GEODATA + }; + + redef record Info += { + ## If libGeoIP support is built in, notices can have geographic + ## information attached to them. + remote_location: geo_location &log &optional; + }; + + ## Notice types which should have the "remote" location looked up. + ## If GeoIP support is not built in, this does nothing. + const lookup_location_types: set[Notice::Type] = {} &redef; + + ## Add a helper to the notice policy for looking up GeoIP data. + redef Notice::policy += { + [$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); }, + $priority = 10], + }; +} + +# This is handled at a high priority in case other notice handlers +# want to use the data. +event notice(n: Notice::Info) &priority=10 + { + if ( ACTION_ADD_GEODATA in n$actions && + |Site::local_nets| > 0 && + ! n?$remote_location ) + { + if ( n?$src && ! Site::is_local_addr(n$src) ) + n$remote_location = lookup_location(n$src); + else if ( n?$dst && ! Site::is_local_addr(n$dst) ) + n$remote_location = lookup_location(n$dst); + } + } \ No newline at end of file