From 17d74c23dbb60bcf2917d3f1b8ce2d8dc46adbd2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 29 Jul 2011 14:55:53 -0500 Subject: [PATCH 01/25] SMTP script refactor. (addresses #509) - message header state tracking is now done by handling mime_one_header instead of parsing the data in the smtp_data event - changed the logging point to be when an smtp_reply is seen in response to the end of a DATA section - the smtp package now uses it's own mime script and logging stream for logging entities, extraction, etc. - fixes for mime file extraction: now logs the extracted file name, and the count of extracted files needed to be maintained in the State record --- policy/protocols/mime/file-extract.bro | 12 +- policy/protocols/smtp/__load__.bro | 4 +- policy/protocols/smtp/base/__load__.bro | 5 +- policy/protocols/smtp/base/main.bro | 195 +++++-------- policy/protocols/smtp/base/mime.bro | 224 ++++++++++++++ policy/protocols/smtp/base/software.bro | 6 +- .../policy.protocols.smtp.basic/smtp.log | 2 + ...item_10.10.1.4:1470-74.53.140.153:25_1.dat | 14 + ...item_10.10.1.4:1470-74.53.140.153:25_2.dat | 275 ++++++++++++++++++ .../smtp_mime.log | 4 + .../policy.protocols.smtp.mime/smtp_mime.log | 4 + testing/btest/Traces/smtp.trace | Bin 0 -> 27850 bytes .../btest/policy/protocols/smtp/basic.test | 4 + .../policy/protocols/smtp/mime-extract.test | 26 ++ testing/btest/policy/protocols/smtp/mime.test | 24 ++ 15 files changed, 657 insertions(+), 142 deletions(-) create mode 100644 policy/protocols/smtp/base/mime.bro create mode 100644 testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log create mode 100644 testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log create mode 100644 testing/btest/Traces/smtp.trace create mode 100644 testing/btest/policy/protocols/smtp/basic.test create mode 100644 testing/btest/policy/protocols/smtp/mime-extract.test create mode 100644 testing/btest/policy/protocols/smtp/mime.test diff --git a/policy/protocols/mime/file-extract.bro b/policy/protocols/mime/file-extract.bro index d6989ad809..81f31d6cd6 100644 --- a/policy/protocols/mime/file-extract.bro +++ b/policy/protocols/mime/file-extract.bro @@ -16,11 +16,13 @@ export { extract_file: bool &default=F; ## Store the file handle here for the file currently being extracted. - extraction_file: file &optional; - + extraction_file: file &log &optional; + }; + + redef record State += { ## Store a count of the number of files that have been transferred in ## this conversation to create unique file names on disk. - num_extracted_files: count &optional; + num_extracted_files: count &default=0; }; } @@ -34,7 +36,7 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=3 { if ( c$mime$extract_file && c$mime$content_len == 0 ) { - local suffix = fmt("%d.dat", ++c$mime$num_extracted_files); + local suffix = fmt("%d.dat", ++c$mime_state$num_extracted_files); local fname = generate_extraction_filename(extraction_prefix, c, suffix); c$mime$extraction_file = open(fname); enable_raw_output(c$mime$extraction_file); @@ -57,4 +59,4 @@ event mime_end_entity(c: connection) &priority=-3 if ( c$mime?$extraction_file ) close(c$mime$extraction_file); } - \ No newline at end of file + diff --git a/policy/protocols/smtp/__load__.bro b/policy/protocols/smtp/__load__.bro index 08161451ac..fd6a7eaeee 100644 --- a/policy/protocols/smtp/__load__.bro +++ b/policy/protocols/smtp/__load__.bro @@ -1,4 +1,4 @@ -@load protocols/smtp/base +@load ./base # This should be optional -@load protocols/smtp/detect-suspicious-orig +@load ./detect-suspicious-orig diff --git a/policy/protocols/smtp/base/__load__.bro b/policy/protocols/smtp/base/__load__.bro index 826efdef0d..efa30304e1 100644 --- a/policy/protocols/smtp/base/__load__.bro +++ b/policy/protocols/smtp/base/__load__.bro @@ -1,2 +1,3 @@ -@load protocols/smtp/base/main -@load protocols/smtp/base/software \ No newline at end of file +@load ./main +@load ./software +@load ./mime diff --git a/policy/protocols/smtp/base/main.bro b/policy/protocols/smtp/base/main.bro index 0f7a674481..53f93b4bcb 100644 --- a/policy/protocols/smtp/base/main.bro +++ b/policy/protocols/smtp/base/main.bro @@ -37,15 +37,10 @@ export { path: vector of addr &log &optional; user_agent: string &log &optional; - ## Indicate if this session is currently transmitting SMTP message - ## envelope headers. - in_headers: bool &default=F; ## Indicate if the "Received: from" headers should still be processed. process_received_from: bool &default=T; - ## Maintain the current header for cases where there is header wrapping. - current_header: string &default=""; - ## Indicate when the message is logged and no longer applicable. - done: bool &default=F; + ## Indicates if client activity has been seen, but not yet logged + has_client_activity: bool &default=F; }; type State: record { @@ -139,26 +134,23 @@ function set_smtp_session(c: connection) if ( ! c?$smtp_state ) c$smtp_state = []; - if ( ! c?$smtp || c$smtp$done ) - { + if ( ! c?$smtp ) c$smtp = new_smtp_log(c); - } } - function smtp_message(c: connection) { - Log::write(SMTP, c$smtp); - - c$smtp$done = T; - # Track the number of messages seen in this session. - ++c$smtp_state$messages_transferred; + if ( c$smtp$has_client_activity ) + Log::write(SMTP, c$smtp); } event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5 { set_smtp_session(c); local upper_command = to_upper(command); + + if ( upper_command != "QUIT" ) + c$smtp$has_client_activity = T; if ( upper_command == "HELO" || upper_command == "EHLO" ) { @@ -175,26 +167,11 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) & else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg ) { - # In case this is not the first message in a session we want to - # essentially write out a log, clear the session tracking, and begin - # new session tracking. - if ( c$smtp_state$messages_transferred > 0 ) - { - smtp_message(c); - set_smtp_session(c); - } - local partially_done = split1(arg, /:[[:blank:]]*/)[2]; c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1]; } - - else if ( upper_command == "DATA" ) - { - c$smtp$in_headers = T; - } } - event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, msg: string, cont_resp: bool) &priority=5 { @@ -202,10 +179,10 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, # This continually overwrites, but we want the last reply, # so this actually works fine. + c$smtp$last_reply = fmt("%d %s", code, msg); + if ( code != 421 && code >= 400 ) { - c$smtp$last_reply = fmt("%d %s", code, msg); - # Raise a notice when an SMTP error about a block list is discovered. if ( bl_error_messages in msg ) { @@ -226,145 +203,103 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, } } -event smtp_data(c: connection, is_orig: bool, data: string) &priority=5 +event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, + msg: string, cont_resp: bool) &priority=-5 { - # Is there something we should be handling from the server? - if ( ! is_orig ) return; - set_smtp_session(c); - - if ( ! c$smtp$in_headers ) + if ( cmd == "." ) { - if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data ) - { - if ( ! c$smtp?$files ) - c$smtp$files = set(); - data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, ""); - add c$smtp$files[data]; - } - return; + # Track the number of messages seen in this session. + ++c$smtp_state$messages_transferred; + smtp_message(c); + c$smtp = new_smtp_log(c); + } + } + +event mime_one_header(c: connection, h: mime_header_rec) &priority=5 + { + if ( ! c?$smtp ) return; + c$smtp$has_client_activity = T; + + if ( h$name == "CONTENT-DISPOSITION" && + /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) + { + if ( ! c$smtp?$files ) c$smtp$files = set(); + add c$smtp$files[sub(h$value, + /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "")]; } - if ( /^[[:blank:]]*$/ in data ) - c$smtp$in_headers = F; + else if ( h$name == "MESSAGE-ID" ) + c$smtp$msg_id = h$value; - # This is to reconstruct headers that tend to wrap around. - if ( /^[[:blank:]]/ in data ) - { - # Remove all but a single space at the beginning (this seems to follow - # the most common behavior). - data = sub(data, /^[[:blank:]]*/, " "); - if ( c$smtp$current_header == "MESSAGE-ID" ) - c$smtp$msg_id += data; - else if ( c$smtp$current_header == "RECEIVED" ) - c$smtp$first_received += data; - else if ( c$smtp$current_header == "IN-REPLY-TO" ) - c$smtp$in_reply_to += data; - else if ( c$smtp$current_header == "SUBJECCT" ) - c$smtp$subject += data; - else if ( c$smtp$current_header == "FROM" ) - c$smtp$from += data; - else if ( c$smtp$current_header == "REPLY-TO" ) - c$smtp$reply_to += data; - else if ( c$smtp$current_header == "USER-AGENT" ) - c$smtp$user_agent += data; - return; - } - # Once there isn't a line starting with a blank, we're not continuing a - # header anymore. - c$smtp$current_header = ""; - - local header_parts = split1(data, /:[[:blank:]]*/); - # TODO: do something in this case? This would definitely be odd. - # Header wrapping needs to be handled more elegantly. This will happen - # if the header value is wrapped immediately after the header key. - if ( |header_parts| != 2 ) - return; - - local header_key = to_upper(header_parts[1]); - c$smtp$current_header = header_key; - - local header_val = header_parts[2]; - - if ( header_key == "MESSAGE-ID" ) - c$smtp$msg_id = header_val; - - else if ( header_key == "RECEIVED" ) + else if ( h$name == "RECEIVED" ) { if ( c$smtp?$first_received ) c$smtp$second_received = c$smtp$first_received; - c$smtp$first_received = header_val; + c$smtp$first_received = h$value; } - - else if ( header_key == "IN-REPLY-TO" ) - c$smtp$in_reply_to = header_val; - - else if ( header_key == "DATE" ) - c$smtp$date = header_val; - - else if ( header_key == "FROM" ) - c$smtp$from = header_val; - - else if ( header_key == "TO" ) + + else if ( h$name == "IN-REPLY-TO" ) + c$smtp$in_reply_to = h$value; + + else if ( h$name == "SUBJECT" ) + c$smtp$subject = h$value; + + else if ( h$name == "FROM" ) + c$smtp$from = h$value; + + else if ( h$name == "REPLY-TO" ) + c$smtp$reply_to = h$value; + + else if ( h$name == "DATE" ) + c$smtp$date = h$value; + + else if ( h$name == "TO" ) { if ( ! c$smtp?$to ) c$smtp$to = set(); - add c$smtp$to[header_val]; + add c$smtp$to[h$value]; } - - else if ( header_key == "REPLY-TO" ) - c$smtp$reply_to = header_val; - - else if ( header_key == "SUBJECT" ) - c$smtp$subject = header_val; - else if ( header_key == "X-ORIGINATING-IP" ) + else if ( h$name == "X-ORIGINATING-IP" ) { - local addresses = find_ip_addresses(header_val); + local addresses = find_ip_addresses(h$name); if ( 1 in addresses ) c$smtp$x_originating_ip = to_addr(addresses[1]); } - else if ( header_key == "X-MAILER" || - header_key == "USER-AGENT" || - header_key == "X-USER-AGENT" ) - { - c$smtp$user_agent = header_val; - # Explicitly set the current header here because there are several - # headers bulked under this same key. - c$smtp$current_header = "USER-AGENT"; - } + else if ( h$name == "X-MAILER" || + h$name == "USER-AGENT" || + h$name == "X-USER-AGENT" ) + c$smtp$user_agent = h$value; } # This event handler builds the "Received From" path by reading the # headers in the mail -event smtp_data(c: connection, is_orig: bool, data: string) &priority=3 +event mime_one_header(c: connection, h: mime_header_rec) &priority=3 { # If we've decided that we're done watching the received headers for # whatever reason, we're done. Could be due to only watching until # local addresses are seen in the received from headers. - if ( c$smtp$current_header != "RECEIVED" || - ! c$smtp$process_received_from ) + if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from) return; - - local text_ip = find_address_in_smtp_header(data); + + local text_ip = find_address_in_smtp_header(h$value); if ( text_ip == "" ) return; local ip = to_addr(text_ip); - + if ( ! addr_matches_host(ip, mail_path_capture) && ! Site::is_private_addr(ip) ) { c$smtp$process_received_from = F; } - if ( c$smtp$path[|c$smtp$path|-1] != ip ) c$smtp$path[|c$smtp$path|] = ip; } - event connection_state_remove(c: connection) &priority=-5 { - if ( c?$smtp && ! c$smtp$done ) + if ( c?$smtp ) smtp_message(c); } diff --git a/policy/protocols/smtp/base/mime.bro b/policy/protocols/smtp/base/mime.bro new file mode 100644 index 0000000000..7ed7c67d51 --- /dev/null +++ b/policy/protocols/smtp/base/mime.bro @@ -0,0 +1,224 @@ +##! Analysis and logging for MIME entities found in SMTP sessions. + +@load utils/strings +@load utils/files +@load ./main + +module SMTP; + +export { + redef enum Notice::Type += { + ## Indicates that an MD5 sum was calculated for a MIME message. + MD5, + }; + + redef enum Log::ID += { SMTP_MIME }; + + type MimeInfo: record { + ## This is the timestamp of when the MIME content transfer began. + ts: time &log; + uid: string &log; + id: conn_id &log; + ## The filename seen in the Content-Disposition header. + filename: string &log &optional; + ## Track how many byte of the MIME encoded file have been seen. + content_len: count &log &default=0; + mime_type: string &log &optional; + ## The calculated MD5 sum for the MIME entity. + md5: string &log &optional; + ## Optionally calculate the file's MD5 sum. Must be set prior to the + ## first data chunk being see in an event. + calc_md5: bool &default=F; + ## This boolean value indicates if an MD5 sum is being calculated + ## for the current file transfer. + calculating_md5: bool &default=F; + ## Optionally write the file to disk. Must be set prior to first + ## data chunk being seen in an event. + extract_file: bool &default=F; + ## Store the file handle here for the file currently being extracted. + extraction_file: file &log &optional; + }; + + type MimeState: record { + ## Store a count of the number of files that have been transferred in + ## this conversation to create unique file names on disk. + num_extracted_files: count &default=0; + ## Track the number of MIME encoded files transferred during this session. + level: count &default=0; + }; + + ## Generate MD5 sums for these filetypes. + const generate_md5 = /application\/x-dosexec/ # Windows and DOS executables + | /application\/x-executable/ # *NIX executable binary + &redef; + + ## Pattern of file mime types to extract from MIME bodies. + const extract_file_types = /NO_DEFAULT/ &redef; + + ## The on-disk prefix for files to be extracted from MIME entity bodies. + const extraction_prefix = "smtp-mime-item" &redef; + + global log_mime: event(rec: MimeInfo); +} + +redef record connection += { + smtp_mime: MimeInfo &optional; + smtp_mime_state: MimeState &optional; +}; + +event bro_init() + { + Log::create_stream(SMTP_MIME, [$columns=MimeInfo, $ev=log_mime]); + } + +function new_mime_session(c: connection): MimeInfo + { + local info: MimeInfo; + + info$ts=network_time(); + info$uid=c$uid; + info$id=c$id; + return info; + } + +function set_session(c: connection, new_entity: bool) + { + if ( ! c?$smtp_mime_state ) + c$smtp_mime_state = []; + + if ( ! c?$smtp_mime || new_entity ) + c$smtp_mime = new_mime_session(c); + } + +event mime_begin_entity(c: connection) &priority=10 + { + if ( ! c?$smtp ) return; + set_session(c, T); + ++c$smtp_mime_state$level; + } + +# This has priority -10 because other handlers need to know the current +# content_len before it's updated by this handler. +event mime_segment_data(c: connection, length: count, data: string) &priority=-10 + { + if ( ! c?$smtp ) return; + c$smtp_mime$content_len = c$smtp_mime$content_len + length; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=7 + { + if ( ! c?$smtp ) return; + if ( c$smtp_mime$content_len == 0 ) + c$smtp_mime$mime_type = split1(identify_data(data, T), /;/)[1]; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=-5 + { + if ( ! c?$smtp_mime ) return; + + if ( c$smtp_mime$content_len == 0 ) + { + if ( generate_md5 in c$smtp_mime$mime_type ) + c$smtp_mime$calc_md5 = T; + + if ( c$smtp_mime$calc_md5 ) + { + c$smtp_mime$calculating_md5 = T; + md5_hash_init(c$id); + } + } + + if ( c$smtp_mime$calculating_md5 ) + md5_hash_update(c$id, data); +} + +## In the event of a content gap during the MIME transfer, detect the state for +## the MD5 sum calculation and stop calculating the MD5 since it would be +## incorrect anyway. +event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5 + { + if ( is_orig || ! c?$smtp_mime ) return; + + if ( c$smtp_mime$calculating_md5 ) + { + c$smtp_mime$calculating_md5 = F; + md5_hash_finish(c$id); + } + } + +event mime_end_entity(c: connection) &priority=-3 + { + # TODO: this check is only due to a bug in mime_end_entity that + # causes the event to be generated twice for the same real event. + if ( ! c?$smtp_mime ) + return; + + if ( c$smtp_mime$calculating_md5 ) + { + c$smtp_mime$md5 = md5_hash_finish(c$id); + + NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h), + $sub=c$smtp_mime$md5, $conn=c]); + } + } + +event mime_one_header(c: connection, h: mime_header_rec) + { + if ( ! c?$smtp ) return; + if ( h$name == "CONTENT-DISPOSITION" && + /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) + c$smtp_mime$filename = sub(h$value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, ""); + } + +event mime_end_entity(c: connection) &priority=-5 + { + if ( ! c?$smtp ) return; + # This check and the delete below are just to cope with a bug where + # mime_end_entity can be generated multiple times for the same event. + if ( ! c?$smtp_mime ) + return; + + # Don't log anything if there wasn't any content. + if ( c$smtp_mime$content_len > 0 ) + Log::write(SMTP_MIME, c$smtp_mime); + + delete c$smtp_mime; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=5 + { + if ( ! c?$smtp ) return; + if ( extract_file_types in c$smtp_mime$mime_type ) + c$smtp_mime$extract_file = T; + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=3 + { + if ( ! c?$smtp ) return; + if ( c$smtp_mime$extract_file && c$smtp_mime$content_len == 0 ) + { + local suffix = fmt("%d.dat", ++c$smtp_mime_state$num_extracted_files); + local fname = generate_extraction_filename(extraction_prefix, c, suffix); + c$smtp_mime$extraction_file = open(fname); + enable_raw_output(c$smtp_mime$extraction_file); + } + } + +event mime_segment_data(c: connection, length: count, data: string) &priority=-5 + { + if ( ! c?$smtp ) return; + if ( c$smtp_mime$extract_file && c$smtp_mime?$extraction_file ) + print c$smtp_mime$extraction_file, data; + } + +event mime_end_entity(c: connection) &priority=-3 + { + if ( ! c?$smtp ) return; + # TODO: this check is only due to a bug in mime_end_entity that + # causes the event to be generated twice for the same real event. + if ( ! c?$smtp_mime ) + return; + + if ( c$smtp_mime?$extraction_file ) + close(c$smtp_mime$extraction_file); + } diff --git a/policy/protocols/smtp/base/software.bro b/policy/protocols/smtp/base/software.bro index 1773a87201..32261bc264 100644 --- a/policy/protocols/smtp/base/software.bro +++ b/policy/protocols/smtp/base/software.bro @@ -45,10 +45,10 @@ export { | /ZimbraWebClient/ &redef; } -event smtp_data(c: connection, is_orig: bool, data: string) &priority=4 +event mime_one_header(c: connection, h: mime_header_rec) &priority=4 { - if ( c$smtp$current_header == "USER-AGENT" && - webmail_user_agents in c$smtp$user_agent ) + if ( ! c?$smtp ) return; + if ( h$name == "USER-AGENT" && webmail_user_agents in c$smtp$user_agent ) c$smtp$is_webmail = T; } diff --git a/testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log new file mode 100644 index 0000000000..290a3c3a5a --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log @@ -0,0 +1,2 @@ +# ts uid id.orig_h id.orig_p id.resp_h id.resp_p helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply files path user_agent +1254722768.219663 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un "NEWS.txt" 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat new file mode 100644 index 0000000000..e91ec57ed2 --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat @@ -0,0 +1,14 @@ +Hello + + + +I send u smtp pcap file + +Find the attachment + + + +GPS + + + diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat new file mode 100644 index 0000000000..09479e05cb --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat @@ -0,0 +1,275 @@ +Version 4.9.9.1 +* Many bug fixes +* Improved editor + +Version 4.9.9.0 +* Support for latest Mingw compiler system builds +* Bug fixes + +Version 4.9.8.9 +* New code tooltip display +* Improved Indent/Unindent and Remove Comment +* Improved automatic indent +* Added support for the "interface" keyword +* WebUpdate should now report installation problems from PackMan +* New splash screen and association icons +* Improved installer +* Many bug fixes + +Version 4.9.8.7 +* Added support for GCC > 3.2 +* Debug variables are now resent during next debug session +* Watched Variables not in correct context are now kept and updated when it is needed +* Added new compiler/linker options: 20 + - Strip executable + - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20 + k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2) + - Enable use of processor specific built-in function +s (mmmx, sse, sse2, pni, 3dnow) +* "Default" button in Compiler Options is back +* Error messages parsing improved +* Bug fixes + +Version 4.9.8.5 +* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value") +* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features. +* Many bug fixes + +Version 4.9.8.4 +* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup +* Improved code completion cache +* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP +* Big speed up in function parameters listing while editing +* Bug fixes + +Version 4.9.8.3 +* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20 + include files can now be generated. +* Improved WebUpdate module +* Many bug fixes + +Version +4.9.8.2 +* New debug feature for DLLs: attach to a running process +* New project option: Use custom Makefile. 20 +* New WebUpdater module. +* Allow user to specify an alternate configuration file in Environment Options 20 + (still can be overriden by using "-c" command line parameter). +* Lots of bug fixes. + +Version 4.9.8.1 +* When creating a DLL, the created static lib respects now the project-defined output directory + +Version 4.9.8.0 +* Changed position of compiler/linker parameters in Project Options. +* Improved help file +* Bug fixes + +Version 4.9.7.9 +* Resource errors are now reported in the Resource sheet +* Many bug fixes + +Version 4.9.7.8 +* Made whole bottom report control floating instead of only debug output. +* Many bug fixes + +Version 4.9.7.7 +* Printing settings are now saved +* New environment options : "watch variable under mouse" and "Report watch errors" +* Bug fixes + +Version 4.9.7.6 +* Debug variable browser +* Added possibility to include in a Template the Project's directorie +s (include, libs and ressources) +* Changed tint of Class browser pictures colors to match the New Look style +* Bug fixes + +Version 4.9.7.5 +* Bug fixes + +Version 4.9.7.4 +* When compiling with debugging symbols, an extra definition is passed to the + compiler: -D__DEBUG__ +* Each project creates a _private.h file containing version + information definitions +* When compiling the current file only, no dependency checks are performed +* ~300% Speed-up in class parser +* Added "External programs" in Tools/Environment Options (for units "Open with") +* Added "Open with" in project units context menu +* Added "Classes" toolbar +* Fixed pre-compilation dependency checks to work correctly +* Added new file menu entry: Save Project As +* Bug-fix for double quotes in devcpp.cfg file read by vUpdate +* Other bug fixes + +Version 4.9.7.3 +* When adding debugging symbols on request, remove "-s" option from linker +* Compiling progress window +* Environment options : "Show progress window" and "Auto-cl +ose progress window" +* Bug fixes + +Version 4.9.7.2 +* Bug fixes + +Version 4.9.7.1 +* "Build priority" per-unit +* "Include file in linking process" per-unit +* New feature: compile current file only +* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744) +* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744) +* Necessary UI changes in Project Options +* Added display of project filename, project output and a summary of the project files in Project Options General tab. +* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0 +* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings) +* Resource files are treated as ordinary files now +* Updates in "Project Options/Files" code +* MSVC import now creates the folders structure of the original VC project +* Bug fixes + +Version 4.9.7.0 +* Allow customizing of per-unit compile command in projects +* Added two new macros: and < +DATETIME> +* Added support for macros in the "default source code" (Tools/Editor Options/Code) +* Separated layout info from project file. It is now kept in a different file + (the same filename as the project's but with extension ".layout"). If you + have your project under CVS control, you ''ll know why this had to happen... +* Compiler settings per-project +* Compiler set per-project +* Implemented new compiler settings framework +* "Compile as C++" per-unit +* "Include file in compilation process" per-unit +* Project version info (creates the relevant VERSIONINFO struct in the private + resource) +* Support XP Themes (creates the CommonControls 6.0 manifest file and includes + it in the private resource) +* Added CVS "login" and "logout" commands +* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows. +* Added "Add Library" button in Project Options +* Bug fixes + +Version 4.9.6.9 +* Implemented search in help files for the word at cursor (context sensitive + help) +* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2) +* Added "Files" tab in CVS form to allow selection of more than one file for + the requested CVS action + 20 +Version 4.9.6.8 +* support for DLL application hosting, for debugging and executing DLLs under Dev-C++. +* New class browser option: "Show inherited members" +* Added support for the '::' member access operator in code-completion +* Added *working* function arguments hint +* Added bracket highlighting. When the caret is on a bracket, that bracket and + its counterpart are highlighted +* Nested folders in project view + +Version 4.9.6.7 +* XP Theme support +* Added CVS commands "Add" and "Remove" +* Added configuration option for "Templates Directory" in "Environment Options" +* Code-completion updates +* Bug fixes + +Version 4.9.6.6 +* Editor colors are initialized properly on Dev-C++ first-run +* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVari +able wizards +* Added file's date/time stamp in File/Properties window +* Current windows listing in Window menu +* Bug fixes + +Version 4.9.6.5 +* CVS support +* Window list (in Window menu) +* bug fixes + +version 4.9.6.4 +* added ENTER key for opening file in project browser, DEL to delete from the project. +* bug fixes + +version 4.9.6.3 +* Bug fixes + +version 4.9.6.2 +* Bug fixes + +version 4.9.6.1 +* New "Abort compilation" button +* Bug fixes +* Now checks for vRoach existance when sending a crash report + +Version 4.9.5.5 +* New option in Editor Options: Show editor hints. User can disable the hints + displayed in the editor when the mouse moves over a word. Since this was the + cause of many errors (although it should be fixed by now), we are giving the + user the option to disable this feature. +* New option in Editor Options (code-completion): Use code-completion cache. + Well, it adds caching to code-completion. Depending on the cache size, + the program may take a bit longer to start-up, bu +t provides very fast + code-completion and the user has all the commands (belonging to the files + he added in the cache) at his fingertips. If, for example, the user adds + "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of + wxWindows! You get the picture... +* Removed "Only show classes from current file" option in class browser settings. + It used to be a checkbox, allowing only two states (on or off), but there is + a third relevant option now: "Project classes" so it didn't fit the purpose... + The user can define this in the class browser's context menu under "View mode". +* Fixed the dreaded "Clock skew detected" compiler warning! +* Fixed many class browser bugs, including some that had to do with class folders. + +Version 4.9.5.4 +* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data) + +Version 4.9.5.3 +* Added ExceptionsAnalyzer. If the devcpp.map file is in + the devcpp.exe directory + then we even get a stack trace in the bug report! +* Added new WebUpdate module (inactive temporarily). +* Added new code for code-completion caching of files (disabled - work in progress). + +Version 4.9.5.2 +* Added new option in class-browser: Use colors + (available when right-clicking the class-browser + and selecting "View mode"). +* Dev-C++ now traps access violation of your programs (and of itself too ;) + +Version 4.9.5.1 +* Implemented the "File/Export/Project to HTML" function. +* Added "Tip of the day" system. +* When running a source file in explorer, don't spawn new instance. + Instead open the file in an already launched Dev-C++. +* Class-parser speed-up (50% to 85% improvement timed!!!) +* Many code-completion updates. Now takes into account context, + class inheritance and visibility (shows items only from files + #included directly or indirectly)! +* Caching of result set of code-completion for speed-up. +* New option "Execution/Parameters" (and "Debug/Param +eters"). + +Version 4.9.5.0 (5.0 beta 5): +* CPU Window (still in development) +* ToDo list +* Backtrace in debugging +* Run to cursor +* Folders in Project and Class Browser +* Send custom commands to GDB +* Makefile can now be customized. +* Modified the behaviour of the -c param : 20 + -c +* Saving of custom syntax parameter group +* Possibility of changing compilers and tools filename. +* Many bug fixes + + +Version 4.9.4.1 (5.0 beta 4.1): + +* back to gcc 2.95.3 +* Profiling support +* new update/packages checker (vUpdate) +* Lots of bugfixes + + diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log new file mode 100644 index 0000000000..9a3d5ac0f9 --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_mime.log @@ -0,0 +1,4 @@ +# ts uid id.orig_h id.orig_p id.resp_h id.resp_p filename content_len mime_type md5 extraction_file +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 79 FAKE_MIME - smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 1918 FAKE_MIME - - +1254722770.692804 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 "NEWS.txt" 10823 FAKE_MIME - smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat diff --git a/testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log b/testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log new file mode 100644 index 0000000000..cd2d543b51 --- /dev/null +++ b/testing/btest/Baseline/policy.protocols.smtp.mime/smtp_mime.log @@ -0,0 +1,4 @@ +# ts uid id.orig_h id.orig_p id.resp_h id.resp_p filename content_len mime_type md5 extraction_file +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - +1254722770.692743 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 - 1918 FAKE_MIME - - +1254722770.692804 UWkUyAuUGXf 10.10.1.4 1470 74.53.140.153 25 "NEWS.txt" 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - diff --git a/testing/btest/Traces/smtp.trace b/testing/btest/Traces/smtp.trace new file mode 100644 index 0000000000000000000000000000000000000000..931b43b3b878dc559b7423df26363bfe75c6a512 GIT binary patch literal 27850 zcmeHw3y@padEO-{>aq6Pq^EIdM;)Do#Arz^u)w}Zf{zdjU?0>zpj|8}$Fsy#Grk7^ ze09Qq&xUc98NAQ>?m^{uN*%g#?yCn4BRRLQtY%Xh z+{2%^-Z%wFeDzRrc<9@tWa8;gN@grj^0TP}?|l9X5Aw0!`SPLXiQ1D-TyOk)eCDeM z*3&~T3aneVg@sj#yc4T3x$~;bS;e9Qyw!UmV`HJn6kZeKp~&R1dqI3Ww>p1!QcaFc zj2%B74(qQ{^Qq**{L=i=to|HZOV6p)LTr9XEiBK@FL@89V=L+O!m4~ybIApdG#~yK z*BjHA312;wo;ZAqQ2GzLDE-+_zmj=1b?Eiq|K);V$avmJB+$#>88qeqjISO7MTcGz z0)4EDK(B(>e@M2?LX(Y;PpI>EE)}z-$wKDB+LT_zTVEbD(g4X<52iDRj|is!ybIGm z{PZhX1DJj$@EeA4_xSm(?Ci`|Zg!?|$J}%wH+Qa*8DF`)P%7332fhg03*Xq+5BKi~ z?jP;MJ!=5>7m54XNsas3?9Aotcy1$Gx&ycq?kB(52De*HM-PVt_uuKl{YBvZ4&eT= z^wWlsojYGQOEdNziR9K|?^t(Bm!N)M0 z$*@|!T|=9G>}|%qK$))|yq6Cj7rg&rC*J9I1MlO&yYb}}!&r%@(ki_StgGhz7ju?V zgjO80%V)RDg5yAoYSjM(3;gwOUSmMv%BzRzr-oh@)Vr8IdmO08i24g}*Qm!p_zLvn z-~eGh_rf(Tt<}vmp~I(T0#9{KAchIdU;_4;7lBPI9n-Krb7;VTWtUeEe)kQ1SMaNL zk#+|7%@V)U889}?K(bu3s;XqUuDNcFD65>a%j>FP*9s~S3MlZ7sfu>lQkgAf*IcWZ z*P~zeYXio2_Z#x+UW)x+KfKJDeE%=HW-^PJJdir@dgLn~FpQb1Q;Mpgf)3AAHF4Ps!GRkxPQ9Ent6Qf7LHs#=amm`OrEteZjL6r-#c(7Zo0yyq zhr%Z&r%sH|gad;EcLo<>L9HrgVP~t3>*Q-{IiI()mRhdYijH$hMaDROx=^*uTyVae zvo@nDmaWXJTcs1z8~5LTcQRqEL~bt}w{I(~ZJl?*F=`qg-vzV0UdPHuvGT#XJ2u#%`8nv)J*Eqikd0k zc0tX}#8YaneAhX3Zuyw2Zm*Z7)a_ff3u<8|JFgb*U#p#roXoCI7hS$FWhbza|y#V2BSc|YYk=5cb>aSLYT;zs1;v6JtM z$718hW+qPrBgbJ@Cnn>^g2yK(!ogTL)V{?nKh4tx5Y<41cve%UY7vWz-%`T=-t~I8YVl#Kd)?zV` zSE#7zXXm2ojiEL;z;%SHg@0y?hkW5wc3F}0RNkqU%o^OP!2$Bp8|0?ens@W4S%eBG z!%%EkSK-tMIdyVdPTc{Uou25%rUr2strShWOcb&Ov+7#4(}B7hG~KLiLlk_DP1F0# zTh(B)oOP(IqUu=2u5o!-fve{{$&@-@NK8@%CEb8SPHICP;_t3*dfy%~m@Gh~=Es-GK$;hq?cjEy>__8GT5 z-DosIjq#9ET^~)ajIJc(K~ixd9Dr{^l0sqP-KoW7I;Q5*=~OUz$J+dP0%-CYw0S_q zmzUDXr8F-Ho65S9#6FvjCz^apfF-MDs?A9E3e**Gor;<0-ecRkZkKC>R7&j~-Ps0HP9B-Y(np>s)no6c^6s@pUk7X2B~r zywCekvWr{MK-?^4sX%~~NqW~`WH$F+iy=$M*NlWH8V2{nbE_>_B5MqrHFzRR0F6$JIbNWg{i2)b6)&IdSKGYa44(ui_F77jg! z1+Vfe8Qpb)vILYx1jV(j=!p{|&`!~*Ml;2_=u7fMFlS{QtTP<~*m;3r=>axuhUtip zd&Zb?5EwS-8e*pX$Ev_dDqejw86wG`TZkVy)(N^s=L8dUow8Y^2v)!fO#=2O79eG1 z+6|~qr@={cej!pVUZ{X4O*lQorS(+Umr;XHG#!;vRL%iNcMj zAU={wxN0Mc=}}T4jI{t(&d!z0O@0vxk4**Cs6gh^r-t4`mpOkgMx&TsWrU)+<!AX0%4R13}Zc7#6dL4peAqyMjHWVX9m?)!yu#;u%u19$J_2E5VHws95S=%YUyqVPMm13ZfL(6kuk zT6%A3gXHbc;H1!qZ0ptR6o9ljT@u!GI%#6z0!zsYtD)Lvt-puX>(lrqHen-`2n{Kr zRLdI^tpsnaXOO`GFYrQ&^924vg&c)-D{mp?xDE$z(?Sq)RLz$vRc8bCloISzIq}Zt z5HrC<)hm+6ghve-sab9faaMV~0b^aNFz)iRo|(E`%n9J>)+n7IkB3eWmU0VDy_iF)x1p+*%-t@#HM5A>V+9n5 z29LAksyu>ol`^xJKw5uVoQGRbZnkP!NDBaGCql)kIGv>TAX}h*?-F$r52k9n zNzKON>Wmr>jq!5A;u>$5aA<%ff~2bD%>1%_y}h zn8&G*sBZN17`RQTAY7s64E=>h- z7ycio=D*Ni^RI~J|9+?Dx3a$G|JK8XF@jL5P~>ie{u!)g>?*t>@6$x{-DX8*x-$7O}G%k%90UcBkE(#@-~^y%;C`0LIS^H?B$o9$lPxk38Q!0)RxzrV zR&y0{K`wdI_S6z@>7z*|d0k65r5Xy-wKH}R9w_%;N(g)Nc!w(+CapNOXw0Tq_+o&W zh!DVmURT3ad%XZ2$7anXozrVj9$F7knvvY2AOsdQe1RR1*(bDQC9I8L{I=WFj9rDM zOxq=>tL1XuMgmyt0PP!&ko;9wWzDjPXBL@NY$O4pj8$t;chrR#AP{9xd<~VUIt@If z*OU5_D!6EgR?1sutzNZUq&%*!NGAHyPV>PQhQSJEi**c?vnl^hbxTB!R2BUVblXtK zLj0)HRR))lYJ*1))`dpW6v+!kph&B=dW8ry?T^6f2E;pBI=NvNi$W+ePaPJ+X_SkO zN&OD^Df1FCLS=ubd#IG*>bj(76Lm-r7SPA0rC|7A&f3UUDj|FsnvTV9UrTZHX?vYA zfgvzpn>3J95xK&MxNsER8kwh2VA_O+6rbeboo(PauP-;Tg+V5()W|BZ4{fZHv(2Kj z4!kwByD3DT#sp!QgwH`)W>0gTit?9^`J5J?VQJQNxNYTxRho;?qBUfv&Xmg4`A94d zt+_TO9uEqSBo-FjD3+h8Bl3i)s`WB#`8w?<6%OI^QG8@R+A{=EwFWWB)?HY0xGa|t z#uu%SOiAC~L=&1r@RV7^7$u5GajZ55_Vae#q?Ife)-+k(u&Yj)?gVM+X(^9{!&qAo zmof^v!SkqAZTJ>U^K0o01hdrqw?8&uEbTSq)wRp;%-*xfcVRy#s zy!`!NN*a0(;DKv@u;U)UP2btw19&s<0g$^s0PcDLyGVTX>>_>X@T&L~KkD==(#>6@ z)#xq^cNb}!4K9YdySqq|rrq5|`hRN|i5h?QV}1RJcZ)wU(()(vf3E3IU}p*b z#Ev^lU-_loouxPP&JwxncR${jyMA7{>$Vnm-E1|v3)yRM*N)liEfj3+X0LZn%K!hK zy(V`>AMeXuHwt$hZE@EZ&NaCU88UF!jv2C}ukL2Z-pm=YKLvMv|HFG)wSnD*pTk4n zlX|{Sb=C7d@W8Woy_Pxoq@L`{L_{ z(N$HlTdSdVYc*QM86(J=?$&B-Ul+1ltFc?FAr&J3$7(gG@qeMe#$OYS|HDp=fAQnK z#{cQdhS6Q|vFq)=dA;4v`qSN7jon%eX?BpBo8zI~T8;nPwHnm;UpsnTzj)yli5Gse zH(t2-@zjCfeV_Uvs zeeBkK?ACoqE#7Y32P+qvRX)1JX1DGG^(p_G>prOQr*7-3@h^zR@2cIn|0ll2|EnJv z#?`7v0$kh@>yl6oi6S=nZ@X(krMCA1%PytliUl^NLh;Rr z)CBouSf~wRX)HQCP}0is7yxE{D=WEB_TtsxphE*)GpKh${e1-`_|4L(-eILQI9@=_ zJZ|@klZh*tl~uuheP?q@QlORcihaB~8<}OZ0;ymXb^BT0`l(~OhIqwtoq9Eknk!Zj zw%56`Vi!oExQDZA-iK~YOR9pq3!ytHdMtEYS3aTYtx<4L)TMiA{8C_4Xw|AtQRPv6 zCY4_-NJWfQn?x|A7+>0343P?FxK%JNtUjr`p;#eNwguju;AuG&k#@(e3r zs21pVvSc)QaU*oZS~bDym{< zrTP)9xywgTkAqd*AdYQT|&xxVB?RWNb!Vj?-cHhb|RQBAUh&M%YG z5)BfmQ{Gn>S^t0LBKog3K<7|FcW{zV(J4yYFbYc7bdk4$TjUHA@L{#e3d*3apwwlo zEcDd{@0xuA$xUglEpn3UD6Mn@l-@L&1KiQ}0zJQmc$jtHb+q?MfoEVD$jAcdLR=4M z>stcz$)}$paaaM5vOk=F;ju9_{)LD8#tYwR5#{uAs_8OGna^Wawr z%L1%E@xq;AJ-T-4D^G~^IP`iXc)&1JK)BojJ=iIcF{MjjhVAm0kR{z|w%3)=yCmQ& z^fp`qUrF(PQQ9jsaaf2iD@p>OtG!m;LJK>zj4z=oCJX@~2*U74L1)gXvua$rR7971 zHM7|~jT5f0xt^tw8@i2xbX^7u@tm7x(@D3kl~Pz?J2NMTD)fjDE3vEAd+X>s8NnOr z{|E#@5v}`l69za>w-a#CcvC2aouEco5s!jlk)`dm=V}4Qum8V8=M13h2Fd_>6skcoDiIFMdnMR@#VRLCsRlAEqwTz#s?Mvz zjrt0uSB439)uLslp=7nXuY3G<3{BQ}@AnWTzbcpVKI~dBkXT=@QRe2U$%$hV6QZ#@ zNAQgTd&1+FXEw-5vqp#(ZPX~PR208m6 zZAB1msF(y>CDt(l%H*NgE7EiU60>dV7>}$#2{w<(KsT!6!YMiqW27 z78RatEdxjHBUeH?T@BduK+7M_cp$59eqfjpO< z&dM;+aUnm#3=||Zx(SXRK=^p*0@8jFOs!R>|AFIjGy;Jm7)!br(CbTh1sWf`uQtU| zd$95It5_j%xkq?i-Fhoxs!QaaMuGi3;O_$L4)`4tM}sn$J`@u9({_OUiM~J0wfE5D znzi2`e9?9dK?!>NFlk5(8Nip|7O*^fKxkkI3a24R*h3;fpLQzc9u=;E>2@HD*{jA= zo*a2@8KI;IE$&{ROT$9~ZQ=u>fpls*a`$+~6{s(wZe?wm2R!XBFxx#ZqJ4e*w#MKO- zrNxF~qMqe@gh|aPSR~J%JcdLMkM4GAxdqzpqQ#H>Gon1r%#mgr71}g_7yNBkyj)Rt zV2x(5rk~bLQW~WIey9Z;+5pAY-niyt;mJ~8eKVy)(srJ4Ewc)*9auJ_THUTl3bEys zol$VE*>^YWyQ4rIO`2@r&!&WCGAAD<$J)*}d_Hck|8=+}}% zqXC-?$4x5-t_U6=lW_8$Ge^e*kWA?>#5GBcx3?cp=|H@_dlzX91qp|N)3Q@=r1@$@ z`={gvY2+hk>Gm$_d^REx1+7oK_AgNobe1|B>$FdaXj=S1yA1D@=Taa}C}m)r#M{$E z0b11fH~+i0_S;{0RpNzD_QnhM|0H!Fao~gBWxVkFuN?^xRsyj4#0z(e_2{aOdtdX+ zm>XWh37<5qjEzX=TcgoiJ;Xs}>KJ+iaQSc@%Sh890fg=!6dNYi=O}d=2zRvE{fXdi zy-r;SRg37mbt{A>d&#Qd+x0?`&69X6q{N*k23hzIVo$&sCPH<&nRzL&J{H7(8q1{> z2)fj802o3+c&56?xCz)#e<0P8Yq|=U1+=GlbM%9{lT=fB56;(dejj)XeQjIB`n!sH zpqVLQl+m`>zW1cL&JR35$AF*Ra8<(Zkd)Xp(C%VkuAUYVn6%1mlt*(RGu@*4cJyeb zbRs6HO}v-npnE2V+~Cle%b*L0fdz(GgWi6dqx2RA(R$aKcFx({LJy!mwuOdGFD>kTVAuS-}-^%jU_s1w>84&TOgjezD#I^;T;Llv^I;xO%GZGv)U8F^L- zGI&}Tr;d(P*qwqG_H|3%WJs2R2zbH*DtWx?NdnRh47v5m6QWk!5L-WS1{6#O;x%be zC&@rQY8^0`;`cl!0=y1@yy!hIxbd?jJn{bqxQqRoK$ONd#B&od2XPkxu%Ufbg3Ovo zk&5Bs`50a@c^GY@I0RxH1{}V&J`Mt|1sO$B9B@C6pwAwT5JV90*6j^a1sD({0BRNN zO`fdKoDlp~3DBwWFZS2??-7mvO6QL9Lr?ngLV3+Fs(KrwAEoJDbSRu7UXN|(x15IT zE`c`{8n25uc?y#U|Fe0R=PF=+PPf?zLD4`OgeRbNxCDBOiU$^r5E8!v*E6jLClVez zdU|OM02?PA(7yvzuo#X%s^c63#tgW91AVqQV#s2!3I9eGstaFQc>)(|<1j`#BN_o6 z(7~t8SRBH`ocCgm6k6dWeAL42v6A3@gl-vQ3M6ZR8_p&HTh8aThhc3(j}?(IXbuZ_ zBoNUHc&aN@2#mmYLzgworH>M{PAoM)#E?f13wzVp9Bpj&+zp5hP7hV8v8h#+JEZPH z*aL8ylseJvg*f^=6!_80+<5?_dJYX}Cn~2sd)v&IVZNPS561LEBw74<99qrnG!_O- z4^gL9$T&DAL;O9`Q+7HS9CKp{qYcyf!(krzJmi9F2$-M}4U|yF1oAk_0{X%WTRhQ# zAi>*$Nf0Y8Xfa`cqo=?hIXhRrl@W)YIzX~)6p1*Qy<+I1S6@_yY+Z2}NkwppI%Wxr zls$F>bihOcx;T#<$dcQ1^op~^We))u=_kE}2yp65noerNw1)h9 zRV}4Qcw~84%=4WoeX2xYXCQmoBEPhfDE+3pF9lT6({1NyiJY& zxkH2f;)NebyztL@k+>N zyVI0h&ot$5$PPV!QWR7Ym5rEzQ|(?p@jhC!Z<%;u+@R+M`BsMeaa@^Ua|37YiCGtO zEvK#MY$XMvDX{+Ye1<&dMVGjxO;Ys=_)adfL0I1k6s`_*Hf+b+76Ik`tdi_30S&;! zxB8rB=r~Cno!o{?#ffSUzKLQ$eAuL%*GE@Y^cEE5TzYXK(2STwk!T79(m2fs;|WKb zNHOT&5g>O+q~srq0Y=pu+P+(fLjgfMtBO7Xhk6hh;|30k0hh|5QSfH~M5%dy{~2~z zM)9AAA7vJ)!g#hF+<2bsJw5p7IJvSQCbIzWNofK&v<}z!vwC%fe$Iy(4^vY0LlsQU_0t z)=dl==O)~AvzL3ydAf~J593;`3?QN+*m2Ks0sWExlEf)g0zP#MDm`8~>IEba;?=UbQT~S84GD#n@L=c;e%OS z(xRHjdu7$BHcQhoOp-*$4x~62SRaj(yM;~c%~n~ATCY0#?4VTJaZg-NB%H+2Yo)k2 zVeXpJm(yE4varIs)c?(>uT#BEgd%OLg-={B5`za1k;+JrDQ3$;V#PpSfESCdbE>?R zGODFv6uUh7BqDL5V6lh2?NM3awgR6fI@=sp$g>Bj@xK9$zX}V$SNBOEM)L5z)cCJ$ zb!z-auJ}93-wYbYc1Nh|BM(6w4l!HpYoPu@3{>~;B0z)P#&YVw-OpGr5VJpye^-)Y z&KLW_`b&ZJM6a=X-%nEq@`uvjAgq5p_wwAAfY)n-`}!Hq9}290+6(KUCsPOBTZ#V{ z!us~+{+9^r-TA(-UK3c4_rm(%YU;q>{iCt35Y`W$`CgE)PCeTf){g|%CwgH$awT=( zLu%+x2y5=;^-mJkw?E$()?W#%kN3iQ^hSa8uL*1LS0DX#!us-a`-~NQ=Bw*SqWg@* z(2oVy2eyTE=tJtEuNmI&lu&l=bYu~cuoHxvIi6S#u9XJ|IKo@-n+>J_oUa~c96o%X zFylYyWX4Bt1T%gFzue&Ge)|c-_zIvr{p4F(>)QLv9xQ?Kp>3fYdj042f!}Z*b$W1s zus-oyZ|R3oUSK`i1uGyJ9Rx-nHH<$4l%GDf$H-$!eDzT7&Y_&J{m}Y za;IiMxv_}a>DLdjkbv3(O0bMT0!C$UfTOL12Mpw=<#k2=7<+&574x9{O0Acs-f!U7b-j`UMt=*hE5^h4`^A$#e{k=E z-{5!n^lwoNlUz!kn@Pr(5{YCoF`Jl4%p|9i(}`H(ToQlcF+ErY0w2vG9pR!q~rO-+_I`+xM*fmhnFUQ%`SL literal 0 HcmV?d00001 diff --git a/testing/btest/policy/protocols/smtp/basic.test b/testing/btest/policy/protocols/smtp/basic.test new file mode 100644 index 0000000000..2717df0f49 --- /dev/null +++ b/testing/btest/policy/protocols/smtp/basic.test @@ -0,0 +1,4 @@ +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT +# @TEST-EXEC: btest-diff smtp.log + +@load protocols/smtp/base/main diff --git a/testing/btest/policy/protocols/smtp/mime-extract.test b/testing/btest/policy/protocols/smtp/mime-extract.test new file mode 100644 index 0000000000..d75ddbdc74 --- /dev/null +++ b/testing/btest/policy/protocols/smtp/mime-extract.test @@ -0,0 +1,26 @@ +# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT +# @TEST-EXEC: btest-diff smtp_mime.log +# @TEST-EXEC: btest-diff smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_1.dat +# @TEST-EXEC: btest-diff smtp-mime-item_10.10.1.4:1470-74.53.140.153:25_2.dat +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test" +# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat +# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat + +@load protocols/smtp/base/main +@load protocols/smtp/base/mime + +redef SMTP::extract_file_types=/text\/plain/; + +event bro_init() + { + Log::remove_default_filter(SMTP::SMTP_MIME); + Log::add_filter(SMTP::SMTP_MIME, [$name="normalized-mime-types", + $pred=function(rec: SMTP::MimeInfo): bool + { + if ( rec?$mime_type ) + rec$mime_type = "FAKE_MIME"; + return T; + } + ]); + } diff --git a/testing/btest/policy/protocols/smtp/mime.test b/testing/btest/policy/protocols/smtp/mime.test new file mode 100644 index 0000000000..41188d5eaa --- /dev/null +++ b/testing/btest/policy/protocols/smtp/mime.test @@ -0,0 +1,24 @@ +# Checks logging of mime types and md5 calculation. Mime type in the log +# is normalized to prevent sensitivity to libmagic version. + +# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h +# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT +# @TEST-EXEC: btest-diff smtp_mime.log + +@load protocols/smtp/base/main +@load protocols/smtp/base/mime + +redef SMTP::generate_md5=/text\/plain/; + +event bro_init() + { + Log::remove_default_filter(SMTP::SMTP_MIME); + Log::add_filter(SMTP::SMTP_MIME, [$name="normalized-mime-types", + $pred=function(rec: SMTP::MimeInfo): bool + { + if ( rec?$mime_type ) + rec$mime_type = "FAKE_MIME"; + return T; + } + ]); + } From 4b741293b16af992a59711d891243f835d028c1c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 29 Jul 2011 15:20:35 -0500 Subject: [PATCH 02/25] Make the doc.coverage test happy. --- doc/scripts/DocSourcesList.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index a26b9dcedd..8fa4c78c21 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -97,6 +97,7 @@ rest_target(${psd} protocols/mime/file-hash.bro) rest_target(${psd} protocols/mime/file-ident.bro) rest_target(${psd} protocols/rpc/base.bro) rest_target(${psd} protocols/smtp/base/main.bro) +rest_target(${psd} protocols/smtp/base/mime.bro) rest_target(${psd} protocols/smtp/base/software.bro) rest_target(${psd} protocols/smtp/detect-suspicious-orig.bro) rest_target(${psd} protocols/ssh/base.bro) From 4ac6d0ae2e4ffa9397bff62f42483f13dac4bb6f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 8 Aug 2011 19:50:45 -0500 Subject: [PATCH 03/25] Fixes for script auto-documentation. - Fixing the parts of the `make restdoc` and `make doc` process that were broken by the last Bro script re-organization - Generated documentation for Bro scripts derived from BiFs now use the original BiF source file as the "original source file" link - Renaming of the internal POLICYDEST definition and other misc places that refer to "policy" scripts; that terminology doesn't make total sense now - Added a documentation blacklist reminder test that will fail if there's scripts that are blacklisted from being documentated because they're still in progress - Some minor Bro script changes to fix small @load dependency errors Addresses #543 --- CMakeLists.txt | 14 ++-- bro-path-dev.in | 10 +-- configure | 14 ++-- doc/scripts/CMakeLists.txt | 10 ++- doc/scripts/DocSourcesList.cmake | 19 ++--- doc/scripts/README | 2 +- doc/scripts/genDocSourcesList.sh | 33 ++++++-- doc/scripts/group_index_generator.py | 7 +- doc/scripts/source/index.rst | 2 +- doc/scripts/source/internal.rst | 4 +- doc/scripts/source/packages.rst | 14 ++-- doc/scripts/source/policy/index.rst | 6 -- doc/scripts/source/scripts/index.rst | 6 ++ scripts/CMakeLists.txt | 5 +- .../frameworks/cluster/setup-connections.bro | 2 + .../frameworks/communication/__load__.bro | 2 +- .../base/frameworks/metrics/conn-example.bro | 4 +- .../base/frameworks/metrics/http-example.bro | 4 +- .../frameworks/notice/actions/email_admin.bro | 3 +- .../notice/extend-email/hostnames.bro | 4 +- scripts/base/frameworks/notice/weird.bro | 1 - scripts/base/protocols/mime/base.bro | 2 +- scripts/base/protocols/mime/file-extract.bro | 6 +- scripts/base/protocols/mime/file-hash.bro | 4 +- scripts/base/protocols/mime/file-ident.bro | 2 +- .../frameworks/communication/listen-clear.bro | 2 + .../frameworks/communication/listen-ssl.bro | 2 + .../policy/frameworks/control/controller.bro | 3 +- scripts/policy/protocols/conn/scan.bro | 2 +- scripts/site/local.bro | 2 +- src/BroDoc.cc | 20 +++-- src/BroDoc.h | 8 +- src/CMakeLists.txt | 5 +- src/util.cc | 84 +++++++------------ src/util.h | 2 +- .../btest/Baseline/core.load-prefixes/output | 8 +- .../doc.autogen-reST-example/example.rst | 2 +- .../Baseline/doc.blacklist-reminder/.stderr | 0 testing/btest/core/load-prefixes.bro | 4 +- testing/btest/doc/blacklist-reminder.test | 8 ++ 40 files changed, 178 insertions(+), 154 deletions(-) delete mode 100644 doc/scripts/source/policy/index.rst create mode 100644 doc/scripts/source/scripts/index.rst create mode 100644 testing/btest/Baseline/doc.blacklist-reminder/.stderr create mode 100644 testing/btest/doc/blacklist-reminder.test diff --git a/CMakeLists.txt b/CMakeLists.txt index ce01d25b12..f1a5e0b772 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,14 +35,16 @@ endif () ## Project/Build Configuration set(BRO_ROOT_DIR ${CMAKE_INSTALL_PREFIX}) -if (NOT POLICYDIR) - # set the default policy installation path (user did not specify one) - set(POLICYDIR ${BRO_ROOT_DIR}/share/bro) +if (NOT BRO_SCRIPT_INSTALL_PATH) + # set the default Bro script installation path (user did not specify one) + set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro) endif () +set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts) -# sanitize the policy install directory into an absolute path +# sanitize the Bro script install directory into an absolute path # (CMake is confused by ~ as a representation of home directory) -get_filename_component(POLICYDIR ${POLICYDIR} ABSOLUTE) +get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} + ABSOLUTE) configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh @@ -212,7 +214,7 @@ message( "\n====================| Bro Build Summary |=====================" "\n" "\nInstall prefix: ${CMAKE_INSTALL_PREFIX}" - "\nPolicy dir: ${POLICYDIR}" + "\nBro Script Path: ${BRO_SCRIPT_INSTALL_PATH}" "\nDebug mode: ${ENABLE_DEBUG}" "\n" "\nCC: ${CMAKE_C_COMPILER}" diff --git a/bro-path-dev.in b/bro-path-dev.in index 57cc8fb2df..81d4f111fc 100755 --- a/bro-path-dev.in +++ b/bro-path-dev.in @@ -1,7 +1,7 @@ #!/bin/sh -# After configured by CMake, this file prints the absolute path to policy -# files that come with the source distributions of Bro as well as policy -# files that are generated by the BIF compiler at compile time. +# After configured by CMake, this file prints the absolute path to Bro scripts +# that come with the source distributions of Bro as well as scripts that are +# generated by the BIF compiler at compile time. # # The intended use of this script is to make it easier to run Bro from # the build directory, avoiding the need to install it. This could be @@ -10,10 +10,10 @@ # BROPATH=`./bro-path-dev` ./src/bro # -broPolicies=${PROJECT_SOURCE_DIR}/scripts:${PROJECT_SOURCE_DIR}/scripts/policy:${PROJECT_SOURCE_DIR}/scripts/site +broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site broGenPolicies=${CMAKE_BINARY_DIR}/src -installedPolicies=${POLICYDIR}:${POLICYDIR}/site +installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site echo .:$broPolicies:$broGenPolicies diff --git a/configure b/configure index 7df0e8b759..2e168f0550 100755 --- a/configure +++ b/configure @@ -22,7 +22,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Installation Directories: --prefix=PREFIX installation directory [/usr/local/bro] - --policydir=PATH policy file installation directory + --scriptdir=PATH root installation directory for Bro scripts [PREFIX/share/bro] Optional Features: @@ -85,7 +85,7 @@ CMakeCacheEntries="" append_cache_entry CMAKE_INSTALL_PREFIX PATH /usr/local/bro append_cache_entry BRO_ROOT_DIR PATH /usr/local/bro append_cache_entry PY_MOD_INSTALL_DIR PATH /usr/local/bro/lib/broctl -append_cache_entry POLICYDIR STRING /usr/local/bro/share/bro +append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro append_cache_entry ENABLE_DEBUG BOOL false append_cache_entry BROv6 BOOL false append_cache_entry ENABLE_PERFTOOLS BOOL false @@ -118,13 +118,13 @@ while [ $# -ne 0 ]; do append_cache_entry CMAKE_INSTALL_PREFIX PATH $optarg append_cache_entry BRO_ROOT_DIR PATH $optarg append_cache_entry PY_MOD_INSTALL_DIR PATH $optarg/lib/broctl - if [ "$user_set_policydir" != "true" ]; then - append_cache_entry POLICYDIR STRING $optarg/share/bro + if [ "$user_set_scriptdir" != "true" ]; then + append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg/share/bro fi ;; - --policydir=*) - append_cache_entry POLICYDIR STRING $optarg - user_set_policydir="true" + --scriptdir=*) + append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg + user_set_scriptdir="true" ;; --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 02d24ee88a..cb7a42929c 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -57,10 +57,12 @@ macro(REST_TARGET srcDir broInput) get_filename_component(relDstDir ${broInput} PATH) set(sumTextSrc ${absSrcPath}) + set(ogSourceFile ${absSrcPath}) if (${extension} STREQUAL ".bif.bro") + set(ogSourceFile ${BIF_SRC_DIR}/${basename}) # the summary text is taken at configure time, but .bif.bro files # may not have been generated yet, so read .bif file instead - set(sumTextSrc ${BIF_SRC_DIR}/${basename}) + set(sumTextSrc ${ogSourceFile}) endif () if (NOT relDstDir) @@ -124,9 +126,9 @@ macro(REST_TARGET srcDir broInput) ARGS -E make_directory ${dstDir} COMMAND "${CMAKE_COMMAND}" ARGS -E copy ${restFile} ${restOutput} - # copy the bro policy script, too + # copy the bro or bif script, too COMMAND "${CMAKE_COMMAND}" - ARGS -E copy ${absSrcPath} ${dstDir} + ARGS -E copy ${ogSourceFile} ${dstDir} # clean up the build directory COMMAND rm ARGS -rf .state *.log *.rst @@ -151,6 +153,8 @@ set(MASTER_PKG_INDEX_TEXT "") foreach (pkg ${MASTER_PKG_LIST}) # strip of the trailing /index for the link name get_filename_component(lnktxt ${pkg} PATH) + # pretty-up the link name by removing common scripts/ prefix + string(REPLACE "scripts/" "" lnktxt "${lnktxt}") set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${lnktxt} <${pkg}>") endforeach () file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}") diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 38f68f53db..54783f61b3 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -13,22 +13,14 @@ set(psd ${PROJECT_SOURCE_DIR}/scripts) rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) -rest_target(${psd} base/bro.init internal) -rest_target(${psd} base/all.bro internal) +rest_target(${psd} base/init-default.bro internal) +rest_target(${psd} base/init-bare.bro internal) rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src common-rw.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src dns-rw.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src finger-rw.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src ftp-rw.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src http-rw.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src ident-rw.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src reporter.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src smb-rw.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/src smtp-rw.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro) rest_target(${psd} base/frameworks/cluster/main.bro) @@ -56,6 +48,7 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro) rest_target(${psd} base/frameworks/reporter/main.bro) rest_target(${psd} base/frameworks/signatures/main.bro) rest_target(${psd} base/frameworks/software/main.bro) +rest_target(${psd} base/frameworks/time-machine/notice.bro) rest_target(${psd} base/protocols/conn/contents.bro) rest_target(${psd} base/protocols/conn/inactivity.bro) rest_target(${psd} base/protocols/conn/main.bro) @@ -72,6 +65,11 @@ rest_target(${psd} base/protocols/http/partial-content.bro) rest_target(${psd} base/protocols/http/utils.bro) rest_target(${psd} base/protocols/irc/dcc-send.bro) rest_target(${psd} base/protocols/irc/main.bro) +rest_target(${psd} base/protocols/mime/base.bro) +rest_target(${psd} base/protocols/mime/file-extract.bro) +rest_target(${psd} base/protocols/mime/file-hash.bro) +rest_target(${psd} base/protocols/mime/file-ident.bro) +rest_target(${psd} base/protocols/rpc/base.bro) rest_target(${psd} base/protocols/smtp/main.bro) rest_target(${psd} base/protocols/ssh/main.bro) rest_target(${psd} base/protocols/ssl/consts.bro) @@ -105,7 +103,6 @@ rest_target(${psd} policy/misc/loaded-scripts.bro) rest_target(${psd} policy/misc/trim-trace-file.bro) rest_target(${psd} policy/protocols/conn/known-hosts.bro) rest_target(${psd} policy/protocols/conn/known-services.bro) -rest_target(${psd} policy/protocols/conn/scan.bro) rest_target(${psd} policy/protocols/dns/auth-addl.bro) rest_target(${psd} policy/protocols/dns/detect-external-names.bro) rest_target(${psd} policy/protocols/ftp/detect.bro) diff --git a/doc/scripts/README b/doc/scripts/README index bd7ec5c065..ca7f28492b 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -44,7 +44,7 @@ of documentation targets. This script should be run after adding new Bro script source files, and the changes commited to git. If a script shouldn't have documentation generated for it, there's also a -blacklist variable that can be maintained in the ``genDocSourcesList.sh`` +blacklist manifest that can be maintained in the ``genDocSourcesList.sh`` script. The blacklist can also be used if you want to define a certain grouping for diff --git a/doc/scripts/genDocSourcesList.sh b/doc/scripts/genDocSourcesList.sh index 0783884372..8ef4ff9c14 100755 --- a/doc/scripts/genDocSourcesList.sh +++ b/doc/scripts/genDocSourcesList.sh @@ -11,8 +11,31 @@ # Specific scripts can be blacklisted below when e.g. they currently aren't # parseable or they just aren't meant to be documented. -blacklist="__load__.bro|test-all.bro|all.bro" -blacklist_addl="hot.conn.bro" +blacklist () + { + if [[ "$blacklist" == "" ]]; then + blacklist="$1" + else + blacklist="$blacklist|$1" + fi + } + +# files passed into this function are meant to be temporary workarounds +# because they're not finished or otherwise can't be loaded for some reason +tmp_blacklist () + { + echo "Warning: temporarily blacklisted files named '$1'" 1>&2 + blacklist $1 + } + +blacklist __load__.bro +blacklist test-all.bro +blacklist all.bro +blacklist init-default.bro +blacklist init-bare.bro + +tmp_blacklist hot.conn.bro +tmp_blacklist scan.bro statictext="\ # DO NOT EDIT @@ -30,8 +53,8 @@ statictext="\ set(psd \${PROJECT_SOURCE_DIR}/scripts) rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) -rest_target(\${psd} base/bro.init internal) -rest_target(\${psd} base/all.bro internal) +rest_target(\${psd} base/init-default.bro internal) +rest_target(\${psd} base/init-bare.bro internal) " if [[ $# -ge 1 ]]; then @@ -58,7 +81,7 @@ scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )` for file in $scriptfiles do f=${file:2} - if [[ (! $f =~ $blacklist) && (! $f =~ $blacklist_addl) ]]; then + if [[ ! $f =~ $blacklist ]]; then echo "rest_target(\${psd} $f)" >> $outfile fi done diff --git a/doc/scripts/group_index_generator.py b/doc/scripts/group_index_generator.py index 74c8c14d65..e9e17f3ca9 100755 --- a/doc/scripts/group_index_generator.py +++ b/doc/scripts/group_index_generator.py @@ -1,9 +1,8 @@ #! /usr/bin/env python # This script automatically generates a reST documents that lists -# a collection of Bro policy scripts that are "grouped" together. -# The summary text (##! comments) of the policy script is embedded -# in the list. +# a collection of Bro scripts that are "grouped" together. +# The summary text (##! comments) of the script is embedded in the list # # 1st argument is the file containing list of groups # 2nd argument is the directory containing ${group}_files lists of @@ -57,6 +56,6 @@ with open(group_list, 'r') as f_group_list: f_group_file.write("\n"); with open(group_file, 'a') as f_group_file: - f_group_file.write("\n:doc:`/policy/%s`\n" % doc_names[i]) + f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i]) for line in summary_comments: f_group_file.write(" " + line) diff --git a/doc/scripts/source/index.rst b/doc/scripts/source/index.rst index 83bceebb0c..a144644208 100644 --- a/doc/scripts/source/index.rst +++ b/doc/scripts/source/index.rst @@ -14,7 +14,7 @@ Contents: internal bifs packages - policy/index + scripts/index Indices and tables ================== diff --git a/doc/scripts/source/internal.rst b/doc/scripts/source/internal.rst index d9b914df65..817c76e725 100644 --- a/doc/scripts/source/internal.rst +++ b/doc/scripts/source/internal.rst @@ -1,5 +1,5 @@ .. This is a stub doc to which the build process can append. -Internal Policy Scripts -======================= +Internal Scripts +================ diff --git a/doc/scripts/source/packages.rst b/doc/scripts/source/packages.rst index a7e76f097a..35f6633fd4 100644 --- a/doc/scripts/source/packages.rst +++ b/doc/scripts/source/packages.rst @@ -1,11 +1,15 @@ .. This is a stub doc to which the build process can append. -Policy Script Packages -====================== +Bro Script Packages +=================== -Bro has the following policy script packages (e.g. collections of related -policy scripts). If the package contains a ``__load__.bro`` script, it -supports being loaded in mass as a whole directory for convenience. +Bro has the following script packages (e.g. collections of related scripts in +a common directory). If the package directory contains a ``__load__.bro`` +script, it supports being loaded in mass as a whole directory for convenience. + +Packages/scripts in the ``base/`` directory are all loaded by default, while +ones in ``policy/`` provide functionality and customization options that are +more appropriate for users to decide whether they'd like to load it or not. .. toctree:: :maxdepth: 1 diff --git a/doc/scripts/source/policy/index.rst b/doc/scripts/source/policy/index.rst deleted file mode 100644 index 6b8ed9319c..0000000000 --- a/doc/scripts/source/policy/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -Index of All Policy Script Documentation -======================================== - -.. toctree:: - :maxdepth: 1 - diff --git a/doc/scripts/source/scripts/index.rst b/doc/scripts/source/scripts/index.rst new file mode 100644 index 0000000000..fd11a3924e --- /dev/null +++ b/doc/scripts/source/scripts/index.rst @@ -0,0 +1,6 @@ +Index of All Bro Script Documentation +===================================== + +.. toctree:: + :maxdepth: 1 + diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 5979e6befd..d7a321627f 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -1,9 +1,8 @@ include(InstallPackageConfigFile) -install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING +install(DIRECTORY ./ DESTINATION ${BRO_SCRIPT_INSTALL_PATH} FILES_MATCHING PATTERN "all.bro" EXCLUDE PATTERN "site/local.bro" EXCLUDE - PATTERN "bro.init" PATTERN "*.bro" PATTERN "*.sig" PATTERN "*.osf" @@ -13,6 +12,6 @@ install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING # user modify-able. InstallPackageConfigFile( ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro - ${POLICYDIR}/site + ${BRO_SCRIPT_INSTALL_PATH}/site local.bro) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 04d474e604..c51b59f70f 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -1,3 +1,5 @@ +@load base/frameworks/communication + module Cluster; event bro_init() &priority=9 diff --git a/scripts/base/frameworks/communication/__load__.bro b/scripts/base/frameworks/communication/__load__.bro index a854726f04..3b18043db1 100644 --- a/scripts/base/frameworks/communication/__load__.bro +++ b/scripts/base/frameworks/communication/__load__.bro @@ -2,4 +2,4 @@ # TODO: get rid of this as soon as the Expr.cc hack is changed. @if ( getenv("ENABLE_COMMUNICATION") != "" ) @load ./main -@endif \ No newline at end of file +@endif diff --git a/scripts/base/frameworks/metrics/conn-example.bro b/scripts/base/frameworks/metrics/conn-example.bro index 9e20798cc0..f4e8e71d86 100644 --- a/scripts/base/frameworks/metrics/conn-example.bro +++ b/scripts/base/frameworks/metrics/conn-example.bro @@ -1,4 +1,4 @@ -@load frameworks/metrics +@load base/frameworks/metrics redef enum Metrics::ID += { CONNS_ORIGINATED, @@ -16,4 +16,4 @@ event connection_established(c: connection) Metrics::add_data(CONNS_ORIGINATED, [$host=c$id$orig_h], 1); Metrics::add_data(CONNS_RESPONDED, [$host=c$id$resp_h], 1); } - \ No newline at end of file + diff --git a/scripts/base/frameworks/metrics/http-example.bro b/scripts/base/frameworks/metrics/http-example.bro index a91fdbb47d..6af4031fd2 100644 --- a/scripts/base/frameworks/metrics/http-example.bro +++ b/scripts/base/frameworks/metrics/http-example.bro @@ -1,4 +1,4 @@ -@load frameworks/metrics +@load base/frameworks/metrics redef enum Metrics::ID += { HTTP_REQUESTS_BY_STATUS_CODE, @@ -17,4 +17,4 @@ event HTTP::log_http(rec: HTTP::Info) Metrics::add_data(HTTP_REQUESTS_BY_HOST, [$index=rec$host], 1); if ( rec?$status_code ) Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $index=fmt("%d", rec$status_code)], 1); - } \ No newline at end of file + } diff --git a/scripts/base/frameworks/notice/actions/email_admin.bro b/scripts/base/frameworks/notice/actions/email_admin.bro index 07a6568327..c03629d885 100644 --- a/scripts/base/frameworks/notice/actions/email_admin.bro +++ b/scripts/base/frameworks/notice/actions/email_admin.bro @@ -1,4 +1,3 @@ - module Notice; export { @@ -25,4 +24,4 @@ event notice(n: Notice::Info) &priority=-5 if ( email != "" ) email_notice_to(n, email, T); } - } \ No newline at end of file + } diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 83cdc4807d..da2d0d5d0f 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -8,7 +8,7 @@ event Notice::notice(n: Notice::Info) &priority=10 return; # This should only be done for notices that are being sent to email. - if ( ACTION_EMAIL !in n$action ) + if ( ACTION_EMAIL !in n$actions ) return; local output = ""; @@ -37,4 +37,4 @@ event Notice::notice(n: Notice::Info) &priority=10 if ( output != "" ) n$email_body_sections[|n$email_body_sections|] = output; - } \ No newline at end of file + } diff --git a/scripts/base/frameworks/notice/weird.bro b/scripts/base/frameworks/notice/weird.bro index 4718dc204a..31ecd484e9 100644 --- a/scripts/base/frameworks/notice/weird.bro +++ b/scripts/base/frameworks/notice/weird.bro @@ -1,4 +1,3 @@ - module Weird; export { diff --git a/scripts/base/protocols/mime/base.bro b/scripts/base/protocols/mime/base.bro index d0212870a4..df495387d7 100644 --- a/scripts/base/protocols/mime/base.bro +++ b/scripts/base/protocols/mime/base.bro @@ -1,7 +1,7 @@ ##! The mime script does analysis of MIME encoded messages seen in certain ##! protocols (only SMTP and POP3 at the moment). -@load utils/strings +@load base/utils/strings module MIME; diff --git a/scripts/base/protocols/mime/file-extract.bro b/scripts/base/protocols/mime/file-extract.bro index d6989ad809..33d2c70513 100644 --- a/scripts/base/protocols/mime/file-extract.bro +++ b/scripts/base/protocols/mime/file-extract.bro @@ -1,5 +1,5 @@ -@load protocols/mime/file-ident -@load utils/files +@load ./file-ident +@load base/utils/files module MIME; @@ -57,4 +57,4 @@ event mime_end_entity(c: connection) &priority=-3 if ( c$mime?$extraction_file ) close(c$mime$extraction_file); } - \ No newline at end of file + diff --git a/scripts/base/protocols/mime/file-hash.bro b/scripts/base/protocols/mime/file-hash.bro index 3384928d58..5139810b8e 100644 --- a/scripts/base/protocols/mime/file-hash.bro +++ b/scripts/base/protocols/mime/file-hash.bro @@ -1,4 +1,4 @@ -@load protocols/mime/file-ident +@load ./file-ident module MIME; @@ -75,4 +75,4 @@ event mime_end_entity(c: connection) &priority=-3 NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h), $sub=c$mime$md5, $conn=c]); } - } \ No newline at end of file + } diff --git a/scripts/base/protocols/mime/file-ident.bro b/scripts/base/protocols/mime/file-ident.bro index ba5310d362..346fde1bba 100644 --- a/scripts/base/protocols/mime/file-ident.bro +++ b/scripts/base/protocols/mime/file-ident.bro @@ -1,4 +1,4 @@ -@load protocols/mime/base +@load ./base module MIME; diff --git a/scripts/policy/frameworks/communication/listen-clear.bro b/scripts/policy/frameworks/communication/listen-clear.bro index 4f96414172..c5dd6e1e70 100644 --- a/scripts/policy/frameworks/communication/listen-clear.bro +++ b/scripts/policy/frameworks/communication/listen-clear.bro @@ -1,5 +1,7 @@ ##! Listen for other Bro instances to make unencrypted connections. +@load base/frameworks/communication + module Communication; export { diff --git a/scripts/policy/frameworks/communication/listen-ssl.bro b/scripts/policy/frameworks/communication/listen-ssl.bro index 32c5f747c2..188d55ea5f 100644 --- a/scripts/policy/frameworks/communication/listen-ssl.bro +++ b/scripts/policy/frameworks/communication/listen-ssl.bro @@ -1,5 +1,7 @@ ##! Listen for other Bro instances and encrypt the connection with SSL. +@load base/frameworks/communication + module Communication; export { diff --git a/scripts/policy/frameworks/control/controller.bro b/scripts/policy/frameworks/control/controller.bro index 8c60ef457a..83fd5e5451 100644 --- a/scripts/policy/frameworks/control/controller.bro +++ b/scripts/policy/frameworks/control/controller.bro @@ -1,3 +1,4 @@ +@load base/frameworks/communication module Control; @@ -99,4 +100,4 @@ event remote_connection_handshake_done(p: event_peer) &priority=-10 # Signal configuration update to peer. event Control::configuration_update_request(); } - } \ No newline at end of file + } diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/protocols/conn/scan.bro index fabb865093..ab715cccb3 100644 --- a/scripts/policy/protocols/conn/scan.bro +++ b/scripts/policy/protocols/conn/scan.bro @@ -1,4 +1,4 @@ -@load frameworks/notice +@load base/frameworks/notice @load port-name module Scan; diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 7e758c72b0..5199cab288 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -43,4 +43,4 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; @load protocols/ssl/known-certs # Load the script to enable SSL/TLS certificate validation. -@load protocols/ssl/validate-certs \ No newline at end of file +@load protocols/ssl/validate-certs diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 7754dfc4c9..302bd04c88 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -20,8 +20,8 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs) if ( rel[0] == '/' || rel[0] == '.' ) { - // The Bro script must not be on a subpath of the policy/ dir of - // BROPATH, so just use the basename as the document title. + // The Bro script isn't being loaded via BROPATH, so just use basename + // as the document title. doc_title = source_filename; } else @@ -33,8 +33,14 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs) doc_title = rel + "/" + source_filename; } + downloadable_filename = source_filename; + + size_t ext_pos = downloadable_filename.find(".bif.bro"); + if ( std::string::npos != ext_pos ) + downloadable_filename.erase(ext_pos + 4); + reST_filename = doc_title; - size_t ext_pos = reST_filename.find(".bro"); + ext_pos = reST_filename.find(".bro"); if ( std::string::npos == ext_pos ) reST_filename += ".rst"; @@ -103,14 +109,14 @@ void BroDoc::AddImport(const std::string& s) { if ( subpath[0] == '/' || subpath[0] == '.' ) { - // it's not a subpath of policy/, so just add the name of it + // it's not a subpath of scripts/, so just add the name of it // as it's given in the @load directive imports.push_back(lname); } else { // combine the base file name of script in the @load directive - // with the subpath of BROPATH's policy/ directory + // with the subpath of BROPATH's scripts/ directory string fname(subpath); char* othertmp = copy_string(lname.c_str()); fname.append("/").append(basename(othertmp)); @@ -167,7 +173,7 @@ void BroDoc::WriteDocFile() const WriteSectionHeading(doc_title.c_str(), '='); WriteToDoc("\n:download:`Original Source File <%s>`\n\n", - source_filename.c_str()); + downloadable_filename.c_str()); WriteSectionHeading("Overview", '-'); WriteStringList("%s\n", "%s\n\n", summary); @@ -185,7 +191,7 @@ void BroDoc::WriteDocFile() const size_t pos = pretty.find("/index"); if ( pos != std::string::npos && pos + 6 == pretty.size() ) pretty = pretty.substr(0, pos); - WriteToDoc(":doc:`%s `", pretty.c_str(), it->c_str()); + WriteToDoc(":doc:`%s `", pretty.c_str(), it->c_str()); } WriteToDoc("\n"); } diff --git a/src/BroDoc.h b/src/BroDoc.h index 975177fc9a..112401253c 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -24,10 +24,9 @@ public: * If the filename doesn't end in ".bro", then ".rst" is just appended. * Any '/' characters in the reST file name that result from choice of * the 'rel' parameter are replaced with '^'. - * @param subpath A string representing a subpath of BROPATH's policy/ - * directory in which the source file is located. It can - * also be full path to the file or a full path that's in BROPATH, - * but in either of those cases, the parameter is essentially + * @param rel A string representing a subpath of the root Bro script + * source/install directory in which the source file is located. + * It can also be an absolute path, but then the parameter is * ignored and the document title is just derived from file name * @param abs The absolute path to the Bro script for which to generate * documentation. @@ -211,6 +210,7 @@ protected: FILE* reST_file; std::string reST_filename; std::string source_filename; // points to the basename of source file + std::string downloadable_filename; // file that will be linked for download std::string doc_title; std::string packet_filter; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a5f096f70..6bc82ebb91 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -408,7 +408,8 @@ set(bro_SRCS collect_headers(bro_HEADERS ${bro_SRCS}) -add_definitions(-DPOLICYDEST="${POLICYDIR}") +add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}") +add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}") add_executable(bro ${bro_SRCS} ${bro_HEADERS}) @@ -423,7 +424,7 @@ set(brolibs target_link_libraries(bro ${brolibs}) install(TARGETS bro DESTINATION bin) -install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${POLICYDIR}) +install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}) set(BRO_EXE bro CACHE STRING "Bro executable binary" FORCE) diff --git a/src/util.cc b/src/util.cc index 528a505f60..90eac56cca 100644 --- a/src/util.cc +++ b/src/util.cc @@ -757,9 +757,9 @@ const char* bro_path() const char* path = getenv("BROPATH"); if ( ! path ) path = ".:" - POLICYDEST ":" - POLICYDEST "/policy" ":" - POLICYDEST "/site"; + BRO_SCRIPT_INSTALL_PATH ":" + BRO_SCRIPT_INSTALL_PATH "/policy" ":" + BRO_SCRIPT_INSTALL_PATH "/site"; return path; } @@ -891,60 +891,36 @@ const char* normalize_path(const char* path) return copy_string(new_path.c_str()); } -// Returns the subpath of BROPATH's policy/ directory in which the loaded -// file in located. If it's not under a subpath of policy/ then the full -// path is returned, else the subpath of policy/ concatentated with any -// directory prefix of the file is returned. -void get_policy_subpath(const char* dir, const char* file, const char** subpath) +// Returns the subpath of the root Bro script install/source directory in +// which the loaded file is located. If it's not under a subpath of that +// directory (e.g. cwd or custom path) then the full path is returned. +void get_script_subpath(const std::string& full_filename, const char** subpath) { - // first figure out if this is a subpath of policy/ - const char* ploc = strstr(dir, "policy"); - if ( ploc ) - if ( ploc[6] == '\0' ) - *subpath = copy_string(ploc + 6); - else if ( ploc[6] == '/' ) - *subpath = copy_string(ploc + 7); - else - *subpath = copy_string(dir); - else - *subpath = copy_string(dir); + size_t p; + std::string my_subpath(full_filename); - // and now add any directory parts of the filename - char full_filename_buf[1024]; - safe_snprintf(full_filename_buf, sizeof(full_filename_buf), - "%s/%s", dir, file); - char* tmp = copy_string(file); - const char* fdir = 0; - - if ( is_dir(full_filename_buf) ) - fdir = file; - - if ( ! fdir ) - fdir = dirname(tmp); - - if ( ! streq(fdir, ".") ) + // get the parent directory of file (if not already a directory) + if ( ! is_dir(full_filename.c_str()) ) { - size_t full_subpath_len = strlen(*subpath) + strlen(fdir) + 1; - bool needslash = false; - if ( strlen(*subpath) != 0 && (*subpath)[strlen(*subpath) - 1] != '/' ) - { - ++full_subpath_len; - needslash = true; - } - - char* full_subpath = new char[full_subpath_len]; - strcpy(full_subpath, *subpath); - if ( needslash ) - strcat(full_subpath, "/"); - strcat(full_subpath, fdir); - delete [] *subpath; - *subpath = full_subpath; + char* tmp = copy_string(full_filename.c_str()); + my_subpath = dirname(tmp); + delete [] tmp; } - const char* normalized_subpath = normalize_path(*subpath); - delete [] tmp; - delete [] *subpath; - *subpath = normalized_subpath; + // first check if this is some subpath of the installed scripts root path, + // if not check if it's a subpath of the script source root path, + // if neither, will just use the given directory + if ( (p=my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos ) + my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH)); + else if ( (p=my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos ) + my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH)); + + // if root path found, remove path separators until next path component + if ( p != std::string::npos ) + while ( my_subpath.size() && my_subpath[0] == '/' ) + my_subpath.erase(0, 1); + + *subpath = normalize_path(my_subpath.c_str()); } extern string current_scanned_file_path; @@ -1001,7 +977,7 @@ FILE* search_for_file(const char* filename, const char* ext, ! is_dir(full_filename_buf) ) { if ( bropath_subpath ) - get_policy_subpath(dir_beginning, filename, bropath_subpath); + get_script_subpath(full_filename_buf, bropath_subpath); return open_file(full_filename_buf, full_filename, load_pkgs); } @@ -1010,7 +986,7 @@ FILE* search_for_file(const char* filename, const char* ext, if ( access(full_filename_buf, R_OK) == 0 ) { if ( bropath_subpath ) - get_policy_subpath(dir_beginning, filename, bropath_subpath); + get_script_subpath(full_filename_buf, bropath_subpath); return open_file(full_filename_buf, full_filename, load_pkgs); } diff --git a/src/util.h b/src/util.h index 82c86da950..392b2ca76a 100644 --- a/src/util.h +++ b/src/util.h @@ -179,7 +179,7 @@ extern const char* bro_path(); extern const char* bro_prefixes(); std::string dot_canon(std::string path, std::string file, std::string prefix = ""); const char* normalize_path(const char* path); -void get_policy_subpath(const char* dir, const char* file, const char** subpath); +void get_script_subpath(const std::string& full_filename, const char** subpath); extern FILE* search_for_file(const char* filename, const char* ext, const char** full_filename, bool load_pkgs, const char** bropath_subpath); diff --git a/testing/btest/Baseline/core.load-prefixes/output b/testing/btest/Baseline/core.load-prefixes/output index 733bc9a594..ea35b3a8c0 100644 --- a/testing/btest/Baseline/core.load-prefixes/output +++ b/testing/btest/Baseline/core.load-prefixes/output @@ -1,4 +1,4 @@ -loaded lcl2.site.bro -loaded lcl.site.bro -loaded lcl2.protocols.http.bro -loaded lcl.protocols.http.bro +loaded lcl2.base.utils.site.bro +loaded lcl.base.utils.site.bro +loaded lcl2.base.protocols.http.bro +loaded lcl.base.protocols.http.bro diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index 2c2ed28369..ec48477d61 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -29,7 +29,7 @@ each of "columns", "event", "filter" depending on exactly what it's doing. :Author: Jon Siwek -:Imports: :doc:`frameworks/notice ` +:Imports: :doc:`policy/frameworks/software/vulnerable ` Summary ~~~~~~~ diff --git a/testing/btest/Baseline/doc.blacklist-reminder/.stderr b/testing/btest/Baseline/doc.blacklist-reminder/.stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/core/load-prefixes.bro b/testing/btest/core/load-prefixes.bro index 54c6bed0ef..1dfc3ac5dd 100644 --- a/testing/btest/core/load-prefixes.bro +++ b/testing/btest/core/load-prefixes.bro @@ -9,11 +9,11 @@ @TEST-END-FILE @TEST-START-FILE lcl.base.utils.site.bro -print "loaded lcl.base.site.bro"; +print "loaded lcl.base.utils.site.bro"; @TEST-END-FILE @TEST-START-FILE lcl2.base.utils.site.bro -print "loaded lcl2.base.site.bro"; +print "loaded lcl2.base.utils.site.bro"; @TEST-END-FILE @TEST-START-FILE lcl.base.protocols.http.bro diff --git a/testing/btest/doc/blacklist-reminder.test b/testing/btest/doc/blacklist-reminder.test new file mode 100644 index 0000000000..8b79d8a28f --- /dev/null +++ b/testing/btest/doc/blacklist-reminder.test @@ -0,0 +1,8 @@ +# This test will fail if there are Bro scripts that have been temporarily +# blacklisted from the documentation generation process for some reason +# (e.g. they're a work-in-progress or otherwise fail to parse). It's meant +# to serve as a reminder that some future action may be needed to generate +# documentation for the blacklisted scripts. +# +# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh +# @TEST-EXEC: btest-diff .stderr From 3437220fe30c481362a6b300903fcfae03849521 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 9 Aug 2011 09:51:03 -0400 Subject: [PATCH 04/25] piped_exec can now write nulls in the "to_write" argument. - Additional test for testing writing null values. --- src/bro.bif | 9 ++++++++- testing/btest/Baseline/bifs.piped_exec/test.txt | Bin 0 -> 9 bytes testing/btest/bifs/piped_exec.bro | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/bifs.piped_exec/test.txt diff --git a/src/bro.bif b/src/bro.bif index d3bbd7c072..144d71af92 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3631,7 +3631,14 @@ function piped_exec%(program: string, to_write: string%): bool return new Val(false, TYPE_BOOL); } - fprintf(f, "%s", to_write->CheckString()); + const u_char* input_data = to_write->Bytes(); + int input_data_len = to_write->Len(); + int bytes_written = fwrite(input_data, 1, input_data_len, f); + if ( bytes_written != input_data_len ) + { + reporter->Error("Failed to write all given data to %s", prog); + return new Val(false, TYPE_BOOL); + } pclose(f); return new Val(true, TYPE_BOOL); diff --git a/testing/btest/Baseline/bifs.piped_exec/test.txt b/testing/btest/Baseline/bifs.piped_exec/test.txt new file mode 100644 index 0000000000000000000000000000000000000000..a23f66ba7ec8a253a4e1a9887baed69b403be783 GIT binary patch literal 9 QcmZQz$Vkn}$!A~y018+F6#xJL literal 0 HcmV?d00001 diff --git a/testing/btest/bifs/piped_exec.bro b/testing/btest/bifs/piped_exec.bro index 4405f0b500..32fd5c5f80 100644 --- a/testing/btest/bifs/piped_exec.bro +++ b/testing/btest/bifs/piped_exec.bro @@ -1,6 +1,12 @@ # @TEST-EXEC: bro %INPUT >output # @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff test.txt + global cmds = "print \"hello world\";"; cmds = string_cat(cmds, "\nprint \"foobar\";"); piped_exec("bro", cmds); + +# Test null output. +piped_exec("cat > test.txt", "\x00\x00hello\x00\x00"); + From b783732f7d5ce83f0a68468cd77310d5de58ee79 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 9 Aug 2011 13:34:12 -0500 Subject: [PATCH 05/25] Workaround for FreeBSD CMake port missing debug flags --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce01d25b12..d5d6944415 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,8 @@ set(EXTRA_COMPILE_FLAGS "-Wall -Wno-unused") if (ENABLE_DEBUG) set(CMAKE_BUILD_TYPE Debug) - set(EXTRA_COMPILE_FLAGS "${EXTRA_COMPILE_FLAGS} -DDEBUG") + # manual add of -g works around its omission in FreeBSD's CMake port + set(EXTRA_COMPILE_FLAGS "${EXTRA_COMPILE_FLAGS} -g -DDEBUG") else () set(CMAKE_BUILD_TYPE RelWithDebInfo) endif () From 8e7a76b5481fead9dd33682451e6cde8b4f23c27 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 10 Aug 2011 13:37:58 -0400 Subject: [PATCH 06/25] HTTP now uses the extract_filename_from_content_disposition function. --- scripts/base/protocols/http/main.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 172fb2bd1c..8cd80bde5f 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -214,7 +214,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr c$http$response_content_length = extract_count(value); else if ( name == "CONTENT-DISPOSITION" && /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) - c$http$filename = sub(value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, ""); + c$http$filename = extract_filename_from_content_disposition(value); } } From 6f50e362db1a5b8c456dde979921b958c0e43dc5 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:32:05 -0400 Subject: [PATCH 07/25] Updates for the build system and site local scripts for cluster. --- scripts/CMakeLists.txt | 21 +++++++++++++------ .../base/frameworks/cluster/nodes/manager.bro | 3 +++ .../base/frameworks/cluster/nodes/proxy.bro | 3 +++ .../base/frameworks/cluster/nodes/worker.bro | 3 +++ .../frameworks/cluster/setup-connections.bro | 1 + scripts/site/local-manager.bro | 7 +++++++ scripts/site/local-proxy.bro | 2 ++ scripts/site/local-worker.bro | 1 + 8 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 scripts/site/local-manager.bro create mode 100644 scripts/site/local-proxy.bro create mode 100644 scripts/site/local-worker.bro diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 5979e6befd..9933508b2d 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -1,18 +1,27 @@ include(InstallPackageConfigFile) install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING - PATTERN "all.bro" EXCLUDE - PATTERN "site/local.bro" EXCLUDE - PATTERN "bro.init" + PATTERN "site/local*" EXCLUDE PATTERN "*.bro" PATTERN "*.sig" - PATTERN "*.osf" + PATTERN "*.fp" ) -# Install as a config file since the local.bro script is meant to be +# Install all local* scripts as config files since they are meant to be # user modify-able. InstallPackageConfigFile( ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro ${POLICYDIR}/site local.bro) - +InstallPackageConfigFile( + ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro + ${POLICYDIR}/site + local-manager.bro) +InstallPackageConfigFile( + ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro + ${POLICYDIR}/site + local-proxy.bro) +InstallPackageConfigFile( + ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro + ${POLICYDIR}/site + local-worker.bro) diff --git a/scripts/base/frameworks/cluster/nodes/manager.bro b/scripts/base/frameworks/cluster/nodes/manager.bro index c9ce8c2d1a..b85c20ae77 100644 --- a/scripts/base/frameworks/cluster/nodes/manager.bro +++ b/scripts/base/frameworks/cluster/nodes/manager.bro @@ -10,6 +10,9 @@ @prefixes += cluster-manager +## Load the script for local site configuration for the manager node. +@load site/local-manager + ## Turn off remote logging since this is the manager and should only log here. redef Log::enable_remote_logging = F; diff --git a/scripts/base/frameworks/cluster/nodes/proxy.bro b/scripts/base/frameworks/cluster/nodes/proxy.bro index 377b087b36..60df27b452 100644 --- a/scripts/base/frameworks/cluster/nodes/proxy.bro +++ b/scripts/base/frameworks/cluster/nodes/proxy.bro @@ -1,6 +1,9 @@ @prefixes += cluster-proxy +## Load the script for local site configuration for proxy nodes. +@load site/local-proxy + ## The proxy only syncs state; does not forward events. redef forward_remote_events = F; redef forward_remote_state_changes = T; diff --git a/scripts/base/frameworks/cluster/nodes/worker.bro b/scripts/base/frameworks/cluster/nodes/worker.bro index cf8620c5d7..53d5d53872 100644 --- a/scripts/base/frameworks/cluster/nodes/worker.bro +++ b/scripts/base/frameworks/cluster/nodes/worker.bro @@ -1,6 +1,9 @@ @prefixes += cluster-worker +## Load the script for local site configuration for the worker nodes. +@load site/local-worker + ## Don't do any local logging. redef Log::enable_local_logging = F; diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 04d474e604..2a16ca1ea1 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -1,3 +1,4 @@ + module Cluster; event bro_init() &priority=9 diff --git a/scripts/site/local-manager.bro b/scripts/site/local-manager.bro new file mode 100644 index 0000000000..aa28bd79da --- /dev/null +++ b/scripts/site/local-manager.bro @@ -0,0 +1,7 @@ +##! Local site policy loaded only by the manager in a cluster. + +# If you are running a cluster you should define your Notice::policy here +# so that notice processing occurs on the manager. +redef Notice::policy += { + +}; diff --git a/scripts/site/local-proxy.bro b/scripts/site/local-proxy.bro new file mode 100644 index 0000000000..1b71cc1870 --- /dev/null +++ b/scripts/site/local-proxy.bro @@ -0,0 +1,2 @@ +##! Local site policy loaded only by the proxies if Bro is running as a cluster. + diff --git a/scripts/site/local-worker.bro b/scripts/site/local-worker.bro new file mode 100644 index 0000000000..b2a100e135 --- /dev/null +++ b/scripts/site/local-worker.bro @@ -0,0 +1 @@ +##! Local site policy loaded only by the workers if Bro is running as a cluster. \ No newline at end of file From 9a06cece67087cf74ef8efc48478580cf5747766 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:32:31 -0400 Subject: [PATCH 08/25] ConnSize analyzer is turned on by default now. --- scripts/base/init-bare.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 30674fafc7..f1468cea6b 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -509,7 +509,7 @@ const tunnel_port = 0/udp &redef; # packets and IP-level bytes transfered by each endpoint. If # true, these values are returned in the connection's endpoint # record val. -const use_conn_size_analyzer = F &redef; +const use_conn_size_analyzer = T &redef; const UDP_INACTIVE = 0; const UDP_ACTIVE = 1; # means we've seen something from this endpoint From 9c2273b7a7dfff8eda1f44ea6c25c5ebc6fab019 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:35:50 -0400 Subject: [PATCH 09/25] Updates for SSH scripts. --- scripts/base/protocols/ssh/main.bro | 197 ++++++------------ .../protocols/ssh/detect-bruteforcing.bro | 79 +++++++ scripts/policy/protocols/ssh/geo-data.bro | 39 ++++ .../protocols/ssh/interesting-hostnames.bro | 50 +++++ scripts/policy/protocols/ssh/software.bro | 8 +- scripts/site/local.bro | 5 + 6 files changed, 237 insertions(+), 141 deletions(-) create mode 100644 scripts/policy/protocols/ssh/detect-bruteforcing.bro create mode 100644 scripts/policy/protocols/ssh/geo-data.bro create mode 100644 scripts/policy/protocols/ssh/interesting-hostnames.bro diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index 7cc87b6684..5233c6da97 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -1,74 +1,58 @@ +##! Base SSH analysis script. The heuristic to blindly determine success or +##! failure for SSH connections is implemented here. At this time, it only +##! uses the size of the data being returned from the server to make the +##! heuristic determination about success of the connection. +##! Requires that :bro:id:`use_conn_size_analyzer` is set to T! The heuristic +##! is not attempted if the connection size analyzer isn't enabled. module SSH; export { redef enum Log::ID += { SSH }; - redef enum Notice::Type += { - Login, - Password_Guessing, - Login_By_Password_Guesser, - Login_From_Interesting_Hostname, - Bytecount_Inconsistency, - }; - type Info: record { ts: time &log; uid: string &log; id: conn_id &log; + ## Indicates if the login was heuristically guessed to be "success" + ## or "failure". status: string &log &optional; - direction: string &log &optional; - remote_location: geo_location &log &optional; + ## Direction of the connection. If the client was a local host + ## logging into an external host, this would be OUTBOUD. INBOUND + ## would be set for the opposite situation. + # TODO: handle local-local and remote-remote better. + direction: Direction &log &optional; + ## The software string given by the client. client: string &log &optional; + ## The software string given by the server. server: string &log &optional; + ## The amount of data returned from the server. This is currently + ## the only measure of the success heuristic and it is logged to + ## assist analysts looking at the logs to make their own determination + ## about the success on a case-by-case basis. resp_size: count &log &default=0; ## Indicate if the SSH session is done being watched. done: bool &default=F; }; - - const password_guesses_limit = 30 &redef; - # The size in bytes at which the SSH connection is presumed to be - # successful. + ## The size in bytes at which the SSH connection is presumed to be + ## successful. const authentication_data_size = 5500 &redef; - # The amount of time to remember presumed non-successful logins to build - # model of a password guesser. - const guessing_timeout = 30 mins &redef; - - # The set of countries for which you'd like to throw notices upon successful login - # requires Bro compiled with libGeoIP support - const watched_countries: set[string] = {"RO"} &redef; - - # Strange/bad host names to originate successful SSH logins - const interesting_hostnames = - /^d?ns[0-9]*\./ | - /^smtp[0-9]*\./ | - /^mail[0-9]*\./ | - /^pop[0-9]*\./ | - /^imap[0-9]*\./ | - /^www[0-9]*\./ | - /^ftp[0-9]*\./ &redef; - - # This is a table with orig subnet as the key, and subnet as the value. - const ignore_guessers: table[subnet] of subnet &redef; - - # If true, we tell the event engine to not look at further data - # packets after the initial SSH handshake. Helps with performance - # (especially with large file transfers) but precludes some - # kinds of analyses (e.g., tracking connection size). + ## If true, we tell the event engine to not look at further data + ## packets after the initial SSH handshake. Helps with performance + ## (especially with large file transfers) but precludes some + ## kinds of analyses (e.g., tracking connection size). const skip_processing_after_detection = F &redef; - # Keeps count of how many rejections a host has had - global password_rejections: table[addr] of TrackCount - &write_expire=guessing_timeout - &synchronized; - - # Keeps track of hosts identified as guessing passwords - # TODO: guessing_timeout doesn't work correctly here. If a user redefs - # the variable, it won't take effect. - global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized; + ## This event is generated when the heuristic thinks that a login + ## was successful. + global heuristic_successful_login: event(c: connection); + + ## This event is generated when the heuristic thinks that a login + ## failed. + global heuristic_failed_login: event(c: connection); global log_ssh: event(rec: Info); } @@ -106,116 +90,51 @@ function check_ssh_connection(c: connection, done: bool) # If this is still a live connection and the byte count has not # crossed the threshold, just return and let the resheduled check happen later. - if ( !done && c$resp$size < authentication_data_size ) + if ( !done && c$resp$num_bytes_ip < authentication_data_size ) return; # Make sure the server has sent back more than 50 bytes to filter out # hosts that are just port scanning. Nothing is ever logged if the server # doesn't send back at least 50 bytes. - if ( c$resp$size < 50 ) + if ( c$resp$num_bytes_ip < 50 ) return; - local status = "failure"; - local direction = Site::is_local_addr(c$id$orig_h) ? "to" : "from"; - local location: geo_location; - location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h); + c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND; + c$ssh$resp_size = c$resp$num_bytes_ip; - if ( done && c$resp$size < authentication_data_size ) + if ( c$resp$num_bytes_ip < authentication_data_size ) { - # presumed failure - if ( c$id$orig_h !in password_rejections ) - password_rejections[c$id$orig_h] = new_track_count(); - - # Track the number of rejections - if ( !(c$id$orig_h in ignore_guessers && - c$id$resp_h in ignore_guessers[c$id$orig_h]) ) - ++password_rejections[c$id$orig_h]$n; - - if ( default_check_threshold(password_rejections[c$id$orig_h]) ) - { - add password_guessers[c$id$orig_h]; - NOTICE([$note=Password_Guessing, - $conn=c, - $msg=fmt("SSH password guessing by %s", c$id$orig_h), - $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n), - $n=password_rejections[c$id$orig_h]$n]); - } - } - # TODO: This is to work around a quasi-bug in Bro which occasionally - # causes the byte count to be oversized. - # Watch for Gregors work that adds an actual counter of bytes transferred. - else if ( c$resp$size < 20000000 ) + c$ssh$status = "failure"; + event SSH::heuristic_failed_login(c); + } + else { # presumed successful login - status = "success"; - c$ssh$done = T; - - if ( c$id$orig_h in password_rejections && - password_rejections[c$id$orig_h]$n > password_guesses_limit && - c$id$orig_h !in password_guessers ) - { - add password_guessers[c$id$orig_h]; - NOTICE([$note=Login_By_Password_Guesser, - $conn=c, - $n=password_rejections[c$id$orig_h]$n, - $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h), - $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]); - } - - local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)", - direction, location$country_code, location$region, location$city, - location$latitude, location$longitude, - id_string(c$id), c$resp$size); - NOTICE([$note=Login, - $conn=c, - $msg=message, - $sub=location$country_code]); - - # Check to see if this login came from an interesting hostname - when ( local hostname = lookup_addr(c$id$orig_h) ) - { - if ( interesting_hostnames in hostname ) - { - NOTICE([$note=Login_From_Interesting_Hostname, - $conn=c, - $msg=fmt("Strange login from %s", hostname), - $sub=hostname]); - } - } - - if ( location$country_code in watched_countries ) - { - - } - + c$ssh$status = "success"; + event SSH::heuristic_successful_login(c); } - else if ( c$resp$size >= 200000000 ) - { - NOTICE([$note=Bytecount_Inconsistency, - $conn=c, - $msg="During byte counting in SSH analysis, an overly large value was seen.", - $sub=fmt("%d",c$resp$size)]); - } - - c$ssh$remote_location = location; - c$ssh$status = status; - c$ssh$direction = direction; - c$ssh$resp_size = c$resp$size; - - Log::write(SSH, c$ssh); # Set the "done" flag to prevent the watching event from rescheduling # after detection is done. - c$ssh$done; + c$ssh$done=T; - # Stop watching this connection, we don't care about it anymore. if ( skip_processing_after_detection ) { + # Stop watching this connection, we don't care about it anymore. skip_further_processing(c$id); set_record_packets(c$id, F); } } +event SSH::heuristic_successful_login(c: connection) &priority=-5 + { + Log::write(SSH, c$ssh); + } +event SSH::heuristic_failed_login(c: connection) &priority=-5 + { + Log::write(SSH, c$ssh); + } + event connection_state_remove(c: connection) &priority=-5 { if ( c?$ssh ) @@ -226,7 +145,7 @@ event ssh_watcher(c: connection) { local id = c$id; # don't go any further if this connection is gone already! - if ( !connection_exists(id) ) + if ( ! connection_exists(id) ) return; check_ssh_connection(c, F); @@ -244,5 +163,9 @@ event ssh_client_version(c: connection, version: string) &priority=5 { set_session(c); c$ssh$client = version; - schedule +15secs { ssh_watcher(c) }; + + # The heuristic detection for SSH relies on the ConnSize analyzer. + # Don't do the heuristics if it's disabled. + if ( use_conn_size_analyzer ) + schedule +15secs { ssh_watcher(c) }; } diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro new file mode 100644 index 0000000000..36e73bfa59 --- /dev/null +++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro @@ -0,0 +1,79 @@ + +module SSH; + +export { + redef enum Notice::Type += { + ## Indicates that a host has been identified as crossing the + ## :bro:id:`password_guesses_limit` threshold with heuristically + ## determined failed logins. + Password_Guessing, + ## Indicates that a host previously identified as a "password guesser" + ## has now had a heuristically successful login attempt. + Login_By_Password_Guesser, + }; + + ## The number of failed SSH connections before a host is designated as + ## guessing passwords. + const password_guesses_limit = 30 &redef; + + ## The amount of time to remember presumed non-successful logins to build + ## model of a password guesser. + const guessing_timeout = 30 mins &redef; + + ## This value can be used to exclude hosts or entire networks from being + ## tracked as potential "guessers". There are cases where the success + ## heuristic fails and this acts as the whitelist. The index represents + ## client subnets and the yield value represents server subnets. + const ignore_guessers: table[subnet] of subnet &redef; + + ## Keeps count of how many rejections a host has had. + global password_rejections: table[addr] of TrackCount + &write_expire=guessing_timeout + &synchronized; + + ## Keeps track of hosts identified as guessing passwords. + global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized; +} + +event SSH::heuristic_successful_login(c: connection) + { + local id = c$id; + + # TODO: this should be migrated to the metrics framework. + if ( id$orig_h in password_rejections && + password_rejections[id$orig_h]$n > password_guesses_limit && + id$orig_h !in password_guessers ) + { + add password_guessers[id$orig_h]; + NOTICE([$note=Login_By_Password_Guesser, + $conn=c, + $n=password_rejections[id$orig_h]$n, + $msg=fmt("Successful SSH login by password guesser %s", id$orig_h), + $sub=fmt("%d failed logins", password_rejections[id$orig_h]$n)]); + } + } + +event SSH::heuristic_failed_login(c: connection) + { + local id = c$id; + + # presumed failure + if ( id$orig_h !in password_rejections ) + password_rejections[id$orig_h] = new_track_count(); + + # Track the number of rejections + # TODO: this should be migrated to the metrics framework. + if ( ! (id$orig_h in ignore_guessers && + id$resp_h in ignore_guessers[id$orig_h]) ) + ++password_rejections[id$orig_h]$n; + + if ( default_check_threshold(password_rejections[id$orig_h]) ) + { + add password_guessers[id$orig_h]; + NOTICE([$note=Password_Guessing, + $conn=c, + $msg=fmt("SSH password guessing by %s", id$orig_h), + $sub=fmt("%d apparently failed logins", password_rejections[id$orig_h]$n), + $n=password_rejections[id$orig_h]$n]); + } + } \ No newline at end of file diff --git a/scripts/policy/protocols/ssh/geo-data.bro b/scripts/policy/protocols/ssh/geo-data.bro new file mode 100644 index 0000000000..97bd0a5803 --- /dev/null +++ b/scripts/policy/protocols/ssh/geo-data.bro @@ -0,0 +1,39 @@ +##! This implements all of the additional information and geodata detections +##! for SSH analysis. + +module SSH; + +export { + redef enum Notice::Type += { + ## If an SSH login is seen to or from a "watched" country based on the + ## :bro:id:`SSH::watched_countries` variable then this notice will + ## be generated. + Login_From_Watched_Country, + }; + + ## The set of countries for which you'd like to throw notices upon + ## successful login + const watched_countries: set[string] = {"RO"} &redef; + + redef record Info += { + ## Add geographic data related to the "remote" host of the connection. + remote_location: geo_location &log &optional; + }; +} + +event SSH::heuristic_successful_login(c: connection) &priority=5 + { + local location: geo_location; + location = (c$ssh$direction == OUTBOUND) ? + lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h); + + # Add the location data to the SSH record. + c$ssh$remote_location = location; + + if ( location$country_code in watched_countries ) + { + NOTICE([$note=Login_From_Watched_Country, + $conn=c, + $msg=fmt("SSH login from watched country: %s", location$country_code)]); + } + } diff --git a/scripts/policy/protocols/ssh/interesting-hostnames.bro b/scripts/policy/protocols/ssh/interesting-hostnames.bro new file mode 100644 index 0000000000..cf6ab7e40a --- /dev/null +++ b/scripts/policy/protocols/ssh/interesting-hostnames.bro @@ -0,0 +1,50 @@ + +module SSH; + +export { + redef enum Notice::Type += { + ## Generated if a login originates from a host matched by the + ## :bro:id:`interesting_hostnames` regular expression. + Login_From_Interesting_Hostname, + ## Generated if a login goes to a host matched by the + ## :bro:id:`interesting_hostnames` regular expression. + Login_To_Interesting_Hostname, + }; + + ## Strange/bad host names to see successful SSH logins from or to. + const interesting_hostnames = + /^d?ns[0-9]*\./ | + /^smtp[0-9]*\./ | + /^mail[0-9]*\./ | + /^pop[0-9]*\./ | + /^imap[0-9]*\./ | + /^www[0-9]*\./ | + /^ftp[0-9]*\./ &redef; +} + +event SSH::heuristic_successful_login(c: connection) + { + # Check to see if this login came from an interesting hostname. + when ( local orig_hostname = lookup_addr(c$id$orig_h) ) + { + if ( interesting_hostnames in orig_hostname ) + { + NOTICE([$note=Login_From_Interesting_Hostname, + $conn=c, + $msg=fmt("Interesting login from hostname: %s", orig_hostname), + $sub=orig_hostname]); + } + } + # Check to see if this login went to an interesting hostname. + when ( local resp_hostname = lookup_addr(c$id$orig_h) ) + { + if ( interesting_hostnames in resp_hostname ) + { + NOTICE([$note=Login_To_Interesting_Hostname, + $conn=c, + $msg=fmt("Interesting login to hostname: %s", resp_hostname), + $sub=resp_hostname]); + } + } + } + diff --git a/scripts/policy/protocols/ssh/software.bro b/scripts/policy/protocols/ssh/software.bro index d40ad513c8..1aa3bce1a2 100644 --- a/scripts/policy/protocols/ssh/software.bro +++ b/scripts/policy/protocols/ssh/software.bro @@ -3,8 +3,8 @@ module SSH; export { redef enum Software::Type += { - SSH_SERVER, - SSH_CLIENT, + SERVER, + CLIENT, }; } @@ -12,7 +12,7 @@ event ssh_client_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, ""); - local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT); + local si = Software::parse(cleaned_version, c$id$orig_h, CLIENT); Software::found(c$id, si); } @@ -20,6 +20,6 @@ event ssh_server_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, ""); - local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER); + local si = Software::parse(cleaned_version, c$id$resp_h, SERVER); Software::found(c$id, si); } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index cd0d6634b2..24a0aa144b 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -45,6 +45,11 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Load the script to enable SSL/TLS certificate validation. @load protocols/ssl/validate-certs +# If you have libGeoIP support built in, do some geographic detections and logging. +@load protocols/ssh/geo-data +@load protocols/ssh/detect-bruteforcing +@load protocols/ssh/interesting-hostnames + # Uncomment this redef if you want to extract SMTP MIME entities for # some file types. The numbers given indicate how many bytes to extract for # the various mime types. From d201215359ad0475bde319373a6af79a64f42033 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 01:37:57 -0400 Subject: [PATCH 10/25] Added the profiling script. --- scripts/policy/misc/profiling.bro | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts/policy/misc/profiling.bro diff --git a/scripts/policy/misc/profiling.bro b/scripts/policy/misc/profiling.bro new file mode 100644 index 0000000000..457675b1d6 --- /dev/null +++ b/scripts/policy/misc/profiling.bro @@ -0,0 +1,19 @@ +##! Turns on profiling of Bro resource consumption. + +module Profiling; + +redef profiling_file = open_log_file("prof"); + +export { + ## Cheap profiling every 15 seconds. + redef profiling_interval = 15 secs &redef; +} + +# Expensive profiling every 5 minutes. +redef expensive_profiling_multiple = 20; + +event bro_init() + { + set_buf(profiling_file, F); + } + From b45c175147847f417ebcadd16dca1597ce2daee1 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 08:26:20 -0400 Subject: [PATCH 11/25] Split out more SMTP analysis functionality. --- scripts/base/protocols/smtp/main.bro | 53 +----------------- scripts/policy/protocols/smtp/blocklists.bro | 58 ++++++++++++++++++++ 2 files changed, 61 insertions(+), 50 deletions(-) create mode 100644 scripts/policy/protocols/smtp/blocklists.bro diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index 2b0991d8f9..02b282894c 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -4,13 +4,6 @@ module SMTP; export { redef enum Log::ID += { SMTP }; - redef enum Notice::Type += { - ## Indicates that the server sent a reply mentioning an SMTP block list. - BL_Error_Message, - ## Indicates the client's address is seen in the block list error message. - BL_Blocked_Host, - }; - type Info: record { ts: time &log; uid: string &log; @@ -59,26 +52,7 @@ export { ## ALL_HOSTS - always capture the entire path. ## NO_HOSTS - never capture the path. const mail_path_capture = ALL_HOSTS &redef; - - # This matches content in SMTP error messages that indicate some - # block list doesn't like the connection/mail. - const bl_error_messages = - /spamhaus\.org\// - | /sophos\.com\/security\// - | /spamcop\.net\/bl/ - | /cbl\.abuseat\.org\// - | /sorbs\.net\// - | /bsn\.borderware\.com\// - | /mail-abuse\.com\// - | /b\.barracudacentral\.com\// - | /psbl\.surriel\.com\// - | /antispam\.imp\.ch\// - | /dyndns\.com\/.*spam/ - | /rbl\.knology\.net\// - | /intercept\.datapacket\.net\// - | /uceprotect\.net\// - | /hostkarma\.junkemailfilter\.com\// &redef; - + global log_smtp: event(rec: Info); ## Configure the default ports for SMTP analysis. @@ -181,27 +155,6 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, # This continually overwrites, but we want the last reply, # so this actually works fine. c$smtp$last_reply = fmt("%d %s", code, msg); - - if ( code != 421 && code >= 400 ) - { - # Raise a notice when an SMTP error about a block list is discovered. - if ( bl_error_messages in msg ) - { - local note = BL_Error_Message; - local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h); - - # Determine if the originator's IP address is in the message. - local ips = find_ip_addresses(msg); - local text_ip = ""; - if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h ) - { - note = BL_Blocked_Host; - message = fmt("%s is on an SMTP block list", c$id$orig_h); - } - - NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]); - } - } } event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, @@ -256,7 +209,7 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5 else if ( h$name == "X-ORIGINATING-IP" ) { - local addresses = find_ip_addresses(h$name); + local addresses = find_ip_addresses(h$value); if ( 1 in addresses ) c$smtp$x_originating_ip = to_addr(addresses[1]); } @@ -274,7 +227,7 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=3 # If we've decided that we're done watching the received headers for # whatever reason, we're done. Could be due to only watching until # local addresses are seen in the received from headers. - if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from) + if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from ) return; local text_ip = find_address_in_smtp_header(h$value); diff --git a/scripts/policy/protocols/smtp/blocklists.bro b/scripts/policy/protocols/smtp/blocklists.bro new file mode 100644 index 0000000000..a3e75318bb --- /dev/null +++ b/scripts/policy/protocols/smtp/blocklists.bro @@ -0,0 +1,58 @@ + +@load base/protocols/smtp + +module SMTP; + +export { + redef enum Notice::Type += { + ## Indicates that the server sent a reply mentioning an SMTP block list. + Blocklist_Error_Message, + ## Indicates the client's address is seen in the block list error message. + Blocklist_Blocked_Host, + }; + + # This matches content in SMTP error messages that indicate some + # block list doesn't like the connection/mail. + const blocklist_error_messages = + /spamhaus\.org\// + | /sophos\.com\/security\// + | /spamcop\.net\/bl/ + | /cbl\.abuseat\.org\// + | /sorbs\.net\// + | /bsn\.borderware\.com\// + | /mail-abuse\.com\// + | /b\.barracudacentral\.com\// + | /psbl\.surriel\.com\// + | /antispam\.imp\.ch\// + | /dyndns\.com\/.*spam/ + | /rbl\.knology\.net\// + | /intercept\.datapacket\.net\// + | /uceprotect\.net\// + | /hostkarma\.junkemailfilter\.com\// &redef; + +} + +event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, + msg: string, cont_resp: bool) &priority=3 + { + if ( code >= 400 && code != 421 ) + { + # Raise a notice when an SMTP error about a block list is discovered. + if ( blocklist_error_messages in msg ) + { + local note = Blocklist_Error_Message; + local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h); + + # Determine if the originator's IP address is in the message. + local ips = find_ip_addresses(msg); + local text_ip = ""; + if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h ) + { + note = Blocklist_Blocked_Host; + message = fmt("%s is on an SMTP block list", c$id$orig_h); + } + + NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]); + } + } + } From 423769c61d563b08669973024d1d81f3260524b1 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 08:26:40 -0400 Subject: [PATCH 12/25] Updates to local.bro --- scripts/policy/protocols/http/detect-MHR.bro | 2 +- scripts/site/local.bro | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 11e1d9f87e..fd54a62aeb 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -1,7 +1,7 @@ ##! This script takes MD5 sums of files transferred over HTTP and checks them with ##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). ##! By default, not all file transfers will have MD5 sums calculated. Read the -##! documentation for the protocols/http/file-hash.bro script to see how to +##! documentation for the base/protocols/http/file-hash.bro script to see how to ##! configure which transfers will have hashes calculated. export { diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 24a0aa144b..f894a30432 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -22,6 +22,7 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Load all of the scripts that detect software in various protocols. @load protocols/http/software +#@load protocols/http/detect-webapps @load protocols/ftp/software @load protocols/smtp/software @load protocols/ssh/software @@ -45,11 +46,19 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Load the script to enable SSL/TLS certificate validation. @load protocols/ssl/validate-certs -# If you have libGeoIP support built in, do some geographic detections and logging. +# If you have libGeoIP support built in, do some geographic detections and +# logging for SSH traffic. @load protocols/ssh/geo-data +# Detect hosts doing SSH bruteforce attacks. @load protocols/ssh/detect-bruteforcing +# Detect logins using "interesting" hostnames. @load protocols/ssh/interesting-hostnames +# Detect MD5 sums in Team Cymru's Malware Hash Registry. +@load protocols/http/detect-MHR +# Detect SQL injection attacks +@load protocols/http/detect-sqli + # Uncomment this redef if you want to extract SMTP MIME entities for # some file types. The numbers given indicate how many bytes to extract for # the various mime types. From 240ae9790bbb7e1ff5f07e5f428fbe8e8ffc67c8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 14:59:01 -0400 Subject: [PATCH 13/25] Small updates for notice framework. - New ACTION_ADD_GEODATA to add geodata to notices in an extension field named remote_location. - Loading extend-email/hostnames by default now that it only does anything when the ACTION_EMAIL action is applied (finally). --- scripts/base/frameworks/notice/__load__.bro | 7 +-- .../frameworks/notice/actions/add-geodata.bro | 47 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 scripts/base/frameworks/notice/actions/add-geodata.bro diff --git a/scripts/base/frameworks/notice/__load__.bro b/scripts/base/frameworks/notice/__load__.bro index bbc1fcae0d..2cc93ee933 100644 --- a/scripts/base/frameworks/notice/__load__.bro +++ b/scripts/base/frameworks/notice/__load__.bro @@ -6,7 +6,8 @@ @load ./actions/drop @load ./actions/email_admin @load ./actions/page +@load ./actions/add-geodata -# Load the script to add hostnames to emails by default. -# NOTE: this exposes a memleak in async DNS lookups. -#@load ./extend-email/hostnames +# There shouldn't be any defaul toverhead from loading these since they +# *should* only do anything when notices have the ACTION_EMAIL action applied. +@load ./extend-email/hostnames diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro new file mode 100644 index 0000000000..71e9c6b490 --- /dev/null +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -0,0 +1,47 @@ +##! This script adds geographic location data to notices for the "remote" +##! host in a connection. It does make the assumption that one of the +##! addresses in a connection is "local" and one is "remote" which is +##! probably a safe assumption to make in most cases. If both addresses +##! are remote, it will use the $src address. + +module Notice; + +export { + redef enum Action += { + ## Indicates that the notice should have geodata added for the + ## "remote" host. :bro:id:`Site::local_nets` must be defined + ## in order for this to work. + ACTION_ADD_GEODATA + }; + + redef record Info += { + ## If libGeoIP support is built in, notices can have geographic + ## information attached to them. + remote_location: geo_location &log &optional; + }; + + ## Notice types which should have the "remote" location looked up. + ## If GeoIP support is not built in, this does nothing. + const lookup_location_types: set[Notice::Type] = {} &redef; + + ## Add a helper to the notice policy for looking up GeoIP data. + redef Notice::policy += { + [$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); }, + $priority = 10], + }; +} + +# This is handled at a high priority in case other notice handlers +# want to use the data. +event notice(n: Notice::Info) &priority=10 + { + if ( ACTION_ADD_GEODATA in n$actions && + |Site::local_nets| > 0 && + ! n?$remote_location ) + { + if ( n?$src && ! Site::is_local_addr(n$src) ) + n$remote_location = lookup_location(n$src); + else if ( n?$dst && ! Site::is_local_addr(n$dst) ) + n$remote_location = lookup_location(n$dst); + } + } \ No newline at end of file From cc258b29aa54ce4a6107b4ee0d669755b0912df4 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 11 Aug 2011 15:16:32 -0400 Subject: [PATCH 14/25] Single character bugfix for hostname notice email extension. --- scripts/base/frameworks/notice/extend-email/hostnames.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 83cdc4807d..500b8ea4bc 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -8,7 +8,7 @@ event Notice::notice(n: Notice::Info) &priority=10 return; # This should only be done for notices that are being sent to email. - if ( ACTION_EMAIL !in n$action ) + if ( ACTION_EMAIL !in n$actions ) return; local output = ""; From c436930acfb0746d044ae666ac23bae52b41cdf2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 12 Aug 2011 22:04:05 -0700 Subject: [PATCH 15/25] Functions can now be logged. The function's code is rendered as ASCII and included as a string. Closes #506. Note that I'm not sure if the formatting is as desired: should the LFs and tabs be rendered as \xXX or removed?. --- src/LogMgr.cc | 19 +++++++++++++++++- src/LogWriterAscii.cc | 1 + .../policy.frameworks.logging.types/ssh.log | Bin 165 -> 278 bytes .../btest/policy/frameworks/logging/types.bro | 12 ++++++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9f8c33a107..461238f025 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -89,7 +89,7 @@ bool LogField::Write(SerializationFormat* fmt) const LogVal::~LogVal() { - if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE) + if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) && present ) delete val.string_val; @@ -130,6 +130,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) case TYPE_ENUM: case TYPE_STRING: case TYPE_FILE: + case TYPE_FUNC: return true; case TYPE_RECORD: @@ -231,6 +232,7 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_ENUM: case TYPE_STRING: case TYPE_FILE: + case TYPE_FUNC: { val.string_val = new string; return fmt->Read(val.string_val, "string"); @@ -343,6 +345,7 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_ENUM: case TYPE_STRING: case TYPE_FILE: + case TYPE_FUNC: return fmt->Write(*val.string_val, "string"); case TYPE_TABLE: @@ -648,6 +651,11 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, // That's ok, we handle it below. } + else if ( t->Tag() == TYPE_FUNC ) + { + // That's ok, we handle it below. + } + else { reporter->Error("unsupported field type for log column"); @@ -1074,6 +1082,15 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; } + case TYPE_FUNC: + { + ODesc d; + const Func* f = val->AsFunc(); + f->Describe(&d); + lval->val.string_val = new string(d.Description()); + break; + } + case TYPE_TABLE: { ListVal* set = val->AsTableVal()->ConvertToPureList(); diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index ad2adbfee1..446d6c8d65 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -155,6 +155,7 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_ENUM: case TYPE_STRING: case TYPE_FILE: + case TYPE_FUNC: { int size = val->val.string_val->size(); if ( size ) diff --git a/testing/btest/Baseline/policy.frameworks.logging.types/ssh.log b/testing/btest/Baseline/policy.frameworks.logging.types/ssh.log index 02e5b6957931b9a559adf898cb06d37e27366d51..5666db73c61ea038aee21a2da9238340327da95f 100644 GIT binary patch delta 143 zcmZ3=IE`t7787UML@jH6Lt`UDBU2MgJri?dQ!~Sf#fJHu!NDF@R%!Y9F%<@h3=Gu@ z5IQqWLBl}7Mgc6FsRm<>zZ!LsX~c6sJO1C@P#1i(o1svb9_Q Dqnajl delta 29 kcmbQnw3Kmz) 0 ) + return "Foo"; + else + return "Bar"; + } + event bro_init() { Log::create_stream(SSH, [$columns=Log]); @@ -56,7 +65,8 @@ event bro_init() $ss=set("AA", "BB", "CC"), $se=empty_set, $vc=vector(10, 20, 30), - $ve=empty_vector + $ve=empty_vector, + $f=foo ]); } From cb31fd3bb9345bf9594d24adce58788be75bec4f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 12 Aug 2011 22:18:45 -0700 Subject: [PATCH 16/25] Logging's path_func now receives the log record as argument. Closes #555. --- scripts/base/frameworks/logging/main.bro | 18 +++++++++--- src/LogMgr.cc | 4 ++- .../ssh-new-default.log | 4 +-- .../output | 28 ++++++++++++------- .../policy/frameworks/logging/path-func.bro | 4 +-- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index e31f931de9..a90dd21984 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -33,10 +33,12 @@ export { ## ## id: The log stream. ## path: A suggested path value, which may be either the filter's ``path`` - ## if defined or a fall-back generated internally. + ## if defined or a fall-back generated internally. + ## rec: An instance of the streams's ``columns`` type with its + ## fields set to the values to logged. ## ## Returns: The path to be used for the filter. - global default_path_func: function(id: ID, path: string) : string &redef; + global default_path_func: function(id: ID, path: string, rec: any) : string &redef; ## Filter customizing logging. type Filter: record { @@ -71,7 +73,15 @@ export { ## different strings for separate calls, but be careful: it's ## easy to flood the disk by returning a new string for each ## connection ... - path_func: function(id: ID, path: string): string &optional; + ## + ## id: The log stream. + ## path: A suggested path value, which may be either the filter's ``path`` + ## if defined or a fall-back generated internally. + ## rec: An instance of the streams's ``columns`` type with its + ## fields set to the values to logged. + ## + ## Returns: The path to be used for the filter. + path_func: function(id: ID, path: string, rec: any): string &optional; ## Subset of column names to record. If not given, all ## columns are recorded. @@ -160,7 +170,7 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool return default_rotation_postprocessors[info$writer](info); } -function default_path_func(id: ID, path: string) : string +function default_path_func(id: ID, path: string, rec: any) : string { # TODO for Seth: Do what you want. :) return path; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 461238f025..4719d04a22 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -902,9 +902,10 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( filter->path_func ) { - val_list vl(2); + val_list vl(3); vl.append(id->Ref()); vl.append(filter->path_val->Ref()); + vl.append(columns->Ref()); Val* v = filter->path_func->Call(&vl); if ( ! v->Type()->Tag() == TYPE_STRING ) @@ -915,6 +916,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) } path = v->AsString()->CheckString(); + Unref(v); #ifdef DEBUG DBG_LOG(DBG_LOGGING, "Path function for filter '%s' on stream '%s' return '%s'", diff --git a/testing/btest/Baseline/policy.frameworks.logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/policy.frameworks.logging.adapt-filter/ssh-new-default.log index 469f2d1991..ee274bb0fa 100644 --- a/testing/btest/Baseline/policy.frameworks.logging.adapt-filter/ssh-new-default.log +++ b/testing/btest/Baseline/policy.frameworks.logging.adapt-filter/ssh-new-default.log @@ -1,3 +1,3 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299718503.40319 1.2.3.4 1234 2.3.4.5 80 success unknown -1299718503.40319 1.2.3.4 1234 2.3.4.5 80 failure US +1313212563.234939 1.2.3.4 1234 2.3.4.5 80 success unknown +1313212563.234939 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/policy.frameworks.logging.path-func/output b/testing/btest/Baseline/policy.frameworks.logging.path-func/output index 25e4ca6696..7e8acf5106 100644 --- a/testing/btest/Baseline/policy.frameworks.logging.path-func/output +++ b/testing/btest/Baseline/policy.frameworks.logging.path-func/output @@ -1,13 +1,21 @@ -static-prefix-0.log -static-prefix-1.log -static-prefix-2.log +static-prefix-0-BR.log +static-prefix-0-MX3.log +static-prefix-0-unknown.log +static-prefix-1-MX.log +static-prefix-1-US.log +static-prefix-2-MX2.log +static-prefix-2-UK.log # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299718503.05867 1.2.3.4 1234 2.3.4.5 80 success unknown -1299718503.05867 1.2.3.4 1234 2.3.4.5 80 success BR -1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX3 +1313212701.542245 1.2.3.4 1234 2.3.4.5 80 success BR # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure US -1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX +1313212701.542245 1.2.3.4 1234 2.3.4.5 80 failure MX3 # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure UK -1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX2 +1313212701.542245 1.2.3.4 1234 2.3.4.5 80 success unknown +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1313212701.542245 1.2.3.4 1234 2.3.4.5 80 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1313212701.542245 1.2.3.4 1234 2.3.4.5 80 failure US +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1313212701.542245 1.2.3.4 1234 2.3.4.5 80 failure MX2 +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1313212701.542245 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/policy/frameworks/logging/path-func.bro b/testing/btest/policy/frameworks/logging/path-func.bro index 79d96e1431..ade6aedbc9 100644 --- a/testing/btest/policy/frameworks/logging/path-func.bro +++ b/testing/btest/policy/frameworks/logging/path-func.bro @@ -21,11 +21,11 @@ export { global c = -1; -function path_func(id: Log::ID, path: string) : string +function path_func(id: Log::ID, path: string, rec: Log) : string { c = (c + 1) % 3; - return fmt("%s-%d", path, c); + return fmt("%s-%d-%s", path, c, rec$country); } event bro_init() From 46d3570bf5af366170b408a04e94062f03fb28ff Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 12 Aug 2011 22:29:38 -0700 Subject: [PATCH 17/25] Turning DNS errors into a warning. It seems these errors aren't Bro's fault, and in any case it's clearly not an internal error. This should finally solve the problem in #255. Closes #255. --- src/DNS_Mgr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 8776f69d55..e6bebda875 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1071,7 +1071,7 @@ void DNS_Mgr::Process() int status = nb_dns_activity(nb_dns, &r, err); if ( status < 0 ) - reporter->InternalError("NB-DNS error in DNS_Mgr::Process (%s)", err); + reporter->Warning("NB-DNS error in DNS_Mgr::Process (%s)", err); else if ( status > 0 ) { From 33b064bdb2ab19f02be8b21a0a828a9c6922fd24 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 12 Aug 2011 22:39:36 -0700 Subject: [PATCH 18/25] Fixing reporter's location tracking. Closes #492. --- src/Obj.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Obj.cc b/src/Obj.cc index 666a57ac93..dfa8ed0148 100644 --- a/src/Obj.cc +++ b/src/Obj.cc @@ -127,6 +127,7 @@ void BroObj::BadTag(const char* msg, const char* t1, const char* t2) const ODesc d; DoMsg(&d, out); reporter->FatalError("%s", d.Description()); + reporter->PopLocation(); } void BroObj::Internal(const char* msg) const @@ -134,6 +135,7 @@ void BroObj::Internal(const char* msg) const ODesc d; DoMsg(&d, msg); reporter->InternalError("%s", d.Description()); + reporter->PopLocation(); } void BroObj::InternalWarning(const char* msg) const @@ -141,6 +143,7 @@ void BroObj::InternalWarning(const char* msg) const ODesc d; DoMsg(&d, msg); reporter->InternalWarning("%s", d.Description()); + reporter->PopLocation(); } void BroObj::AddLocation(ODesc* d) const From 00de88f4cb03fb90a8e66203c94dd4689258af4d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 10 Aug 2011 12:28:36 -0500 Subject: [PATCH 19/25] Fix reporter using part of the actual message as a format string When not reporting via events, the final contents of the message buffer after formatting was being used as a format string to fprintf instead of writing out the actual string. --- src/Reporter.cc | 2 +- .../btest/Baseline/core.reporter-fmt-strings/output | 1 + testing/btest/core/reporter-fmt-strings.bro | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/core.reporter-fmt-strings/output create mode 100644 testing/btest/core/reporter-fmt-strings.bro diff --git a/src/Reporter.cc b/src/Reporter.cc index 4a8e35e650..053d6370d7 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -302,7 +302,7 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Conne s += buffer; s += "\n"; - fprintf(out, s.c_str()); + fprintf(out, "%s", s.c_str()); } if ( alloced ) diff --git a/testing/btest/Baseline/core.reporter-fmt-strings/output b/testing/btest/Baseline/core.reporter-fmt-strings/output new file mode 100644 index 0000000000..10a883cb5d --- /dev/null +++ b/testing/btest/Baseline/core.reporter-fmt-strings/output @@ -0,0 +1 @@ +error in /Users/jsiwek/tmp/bro/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s)) diff --git a/testing/btest/core/reporter-fmt-strings.bro b/testing/btest/core/reporter-fmt-strings.bro new file mode 100644 index 0000000000..0e0be77844 --- /dev/null +++ b/testing/btest/core/reporter-fmt-strings.bro @@ -0,0 +1,10 @@ +# The format string below should end up as a literal part of the reporter's +# error message to stderr and shouldn't be replaced internally. +# +# @TEST-EXEC-FAIL: bro %INPUT >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output + +event bro_init() +{ + event dont_interpret_this("%s"); +} From 03a73899a97360cd79c25054331d5fe87c1391e2 Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Thu, 11 Aug 2011 12:15:03 -0700 Subject: [PATCH 20/25] Fix ConnSize_Analyzer with ConnCompressor. The num_pkts and num_bytes_ip in endpoint are optional and should only be assigned to if ConnSize_Anlyzer is active. --- src/ConnCompressor.cc | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/ConnCompressor.cc b/src/ConnCompressor.cc index e173463205..2bf28fe06a 100644 --- a/src/ConnCompressor.cc +++ b/src/ConnCompressor.cc @@ -866,15 +866,10 @@ void ConnCompressor::Event(const PendingConn* pending, double t, if ( ConnSize_Analyzer::Available() ) { + // Fill in optional fields if ConnSize_Analyzer is on. orig_endp->Assign(2, new Val(pending->num_pkts, TYPE_COUNT)); orig_endp->Assign(3, new Val(pending->num_bytes_ip, TYPE_COUNT)); } - else - { - orig_endp->Assign(2, new Val(0, TYPE_COUNT)); - orig_endp->Assign(3, new Val(0, TYPE_COUNT)); - } - resp_endp->Assign(0, new Val(0, TYPE_COUNT)); resp_endp->Assign(1, new Val(resp_state, TYPE_COUNT)); @@ -900,14 +895,10 @@ void ConnCompressor::Event(const PendingConn* pending, double t, if ( ConnSize_Analyzer::Available() ) { + // Fill in optional fields if ConnSize_Analyzer is on resp_endp->Assign(2, new Val(pending->num_pkts, TYPE_COUNT)); resp_endp->Assign(3, new Val(pending->num_bytes_ip, TYPE_COUNT)); } - else - { - resp_endp->Assign(2, new Val(0, TYPE_COUNT)); - resp_endp->Assign(3, new Val(0, TYPE_COUNT)); - } DBG_LOG(DBG_COMPRESSOR, "%s swapped direction", fmt_conn_id(pending)); } From 560685f1c6527e7f83b5cdd947691f4eef14f737 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 11 Aug 2011 13:09:48 -0500 Subject: [PATCH 21/25] Fix redef'ing records with &default empty set fields. Attributes have state to track whether they're in a record and should apply to a record field, but this state wasn't being set for TypeDecls that are part of a redef'd record. Closes #460 --- src/parse.y | 8 ++++---- .../btest/Baseline/language.record-extension/output | 12 ++++++++++-- testing/btest/language/record-extension.bro | 2 ++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/parse.y b/src/parse.y index c3624bfd2d..7964fa1bcc 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1070,10 +1070,10 @@ decl: } | TOK_REDEF TOK_RECORD global_id TOK_ADD_TO - '{' { do_doc_token_start(); } type_decl_list '}' opt_attr ';' + '{' { ++in_record; do_doc_token_start(); } + type_decl_list + { --in_record; do_doc_token_stop(); } '}' opt_attr ';' { - do_doc_token_stop(); - if ( ! $3->Type() ) $3->Error("unknown identifier"); else @@ -1083,7 +1083,7 @@ decl: $3->Error("not a record type"); else { - const char* error = add_to->AddFields($7, $9); + const char* error = add_to->AddFields($7, $10); if ( error ) $3->Error(error); else if ( generate_documentation ) diff --git a/testing/btest/Baseline/language.record-extension/output b/testing/btest/Baseline/language.record-extension/output index 1e38084e0c..b60b1c93b6 100644 --- a/testing/btest/Baseline/language.record-extension/output +++ b/testing/btest/Baseline/language.record-extension/output @@ -1,2 +1,10 @@ -[a=21, b=, c=42, d=] -[a=21, b=, c=42, d=XXX] +[a=21, b=, myset={ + +}, c=42, d=, anotherset={ + +}] +[a=21, b=, myset={ + +}, c=42, d=XXX, anotherset={ + +}] diff --git a/testing/btest/language/record-extension.bro b/testing/btest/language/record-extension.bro index c05b9da5e8..21b704ca7a 100644 --- a/testing/btest/language/record-extension.bro +++ b/testing/btest/language/record-extension.bro @@ -4,11 +4,13 @@ type Foo: record { a: count; b: count &optional; + myset: set[count] &default=set(); }; redef record Foo += { c: count &default=42; d: count &optional; + anotherset: set[count] &default=set(); }; global f1: Foo = [$a=21]; From 6f060a58d6a5f8f38aa97f2ba94382a247956ec2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 12 Aug 2011 09:54:28 -0500 Subject: [PATCH 22/25] Fix vector initialization for lists of records with optional types. If possible the list elements now get promoted to the yield type of the vector. There was also a problem with the value returned by the record constructor expression's eval being completely unref'd since the vector element assignment function doesn't ref the element -- so I changed it to ref values if they just constructed before assigning them to the vector. Addresses #485. --- src/Expr.cc | 3 ++- .../language.vector-list-init-records/output | 3 +++ .../language/vector-list-init-records.bro | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/language.vector-list-init-records/output create mode 100644 testing/btest/language/vector-list-init-records.bro diff --git a/src/Expr.cc b/src/Expr.cc index 54cd8f6ff4..c4fbe5930a 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -5020,8 +5020,9 @@ Val* ListExpr::InitVal(const BroType* t, Val* aggr) const loop_over_list(exprs, i) { Expr* e = exprs[i]; + check_and_promote_expr(e, vec->Type()->AsVectorType()->YieldType()); Val* v = e->Eval(0); - if ( ! vec->Assign(i, v, e) ) + if ( ! vec->Assign(i, v->RefCnt() == 1 ? v->Ref() : v, e) ) { e->Error(fmt("type mismatch at index %d", i)); return 0; diff --git a/testing/btest/Baseline/language.vector-list-init-records/output b/testing/btest/Baseline/language.vector-list-init-records/output new file mode 100644 index 0000000000..eccb029b4a --- /dev/null +++ b/testing/btest/Baseline/language.vector-list-init-records/output @@ -0,0 +1,3 @@ +element 0 = [s=bar, o=check] +element 1 = [s=baz, o=] +[[s=bar, o=check], [s=baz, o=]] diff --git a/testing/btest/language/vector-list-init-records.bro b/testing/btest/language/vector-list-init-records.bro new file mode 100644 index 0000000000..ee2b78c4a5 --- /dev/null +++ b/testing/btest/language/vector-list-init-records.bro @@ -0,0 +1,20 @@ +# Initializing a vector with a list of records should promote elements as +# necessary to match the vector's yield type. + +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +type Foo: record { + s: string; + o: string &optional; +}; + +const v: vector of Foo = { + [$s="bar", $o="check"], + [$s="baz"] +}; + +for ( i in v ) + print fmt("element %d = %s", i, v[i]); + +print v; From a3147033e293b62c81fa121466b667130e0e1d02 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Sat, 13 Aug 2011 09:45:42 -0500 Subject: [PATCH 23/25] Update doc sources and touch up a few script comments. --- doc/scripts/DocSourcesList.cmake | 17 +++++++++++++++-- .../base/frameworks/cluster/nodes/manager.bro | 2 +- scripts/base/frameworks/cluster/nodes/proxy.bro | 2 +- .../base/frameworks/cluster/nodes/worker.bro | 2 +- scripts/base/frameworks/logging/main.bro | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 54783f61b3..8290d3d102 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -34,9 +34,8 @@ rest_target(${psd} base/frameworks/dpd/main.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/writers/ascii.bro) -rest_target(${psd} base/frameworks/metrics/conn-example.bro) -rest_target(${psd} base/frameworks/metrics/http-example.bro) rest_target(${psd} base/frameworks/metrics/main.bro) +rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro) rest_target(${psd} base/frameworks/notice/actions/drop.bro) rest_target(${psd} base/frameworks/notice/actions/email_admin.bro) rest_target(${psd} base/frameworks/notice/actions/page.bro) @@ -70,6 +69,8 @@ rest_target(${psd} base/protocols/mime/file-extract.bro) rest_target(${psd} base/protocols/mime/file-hash.bro) rest_target(${psd} base/protocols/mime/file-ident.bro) rest_target(${psd} base/protocols/rpc/base.bro) +rest_target(${psd} base/protocols/smtp/entities-excerpt.bro) +rest_target(${psd} base/protocols/smtp/entities.bro) rest_target(${psd} base/protocols/smtp/main.bro) rest_target(${psd} base/protocols/ssh/main.bro) rest_target(${psd} base/protocols/ssl/consts.bro) @@ -93,6 +94,9 @@ rest_target(${psd} policy/frameworks/control/controllee.bro) rest_target(${psd} policy/frameworks/control/controller.bro) rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro) rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro) +rest_target(${psd} policy/frameworks/metrics/conn-example.bro) +rest_target(${psd} policy/frameworks/metrics/http-example.bro) +rest_target(${psd} policy/frameworks/metrics/ssl-example.bro) rest_target(${psd} policy/frameworks/software/version-changes.bro) rest_target(${psd} policy/frameworks/software/vulnerable.bro) rest_target(${psd} policy/integration/barnyard2/base.bro) @@ -100,6 +104,7 @@ rest_target(${psd} policy/integration/barnyard2/event.bro) rest_target(${psd} policy/integration/barnyard2/types.bro) rest_target(${psd} policy/misc/analysis-groups.bro) rest_target(${psd} policy/misc/loaded-scripts.bro) +rest_target(${psd} policy/misc/profiling.bro) rest_target(${psd} policy/misc/trim-trace-file.bro) rest_target(${psd} policy/protocols/conn/known-hosts.bro) rest_target(${psd} policy/protocols/conn/known-services.bro) @@ -115,8 +120,12 @@ rest_target(${psd} policy/protocols/http/headers.bro) rest_target(${psd} policy/protocols/http/software.bro) rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro) rest_target(${psd} policy/protocols/http/var-extraction-uri.bro) +rest_target(${psd} policy/protocols/smtp/blocklists.bro) rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro) rest_target(${psd} policy/protocols/smtp/software.bro) +rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro) +rest_target(${psd} policy/protocols/ssh/geo-data.bro) +rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro) rest_target(${psd} policy/protocols/ssh/software.bro) rest_target(${psd} policy/protocols/ssl/known-certs.bro) rest_target(${psd} policy/protocols/ssl/validate-certs.bro) @@ -124,4 +133,8 @@ rest_target(${psd} policy/tuning/defaults/packet-fragments.bro) rest_target(${psd} policy/tuning/defaults/remove-high-volume-notices.bro) rest_target(${psd} policy/tuning/defaults/warnings.bro) rest_target(${psd} policy/tuning/track-all-assets.bro) +rest_target(${psd} site/local-manager.bro) +rest_target(${psd} site/local-proxy.bro) +rest_target(${psd} site/local-worker.bro) rest_target(${psd} site/local.bro) +rest_target(${psd} test-all-policy.bro) diff --git a/scripts/base/frameworks/cluster/nodes/manager.bro b/scripts/base/frameworks/cluster/nodes/manager.bro index b85c20ae77..78b9fb7788 100644 --- a/scripts/base/frameworks/cluster/nodes/manager.bro +++ b/scripts/base/frameworks/cluster/nodes/manager.bro @@ -10,7 +10,7 @@ @prefixes += cluster-manager -## Load the script for local site configuration for the manager node. +# Load the script for local site configuration for the manager node. @load site/local-manager ## Turn off remote logging since this is the manager and should only log here. diff --git a/scripts/base/frameworks/cluster/nodes/proxy.bro b/scripts/base/frameworks/cluster/nodes/proxy.bro index 60df27b452..8340bf1be8 100644 --- a/scripts/base/frameworks/cluster/nodes/proxy.bro +++ b/scripts/base/frameworks/cluster/nodes/proxy.bro @@ -1,7 +1,7 @@ @prefixes += cluster-proxy -## Load the script for local site configuration for proxy nodes. +# Load the script for local site configuration for proxy nodes. @load site/local-proxy ## The proxy only syncs state; does not forward events. diff --git a/scripts/base/frameworks/cluster/nodes/worker.bro b/scripts/base/frameworks/cluster/nodes/worker.bro index 53d5d53872..f534e0aecc 100644 --- a/scripts/base/frameworks/cluster/nodes/worker.bro +++ b/scripts/base/frameworks/cluster/nodes/worker.bro @@ -1,7 +1,7 @@ @prefixes += cluster-worker -## Load the script for local site configuration for the worker nodes. +# Load the script for local site configuration for the worker nodes. @load site/local-worker ## Don't do any local logging. diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index e31f931de9..adcf8f231f 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -33,7 +33,7 @@ export { ## ## id: The log stream. ## path: A suggested path value, which may be either the filter's ``path`` - ## if defined or a fall-back generated internally. + ## if defined or a fall-back generated internally. ## ## Returns: The path to be used for the filter. global default_path_func: function(id: ID, path: string) : string &redef; From 2ca791f110d30691d50d62689ce109dfc3574ae6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 13 Aug 2011 12:09:32 -0700 Subject: [PATCH 24/25] Updating submodule(s). --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 7cdd9c39d9..a3a9410ded 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 7cdd9c39d97c2984293fbe4a6dbe9ac0b33ecbfa +Subproject commit a3a9410dedc842f6bb9859642f334ed354633b57 diff --git a/aux/bro-aux b/aux/bro-aux index 86990f1640..d68b98bb99 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 86990f1640d986e39d5bb1287dbeb03b59a464f0 +Subproject commit d68b98bb995a105b257f805ec4ff22c4929c7476 diff --git a/aux/broccoli b/aux/broccoli index 45f5772400..03e6d398ed 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 45f577240089d63dd0dc58be564280725a97acec +Subproject commit 03e6d398edf422140ba9f50e6fabbec33ee2f3cb diff --git a/aux/broctl b/aux/broctl index dbbe6c81ef..fbc5e86caf 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit dbbe6c81ef40666338c950d8f69dc8597f2adc70 +Subproject commit fbc5e86caf0ee228c72085166c8bb40d74a26fb3 diff --git a/aux/btest b/aux/btest index ab78a66dd7..d1c620d98c 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit ab78a66dd782f165ddf921faaf1f065b2f987481 +Subproject commit d1c620d98ce9d9c0b203314108b413784965d2ed From 2af9d9bc20e6aa930b4f243b5a386fbb07150a70 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 13 Aug 2011 12:15:13 -0700 Subject: [PATCH 25/25] Updating submodule(s). --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index fbc5e86caf..c39622855e 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit fbc5e86caf0ee228c72085166c8bb40d74a26fb3 +Subproject commit c39622855e3c3a5cc94c7376f86184ed1db1939a