diff --git a/CHANGES b/CHANGES index 5be56b8f18..ce7d654166 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,15 @@ +2.0-beta-139 | 2011-12-19 07:06:29 -0800 + + * The hostname notice email extension works now, plus a general + mechanism for adding delayed information to notices. (Seth Hall) + + * Fix &default fields in records not being initialized in coerced + assignments. Addresses #722. (Jon Siwek) + + * Make log headers include the type of data stored inside a set or + vector ("vector[string]"). (Bernhard Amann) + 2.0-beta-126 | 2011-12-18 15:18:05 -0800 * DNS updates. (Seth Hall) diff --git a/VERSION b/VERSION index f9bcabd87f..03329dc416 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-126 +2.0-beta-139 diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index a73810c726..17a597678d 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -2,31 +2,46 @@ module Notice; -# This probably doesn't actually work due to the async lookup_addr. +# We have to store references to the notices here because the when statement +# clones the frame which doesn't give us access to modify values outside +# of it's execution scope. (we get a clone of the notice instead of a +# reference to the original notice) +global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs; + event Notice::notice(n: Notice::Info) &priority=10 { if ( ! n?$src && ! n?$dst ) return; - + # This should only be done for notices that are being sent to email. if ( ACTION_EMAIL !in n$actions ) return; - + + # I'm not recovering gracefully from the when statements because I want + # the notice framework to detect that something has exceeded the maximum + # allowed email delay and tell the user. + local uid = unique_id(""); + tmp_notice_storage[uid] = n; + local output = ""; if ( n?$src ) { + add n$email_delay_tokens["hostnames-src"]; when ( local src_name = lookup_addr(n$src) ) { - output = string_cat("orig_h/src hostname: ", src_name, "\n"); - n$email_body_sections[|n$email_body_sections|] = output; + output = string_cat("orig/src hostname: ", src_name, "\n"); + tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output; + delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-src"]; } } if ( n?$dst ) { + add n$email_delay_tokens["hostnames-dst"]; when ( local dst_name = lookup_addr(n$dst) ) { - output = string_cat("resp_h/dst hostname: ", dst_name, "\n"); - n$email_body_sections[|n$email_body_sections|] = output; + output = string_cat("resp/dst hostname: ", dst_name, "\n"); + tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output; + delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-dst"]; } } } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 7d98c6464c..72b87a8044 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -8,9 +8,9 @@ module Notice; export { - redef enum Log::ID += { + redef enum Log::ID += { ## This is the primary logging stream for notices. - LOG, + LOG, ## This is the notice policy auditing log. It records what the current ## notice policy is at Bro init time. POLICY_LOG, @@ -18,25 +18,25 @@ export { ALARM_LOG, }; - ## Scripts creating new notices need to redef this enum to add their own + ## Scripts creating new notices need to redef this enum to add their own ## specific notice types which would then get used when they call the ## :bro:id:`NOTICE` function. The convention is to give a general category ## along with the specific notice separating words with underscores and using ## leading capitals on each word except for abbreviations which are kept in - ## all capitals. For example, SSH::Login is for heuristically guessed + ## all capitals. For example, SSH::Login is for heuristically guessed ## successful SSH logins. type Type: enum { ## Notice reporting a count of how often a notice occurred. Tally, }; - + ## These are values representing actions that can be taken with notices. type Action: enum { ## Indicates that there is no action to be taken. ACTION_NONE, ## Indicates that the notice should be sent to the notice logging stream. ACTION_LOG, - ## Indicates that the notice should be sent to the email address(es) + ## Indicates that the notice should be sent to the email address(es) ## configured in the :bro:id:`Notice::mail_dest` variable. ACTION_EMAIL, ## Indicates that the notice should be alarmed. A readable ASCII @@ -47,30 +47,30 @@ export { ## duplicate notice suppression that the notice framework does. ACTION_NO_SUPPRESS, }; - - ## The notice framework is able to do automatic notice supression by + + ## The notice framework is able to do automatic notice supression by ## utilizing the $identifier field in :bro:type:`Info` records. ## Set this to "0secs" to completely disable automated notice suppression. const default_suppression_interval = 1hrs &redef; - + type Info: record { ts: time &log &optional; uid: string &log &optional; id: conn_id &log &optional; - + ## These are shorthand ways of giving the uid and id to a notice. The ## reference to the actual connection will be deleted after applying ## the notice policy. conn: connection &optional; iconn: icmp_conn &optional; - + ## The :bro:enum:`Notice::Type` of the notice. note: Type &log; ## The human readable message for the notice. msg: string &log &optional; ## The human readable sub-message. sub: string &log &optional; - + ## Source address, if we don't have a :bro:type:`conn_id`. src: addr &log &optional; ## Destination address. @@ -79,33 +79,39 @@ export { p: port &log &optional; ## Associated count, or perhaps a status code. n: count &log &optional; - + ## Peer that raised this notice. src_peer: event_peer &optional; ## Textual description for the peer that raised this notice. peer_descr: string &log &optional; - + ## The actions which have been applied to this notice. actions: set[Notice::Action] &log &optional; - + ## These are policy items that returned T and applied their action ## to the notice. policy_items: set[count] &log &optional; - + ## By adding chunks of text into this element, other scripts can ## expand on notices that are being emailed. The normal way to add text ## is to extend the vector by handling the :bro:id:`Notice::notice` ## event and modifying the notice in place. - email_body_sections: vector of string &default=vector(); - + email_body_sections: vector of string &optional; + + ## Adding a string "token" to this set will cause the notice framework's + ## built-in emailing functionality to delay sending the email until + ## either the token has been removed or the email has been delayed + ## for :bro:id:`max_email_delay`. + email_delay_tokens: set[string] &optional; + ## This field is to be provided when a notice is generated for the ## purpose of deduplicating notices. The identifier string should - ## be unique for a single instance of the notice. This field should be - ## filled out in almost all cases when generating notices to define + ## be unique for a single instance of the notice. This field should be + ## filled out in almost all cases when generating notices to define ## when a notice is conceptually a duplicate of a previous notice. - ## - ## For example, an SSL certificate that is going to expire soon should - ## always have the same identifier no matter the client IP address + ## + ## For example, an SSL certificate that is going to expire soon should + ## always have the same identifier no matter the client IP address ## that connected and resulted in the certificate being exposed. In ## this case, the resp_h, resp_p, and hash of the certificate would be ## used to create this value. The hash of the cert is included @@ -114,19 +120,19 @@ export { ## Another example might be a host downloading a file which triggered ## a notice because the MD5 sum of the file it downloaded was known ## by some set of intelligence. In that case, the orig_h (client) - ## and MD5 sum would be used in this field to dedup because if the + ## and MD5 sum would be used in this field to dedup because if the ## same file is downloaded over and over again you really only want to ## know about it a single time. This makes it possible to send those ## notices to email without worrying so much about sending thousands ## of emails. identifier: string &optional; - + ## This field indicates the length of time that this - ## unique notice should be suppressed. This field is automatically + ## unique notice should be suppressed. This field is automatically ## filled out and should not be written to by any other script. suppress_for: interval &log &optional; }; - + ## Ignored notice types. const ignored_types: set[Notice::Type] = {} &redef; ## Emailed notice types. @@ -135,10 +141,10 @@ export { const alarmed_types: set[Notice::Type] = {} &redef; ## Types that should be suppressed for the default suppression interval. const not_suppressed_types: set[Notice::Type] = {} &redef; - ## This table can be used as a shorthand way to modify suppression + ## This table can be used as a shorthand way to modify suppression ## intervals for entire notice types. const type_suppression_intervals: table[Notice::Type] of interval = {} &redef; - + ## This is the record that defines the items that make up the notice policy. type PolicyItem: record { ## This is the exact positional order in which the :bro:type:`PolicyItem` @@ -149,20 +155,20 @@ export { priority: count &log &default=5; ## An action given to the notice if the predicate return true. action: Notice::Action &log &default=ACTION_NONE; - ## The pred (predicate) field is a function that returns a boolean T - ## or F value. If the predicate function return true, the action in - ## this record is applied to the notice that is given as an argument - ## to the predicate function. If no predicate is supplied, it's + ## The pred (predicate) field is a function that returns a boolean T + ## or F value. If the predicate function return true, the action in + ## this record is applied to the notice that is given as an argument + ## to the predicate function. If no predicate is supplied, it's ## assumed that the PolicyItem always applies. pred: function(n: Notice::Info): bool &log &optional; - ## Indicates this item should terminate policy processing if the + ## Indicates this item should terminate policy processing if the ## predicate returns T. halt: bool &log &default=F; ## This defines the length of time that this particular notice should ## be supressed. suppress_for: interval &log &optional; }; - + ## This is the where the :bro:id:`Notice::policy` is defined. All notice ## processing is done through this variable. const policy: set[PolicyItem] = { @@ -177,66 +183,68 @@ export { [$pred(n: Notice::Info) = { return (n$note in Notice::emailed_types); }, $action = ACTION_EMAIL, $priority = 8], - [$pred(n: Notice::Info) = { - if (n$note in Notice::type_suppression_intervals) + [$pred(n: Notice::Info) = { + if (n$note in Notice::type_suppression_intervals) { n$suppress_for=Notice::type_suppression_intervals[n$note]; return T; } - return F; + return F; }, $action = ACTION_NONE, $priority = 8], [$action = ACTION_LOG, $priority = 0], } &redef; - + ## Local system sendmail program. const sendmail = "/usr/sbin/sendmail" &redef; ## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action ## or to send bulk alarm logs on rotation with :bro:enum:`ACTION_ALARM`. const mail_dest = "" &redef; - + ## Address that emails will be from. const mail_from = "Big Brother " &redef; ## Reply-to address used in outbound email. const reply_to = "" &redef; ## Text string prefixed to the subject of all emails sent out. const mail_subject_prefix = "[Bro]" &redef; + ## The maximum amount of time a plugin can delay email from being sent. + const max_email_delay = 15secs &redef; ## A log postprocessing function that implements emailing the contents ## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`. ## The rotated log is removed upon being sent. global log_mailing_postprocessor: function(info: Log::RotationInfo): bool; - ## This is the event that is called as the entry point to the - ## notice framework by the global :bro:id:`NOTICE` function. By the time + ## This is the event that is called as the entry point to the + ## notice framework by the global :bro:id:`NOTICE` function. By the time ## this event is generated, default values have already been filled out in - ## the :bro:type:`Notice::Info` record and synchronous functions in the + ## the :bro:type:`Notice::Info` record and synchronous functions in the ## :bro:id:`Notice:sync_functions` have already been called. The notice ## policy has also been applied. global notice: event(n: Info); - ## This is a set of functions that provide a synchronous way for scripts + ## This is a set of functions that provide a synchronous way for scripts ## extending the notice framework to run before the normal event based ## notice pathway that most of the notice framework takes. This is helpful ## in cases where an action against a notice needs to happen immediately ## and can't wait the short time for the event to bubble up to the top of - ## the event queue. An example is the IP address dropping script that - ## can block IP addresses that have notices generated because it + ## the event queue. An example is the IP address dropping script that + ## can block IP addresses that have notices generated because it ## needs to operate closer to real time than the event queue allows it to. - ## Normally the event based extension model using the + ## Normally the event based extension model using the ## :bro:id:`Notice::notice` event will work fine if there aren't harder ## real time constraints. const sync_functions: set[function(n: Notice::Info)] = set() &redef; - + ## This event is generated when a notice begins to be suppressed. global begin_suppression: event(n: Notice::Info); ## This event is generated on each occurence of an event being suppressed. global suppressed: event(n: Notice::Info); ## This event is generated when a notice stops being suppressed. global end_suppression: event(n: Notice::Info); - + ## Call this function to send a notice in an email. It is already used ## by default with the built in :bro:enum:`ACTION_EMAIL` and ## :bro:enum:`ACTION_PAGE` actions. @@ -248,12 +256,12 @@ export { ## dest: recipient string to use for the mail ## Returns: a string of mail headers to which an email body can be appended global email_headers: function(subject_desc: string, dest: string): string; - + ## This event can be handled to access the :bro:type:`Info` ## record as it is sent on to the logging framework. global log_notice: event(rec: Info); - - ## This is an internal wrapper for the global NOTICE function. Please + + ## This is an internal wrapper for the global NOTICE function. Please ## disregard. global internal_NOTICE: function(n: Notice::Info); } @@ -264,22 +272,22 @@ function per_notice_suppression_interval(t: table[Notice::Type, string] of Notic local n: Notice::Type; local s: string; [n,s] = idx; - + local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts); if ( suppress_time < 0secs ) suppress_time = 0secs; - + # If there is no more suppression time left, the notice needs to be sent # to the end_suppression event. if ( suppress_time == 0secs ) event Notice::end_suppression(t[n,s]); - + return suppress_time; } -# This is the internally maintained notice suppression table. It's +# This is the internally maintained notice suppression table. It's # indexed on the Notice::Type and the $identifier field from the notice. -global suppressing: table[Type, string] of Notice::Info = {} +global suppressing: table[Type, string] of Notice::Info = {} &create_expire=0secs &expire_func=per_notice_suppression_interval; @@ -306,7 +314,7 @@ function log_mailing_postprocessor(info: Log::RotationInfo): bool event bro_init() &priority=5 { Log::create_stream(Notice::LOG, [$columns=Info, $ev=log_notice]); - + Log::create_stream(Notice::ALARM_LOG, [$columns=Notice::Info]); # If Bro is configured for mailing notices, set up mailing for alarms. # Make sure that this alarm log is also output as text so that it can @@ -347,25 +355,49 @@ function email_headers(subject_desc: string, dest: string): string return header_text; } +event delay_sending_email(n: Notice::Info, dest: string, extend: bool) + { + email_notice_to(n, dest, extend); + } + function email_notice_to(n: Notice::Info, dest: string, extend: bool) { if ( reading_traces() || dest == "" ) return; - + + if ( extend ) + { + if ( |n$email_delay_tokens| > 0 ) + { + # If we still are within the max_email_delay, keep delaying. + if ( n$ts + max_email_delay > network_time() ) + { + schedule 1sec { delay_sending_email(n, dest, extend) }; + return; + } + else + { + event reporter_info(network_time(), + fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens), + ""); + } + } + } + local email_text = email_headers(fmt("%s", n$note), dest); - + # First off, finish the headers and include the human readable messages # then leave a blank line after the message. email_text = string_cat(email_text, "\nMessage: ", n$msg); if ( n?$sub ) email_text = string_cat(email_text, "\nSub-message: ", n$sub); - + email_text = string_cat(email_text, "\n\n"); - + # Next, add information about the connection if it exists. if ( n?$id ) { - email_text = string_cat(email_text, "Connection: ", + email_text = string_cat(email_text, "Connection: ", fmt("%s", n$id$orig_h), ":", fmt("%d", n$id$orig_p), " -> ", fmt("%s", n$id$resp_h), ":", fmt("%d", n$id$resp_p), "\n"); if ( n?$uid ) @@ -373,17 +405,18 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) } else if ( n?$src ) email_text = string_cat(email_text, "Address: ", fmt("%s", n$src), "\n"); - + # Add the extended information if it's requested. if ( extend ) { + email_text = string_cat(email_text, "\nEmail Extensions\n"); + email_text = string_cat(email_text, "----------------\n"); for ( i in n$email_body_sections ) { - email_text = string_cat(email_text, "******************\n"); email_text = string_cat(email_text, n$email_body_sections[i], "\n"); } } - + email_text = string_cat(email_text, "\n\n--\n[Automatically generated]\n\n"); piped_exec(fmt("%s -t -oi", sendmail), email_text); } @@ -396,10 +429,10 @@ event notice(n: Notice::Info) &priority=-5 Log::write(Notice::LOG, n); if ( ACTION_ALARM in n$actions ) Log::write(Notice::ALARM_LOG, n); - + # Normally suppress further notices like this one unless directed not to. # n$identifier *must* be specified for suppression to function at all. - if ( n?$identifier && + if ( n?$identifier && ACTION_NO_SUPPRESS !in n$actions && [n$note, n$identifier] !in suppressing && n$suppress_for != 0secs ) @@ -408,8 +441,8 @@ event notice(n: Notice::Info) &priority=-5 event Notice::begin_suppression(n); } } - -## This determines if a notice is being suppressed. It is only used + +## This determines if a notice is being suppressed. It is only used ## internally as part of the mechanics for the global NOTICE function. function is_being_suppressed(n: Notice::Info): bool { @@ -421,7 +454,7 @@ function is_being_suppressed(n: Notice::Info): bool else return F; } - + # Executes a script with all of the notice fields put into the # new process' environment as "BRO_ARG_" variables. function execute_with_notice(cmd: string, n: Notice::Info) @@ -430,9 +463,9 @@ function execute_with_notice(cmd: string, n: Notice::Info) #local tgs = tags(n); #system_env(cmd, tags); } - -# This is run synchronously as a function before all of the other -# notice related functions and events. It also modifies the + +# This is run synchronously as a function before all of the other +# notice related functions and events. It also modifies the # :bro:type:`Notice::Info` record in place. function apply_policy(n: Notice::Info) { @@ -447,7 +480,7 @@ function apply_policy(n: Notice::Info) if ( ! n?$uid ) n$uid = n$conn$uid; } - + if ( n?$id ) { if ( ! n?$src ) @@ -469,15 +502,20 @@ function apply_policy(n: Notice::Info) if ( ! n?$src_peer ) n$src_peer = get_event_peer(); if ( ! n?$peer_descr ) - n$peer_descr = n$src_peer?$descr ? + n$peer_descr = n$src_peer?$descr ? n$src_peer$descr : fmt("%s", n$src_peer$host); - + if ( ! n?$actions ) n$actions = set(); - + + if ( ! n?$email_body_sections ) + n$email_body_sections = vector(); + if ( ! n?$email_delay_tokens ) + n$email_delay_tokens = set(); + if ( ! n?$policy_items ) n$policy_items = set(); - + for ( i in ordered_policy ) { # If there's no predicate or the predicate returns F. @@ -485,51 +523,51 @@ function apply_policy(n: Notice::Info) { add n$actions[ordered_policy[i]$action]; add n$policy_items[int_to_count(i)]; - - # If the predicate matched and there was a suppression interval, + + # If the predicate matched and there was a suppression interval, # apply it to the notice now. if ( ordered_policy[i]?$suppress_for ) n$suppress_for = ordered_policy[i]$suppress_for; - + # If the policy item wants to halt policy processing, do it now! if ( ordered_policy[i]$halt ) break; } } - + # Apply the suppression time after applying the policy so that policy - # items can give custom suppression intervals. If there is no + # items can give custom suppression intervals. If there is no # suppression interval given yet, the default is applied. if ( ! n?$suppress_for ) n$suppress_for = default_suppression_interval; - + # Delete the connection record if it's there so we aren't sending that - # to remote machines. It can cause problems due to the size of the + # to remote machines. It can cause problems due to the size of the # connection record. if ( n?$conn ) delete n$conn; if ( n?$iconn ) delete n$iconn; } - -# Create the ordered notice policy automatically which will be used at runtime + +# Create the ordered notice policy automatically which will be used at runtime # for prioritized matching of the notice policy. event bro_init() &priority=10 { # Create the policy log here because it's only written to in this handler. Log::create_stream(Notice::POLICY_LOG, [$columns=PolicyItem]); - + local tmp: table[count] of set[PolicyItem] = table(); for ( pi in policy ) { if ( pi$priority < 0 || pi$priority > 10 ) Reporter::fatal("All Notice::PolicyItem priorities must be within 0 and 10"); - + if ( pi$priority !in tmp ) tmp[pi$priority] = set(); add tmp[pi$priority][pi]; } - + local rev_count = vector(10,9,8,7,6,5,4,3,2,1,0); for ( i in rev_count ) { @@ -545,7 +583,7 @@ event bro_init() &priority=10 } } } - + function internal_NOTICE(n: Notice::Info) { # Suppress this notice if necessary.