From a7db43efb1c2cc44fab7b3a5267ebdf23a867d10 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 26 Jul 2011 16:03:03 -0500 Subject: [PATCH 1/5] Fixes for email_notice_to() function. Newline characters need escaping so that an echo command can interpret them into a newline in the output piped to sendmail, else sendmail can't parse the headers correctly. I made the echo command a configurable option of the notice framework in case `echo -e` is overshadowed by some shell-specific implementation that doesn't support that option for interpreting char sequences. --- policy/frameworks/notice/base/main.bro | 28 +++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/policy/frameworks/notice/base/main.bro b/policy/frameworks/notice/base/main.bro index 2ed7fc1690..38fa41a4c5 100644 --- a/policy/frameworks/notice/base/main.bro +++ b/policy/frameworks/notice/base/main.bro @@ -90,7 +90,8 @@ export { ## By adding chunks of text into this element, other scripts can ## expand on notices that are being emailed. The normal way to add text ## is to extend the vector by handling the :bro:id:`Notice::notice` - ## event and modifying the notice in place. + ## event and modifying the notice in place. For newline characters + ## to be rendered properly, they must be escaped as "\\n". email_body_sections: vector of string &default=vector(); }; @@ -138,6 +139,9 @@ export { ## Local system sendmail program. const sendmail = "/usr/sbin/sendmail" &redef; + ## Local system echo/printf program that supports interpretted character + ## sequences (e.g. "\n"), for use in with conjunction with sendmail. + const echocmd = "echo -e" &redef; ## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action. const mail_dest = "" &redef; @@ -228,33 +232,33 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) return; local email_text = cat( - "From: ", mail_from, "\n", - "Subject: ", mail_subject_prefix, " ", n$note, "\n", - "To: ", dest, "\n", + "From: ", mail_from, "\\n", + "Subject: ", mail_subject_prefix, " ", n$note, "\\n", + "To: ", dest, "\\n", # TODO: BiF to get version (the resource_usage Bif seems like overkill). - "User-Agent: Bro-IDS/?.?.?\n"); + "User-Agent: Bro-IDS/?.?.?\\n"); if ( reply_to != "" ) - email_text = cat(email_text, "Reply-To: ", reply_to, "\n"); + email_text = cat(email_text, "Reply-To: ", reply_to, "\\n"); # The notice emails always start off with the human readable message. - email_text = cat(email_text, "\n", n$msg, "\n"); + email_text = cat(email_text, "\\n", n$msg, "\\n"); # Add the extended information if it's requested. if ( extend ) { for ( i in n$email_body_sections ) { - email_text = cat(email_text, "******************\n"); - email_text = cat(email_text, n$email_body_sections[i], "\n"); + email_text = cat(email_text, "******************\\n"); + email_text = cat(email_text, n$email_body_sections[i], "\\n"); } } - email_text = cat(email_text, "\n\n--\n[Automatically generated]\n\n"); + email_text = cat(email_text, "\\n\\n--\\n[Automatically generated]\\n\\n"); local mail_cmd = - fmt("echo \"%s\" | %s -t -oi %s", - str_shell_escape(email_text), sendmail); + fmt("%s \"%s\" | %s -t -oi", + echocmd, str_shell_escape(email_text), sendmail); system(mail_cmd); } From 73bb046b97e843d2229c96abe70b96cafbe03983 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 27 Jul 2011 08:14:28 -0700 Subject: [PATCH 2/5] Revert "Fixes for email_notice_to() function." This reverts commit a7db43efb1c2cc44fab7b3a5267ebdf23a867d10. We decided to redo the interface between Bro and sendmail. --- policy/frameworks/notice/base/main.bro | 28 +++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/policy/frameworks/notice/base/main.bro b/policy/frameworks/notice/base/main.bro index 38fa41a4c5..2ed7fc1690 100644 --- a/policy/frameworks/notice/base/main.bro +++ b/policy/frameworks/notice/base/main.bro @@ -90,8 +90,7 @@ export { ## By adding chunks of text into this element, other scripts can ## expand on notices that are being emailed. The normal way to add text ## is to extend the vector by handling the :bro:id:`Notice::notice` - ## event and modifying the notice in place. For newline characters - ## to be rendered properly, they must be escaped as "\\n". + ## event and modifying the notice in place. email_body_sections: vector of string &default=vector(); }; @@ -139,9 +138,6 @@ export { ## Local system sendmail program. const sendmail = "/usr/sbin/sendmail" &redef; - ## Local system echo/printf program that supports interpretted character - ## sequences (e.g. "\n"), for use in with conjunction with sendmail. - const echocmd = "echo -e" &redef; ## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action. const mail_dest = "" &redef; @@ -232,33 +228,33 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) return; local email_text = cat( - "From: ", mail_from, "\\n", - "Subject: ", mail_subject_prefix, " ", n$note, "\\n", - "To: ", dest, "\\n", + "From: ", mail_from, "\n", + "Subject: ", mail_subject_prefix, " ", n$note, "\n", + "To: ", dest, "\n", # TODO: BiF to get version (the resource_usage Bif seems like overkill). - "User-Agent: Bro-IDS/?.?.?\\n"); + "User-Agent: Bro-IDS/?.?.?\n"); if ( reply_to != "" ) - email_text = cat(email_text, "Reply-To: ", reply_to, "\\n"); + email_text = cat(email_text, "Reply-To: ", reply_to, "\n"); # The notice emails always start off with the human readable message. - email_text = cat(email_text, "\\n", n$msg, "\\n"); + email_text = cat(email_text, "\n", n$msg, "\n"); # Add the extended information if it's requested. if ( extend ) { for ( i in n$email_body_sections ) { - email_text = cat(email_text, "******************\\n"); - email_text = cat(email_text, n$email_body_sections[i], "\\n"); + email_text = cat(email_text, "******************\n"); + email_text = cat(email_text, n$email_body_sections[i], "\n"); } } - email_text = cat(email_text, "\\n\\n--\\n[Automatically generated]\\n\\n"); + email_text = cat(email_text, "\n\n--\n[Automatically generated]\n\n"); local mail_cmd = - fmt("%s \"%s\" | %s -t -oi", - echocmd, str_shell_escape(email_text), sendmail); + fmt("echo \"%s\" | %s -t -oi %s", + str_shell_escape(email_text), sendmail); system(mail_cmd); } From 19dab4fdda97ec81e196526bab41b8f6b20af5a6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 27 Jul 2011 13:52:24 -0500 Subject: [PATCH 3/5] Add new piped_exec BiF. And changing email_notice_to() function in notice framework to interface with sendmail through it. --- policy/frameworks/notice/base/main.bro | 20 ++++++++----------- src/bro.bif | 20 +++++++++++++++++++ testing/btest/Baseline/bifs.piped_exec/output | 2 ++ testing/btest/bifs/piped_exec.bro | 6 ++++++ 4 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 testing/btest/Baseline/bifs.piped_exec/output create mode 100644 testing/btest/bifs/piped_exec.bro diff --git a/policy/frameworks/notice/base/main.bro b/policy/frameworks/notice/base/main.bro index 2ed7fc1690..cbab9a26bc 100644 --- a/policy/frameworks/notice/base/main.bro +++ b/policy/frameworks/notice/base/main.bro @@ -227,35 +227,31 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) if ( reading_traces() || dest == "" ) return; - local email_text = cat( + local email_text = string_cat( "From: ", mail_from, "\n", - "Subject: ", mail_subject_prefix, " ", n$note, "\n", + "Subject: ", mail_subject_prefix, " ", fmt("%s", n$note), "\n", "To: ", dest, "\n", # TODO: BiF to get version (the resource_usage Bif seems like overkill). "User-Agent: Bro-IDS/?.?.?\n"); if ( reply_to != "" ) - email_text = cat(email_text, "Reply-To: ", reply_to, "\n"); + email_text = string_cat(email_text, "Reply-To: ", reply_to, "\n"); # The notice emails always start off with the human readable message. - email_text = cat(email_text, "\n", n$msg, "\n"); + email_text = string_cat(email_text, "\n", n$msg, "\n"); # Add the extended information if it's requested. if ( extend ) { for ( i in n$email_body_sections ) { - email_text = cat(email_text, "******************\n"); - email_text = cat(email_text, n$email_body_sections[i], "\n"); + email_text = string_cat(email_text, "******************\n"); + email_text = string_cat(email_text, n$email_body_sections[i], "\n"); } } - email_text = cat(email_text, "\n\n--\n[Automatically generated]\n\n"); - - local mail_cmd = - fmt("echo \"%s\" | %s -t -oi %s", - str_shell_escape(email_text), sendmail); - system(mail_cmd); + email_text = string_cat(email_text, "\n\n--\n[Automatically generated]\n\n"); + piped_exec(fmt("%s -t -oi", sendmail), email_text); } event notice(n: Notice::Info) &priority=-5 diff --git a/src/bro.bif b/src/bro.bif index 6472e4b259..c433cb2eee 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -9,6 +9,7 @@ #include #include #include +#include #include "Reporter.h" @@ -3613,3 +3614,22 @@ function NFS3::mode2string%(mode: count%): string return new StringVal(str); %} + +## Opens a program with popen() and writes a given to string to the returned +## stream to send it to the opened process's stdin. +## program: a string naming the program to execute +## to_write: a string to pipe to the opened program's process over stdin +## Returns: F if popen'ing the program failed, else T +function piped_exec%(program: string, to_write: string%): bool + %{ + const char* prog = program->CheckString(); + FILE* f = popen(prog, "w"); + if ( !f ) + { + reporter->Error("Failed to popen %s", prog); + return new Val(false, TYPE_BOOL); + } + fprintf(f, "%s", to_write->CheckString()); + pclose(f); + return new Val(true, TYPE_BOOL); + %} diff --git a/testing/btest/Baseline/bifs.piped_exec/output b/testing/btest/Baseline/bifs.piped_exec/output new file mode 100644 index 0000000000..0d3fba25fe --- /dev/null +++ b/testing/btest/Baseline/bifs.piped_exec/output @@ -0,0 +1,2 @@ +hello world +foobar diff --git a/testing/btest/bifs/piped_exec.bro b/testing/btest/bifs/piped_exec.bro new file mode 100644 index 0000000000..4405f0b500 --- /dev/null +++ b/testing/btest/bifs/piped_exec.bro @@ -0,0 +1,6 @@ +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +global cmds = "print \"hello world\";"; +cmds = string_cat(cmds, "\nprint \"foobar\";"); +piped_exec("bro", cmds); From 451b43498fdb88f2f7cc997f38d69ca2de77d01c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 27 Jul 2011 15:11:05 -0500 Subject: [PATCH 4/5] Another fix to the default-loaded-scripts test. sed on some platforms like OS X (maybe FreeBSD in general) won't recognize semi-colon delimited commands as multiple commands, instead use the -e option multiple times to build the command list. --- testing/btest/policy/misc/default-loaded-scripts.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/btest/policy/misc/default-loaded-scripts.test b/testing/btest/policy/misc/default-loaded-scripts.test index ca67058dd5..c7f0d12a23 100644 --- a/testing/btest/policy/misc/default-loaded-scripts.test +++ b/testing/btest/policy/misc/default-loaded-scripts.test @@ -7,6 +7,6 @@ # @TEST-EXEC: bro misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log -# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed ':a;$!N;s/^\(.*\).*\n\1.*/\1/;ta' >prefix +# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log # @TEST-EXEC: btest-diff canonified_loaded_scripts.log From 771728720d4f4e97a8906ef59ac45a2167919132 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 28 Jul 2011 14:54:18 -0500 Subject: [PATCH 5/5] Normalize Notice::Type identifiers per convention. (closes #484) --- policy/frameworks/notice/base/weird.bro | 16 ++++++++-------- policy/frameworks/packet-filter/netstats.bro | 4 ++-- policy/hot.conn.bro | 12 ++++++------ policy/protocols/dns/base/detect.bro | 6 +++--- policy/protocols/http/base/file-ident.bro | 4 ++-- .../defaults/remove-high-volume-notices.bro | 12 +++++------- .../Baseline/language.rare-events/notice.log | 2 -- 7 files changed, 26 insertions(+), 30 deletions(-) delete mode 100644 testing/btest/Baseline/language.rare-events/notice.log diff --git a/policy/frameworks/notice/base/weird.bro b/policy/frameworks/notice/base/weird.bro index 149df5ea2b..6cbaa25a4a 100644 --- a/policy/frameworks/notice/base/weird.bro +++ b/policy/frameworks/notice/base/weird.bro @@ -8,13 +8,13 @@ export { redef enum Notice::Type += { ## Generic unusual but alarm-worthy activity. - WeirdActivity, + Weird_Activity, ## Possible evasion; usually just chud. - RetransmissionInconsistency, + Retransmission_Inconsistency, ## Could mean packet drop; could also be chud. - AckAboveHole, + Ack_Above_Hole, ## Data has sequence hole; perhaps due to filtering. - ContentGap, + Content_Gap, }; type Info: record { @@ -295,7 +295,7 @@ function report_weird(t: time, name: string, id: string, have_conn: bool, if ( action in notice_actions && ! no_log ) { local n: Notice::Info; - n$note = WeirdActivity; + n$note = Weird_Activity; n$msg = info$msg; if ( have_conn ) n$conn = current_conn; @@ -401,7 +401,7 @@ event rexmit_inconsistency(c: connection, t1: string, t2: string) { if ( c$id !in did_inconsistency_msg ) { - NOTICE([$note=RetransmissionInconsistency, + NOTICE([$note=Retransmission_Inconsistency, $conn=c, $msg=fmt("%s rexmit inconsistency (%s) (%s)", id_string(c$id), t1, t2)]); @@ -411,13 +411,13 @@ event rexmit_inconsistency(c: connection, t1: string, t2: string) event ack_above_hole(c: connection) { - NOTICE([$note=AckAboveHole, $conn=c, + NOTICE([$note=Ack_Above_Hole, $conn=c, $msg=fmt("%s ack above a hole", id_string(c$id))]); } event content_gap(c: connection, is_orig: bool, seq: count, length: count) { - NOTICE([$note=ContentGap, $conn=c, + NOTICE([$note=Content_Gap, $conn=c, $msg=fmt("%s content gap (%s %d/%d)%s", id_string(c$id), is_orig ? ">" : "<", seq, length, is_external_connection(c) ? " [external]" : "")]); diff --git a/policy/frameworks/packet-filter/netstats.bro b/policy/frameworks/packet-filter/netstats.bro index 226f22258b..bd95d95ff0 100644 --- a/policy/frameworks/packet-filter/netstats.bro +++ b/policy/frameworks/packet-filter/netstats.bro @@ -7,7 +7,7 @@ module PacketFilter; export { redef enum Notice::Type += { ## Bro reported packets dropped by the packet filter. - DroppedPackets, + Dropped_Packets, }; ## This is the interval between individual statistics collection. @@ -22,7 +22,7 @@ event net_stats_update(last_stat: NetStats) { local new_recvd = ns$pkts_recvd - last_stat$pkts_recvd; local new_link = ns$pkts_link - last_stat$pkts_link; - NOTICE([$note=DroppedPackets, + NOTICE([$note=Dropped_Packets, $msg=fmt("%d packets dropped after filtering, %d received%s", new_dropped, new_recvd + new_dropped, new_link != 0 ? fmt(", %d on link", new_link) : "")]); diff --git a/policy/hot.conn.bro b/policy/hot.conn.bro index 0ac070832c..3b29504f73 100644 --- a/policy/hot.conn.bro +++ b/policy/hot.conn.bro @@ -1,10 +1,10 @@ @load hot redef enum Notice += { - SensitiveConnection, # connection marked "hot" + Sensitive_Connection, # connection marked "hot" }; -# Whether to translate the local address in SensitiveConnection notices +# Whether to translate the local address in Sensitive_Connection notices # to a hostname. Meant as a demonstration of the "when" construct. const xlate_hot_local_addr = F &redef; @@ -40,16 +40,16 @@ function full_id_string(c: connection): string } -# Low-level routine that generates the actual SensitiveConnection +# Low-level routine that generates the actual Sensitive_Connection # notice associated with a "hot" connection. function do_hot_notice(c: connection, dir: string, host: string) { - NOTICE([$note=SensitiveConnection, $conn=c, + NOTICE([$note=Sensitive_Connection, $conn=c, $msg=fmt("hot: %s %s local host: %s", full_id_string(c), dir, host)]); } -# Generate a SensitiveConnection notice with the local hostname +# Generate a Sensitive_Connection notice with the local hostname # translated. Mostly intended as a demonstration of using "when". function gen_hot_notice_with_hostnames(c: connection) { @@ -78,7 +78,7 @@ function log_hot_conn(c: connection) if ( xlate_hot_local_addr ) gen_hot_notice_with_hostnames(c); else - NOTICE([$note=SensitiveConnection, $conn=c, + NOTICE([$note=Sensitive_Connection, $conn=c, $msg=fmt("hot: %s", full_id_string(c))]); add hot_conns_reported[c$id][msg]; diff --git a/policy/protocols/dns/base/detect.bro b/policy/protocols/dns/base/detect.bro index c1b8b47f8d..d22ca913be 100644 --- a/policy/protocols/dns/base/detect.bro +++ b/policy/protocols/dns/base/detect.bro @@ -2,7 +2,7 @@ ##! ##! Notices raised: ##! -##! * :bro:enum:`DNS::ExternalName` +##! * :bro:enum:`DNS::External_Name` ##! ##! A remote host resolves to a local host, but the name is not considered ##! to be within a local zone. :bro:id:`local_zones` variable **must** @@ -17,7 +17,7 @@ export { ## Raised when a non-local name is found to be pointing at a local host. ## This only works appropriately when all of your authoritative DNS ## servers are located in your :bro:id:`Site::local_nets`. - ExternalName, + External_Name, }; } @@ -32,7 +32,7 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori !Site::is_local_addr(c$id$resp_h) && # response from an external nameserver !Site::is_local_name(ans$query) ) # name isn't in a local zone. { - NOTICE([$note=ExternalName, + NOTICE([$note=External_Name, $msg=fmt("%s is pointing to a local host - %s.", ans$query, a), $conn=c]); } diff --git a/policy/protocols/http/base/file-ident.bro b/policy/protocols/http/base/file-ident.bro index 1061b2b2f2..cd9f891110 100644 --- a/policy/protocols/http/base/file-ident.bro +++ b/policy/protocols/http/base/file-ident.bro @@ -16,7 +16,7 @@ export { redef enum Notice::Type += { # This notice is thrown when the file extension doesn't # seem to match the file contents. - IncorrectFileType, + Incorrect_File_Type, }; redef record Info += { @@ -59,7 +59,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori { local url = build_url_http(c$http); local message = fmt("%s %s %s", msg, c$http$method, url); - NOTICE([$note=IncorrectFileType, + NOTICE([$note=Incorrect_File_Type, $msg=message, $conn=c, $method=c$http$method, diff --git a/policy/tuning/defaults/remove-high-volume-notices.bro b/policy/tuning/defaults/remove-high-volume-notices.bro index 0dc58e278a..1133bf952b 100644 --- a/policy/tuning/defaults/remove-high-volume-notices.bro +++ b/policy/tuning/defaults/remove-high-volume-notices.bro @@ -3,11 +3,9 @@ # Remove these notices from logging since they can be too noisy. redef Notice::ignored_types += { - Weird::ContentGap, - Weird::AckAboveHole, - Weird::RetransmissionInconsistency, + Weird::Content_Gap, + Weird::Ack_Above_Hole, + Weird::Retransmission_Inconsistency, ## Only allow these to go in the weird log. - Weird::WeirdActivity, - #DynDisable::ProtocolViolation, - -}; \ No newline at end of file + Weird::Weird_Activity, +}; diff --git a/testing/btest/Baseline/language.rare-events/notice.log b/testing/btest/Baseline/language.rare-events/notice.log deleted file mode 100644 index c0b6154ac2..0000000000 --- a/testing/btest/Baseline/language.rare-events/notice.log +++ /dev/null @@ -1,2 +0,0 @@ -# ts uid id.orig_h id.orig_p id.resp_h id.resp_p victim note msg sub src dst p n action tag do_alarm -1308596064.17872 - - - - - - PacketFilter::DroppedPackets 2 packets dropped after filtering, 1109 received, 10000 on link - - - - - Notice::ACTION_FILE @UWkUyAuUGXf F