diff --git a/doc/broids/index.rst b/doc/broids/index.rst index b3317815a7..46e0d6ded6 100644 --- a/doc/broids/index.rst +++ b/doc/broids/index.rst @@ -22,19 +22,10 @@ Detecting an FTP Bruteforce attack and notifying For the purpose of this exercise, we define FTP bruteforcing as too many rejected usernames and passwords occurring from a single address. We start by defining a threshold for the number of attempts and a -monitoring interval in minutes. +monitoring interval in minutes as well as a new notice type. - .. code:: bro - - export { - ## How many rejected usernames or passwords are required before being - ## considered to be bruteforcing. - const bruteforce_threshold: double = 20 &redef; - - ## The time period in which the threshold needs to be crossed before - ## being reset. - const bruteforce_measurement_interval = 15mins &redef; - } +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro + :lines: 9-25 Now, using the ftp_reply event, we check for error codes from the `500 series `_ @@ -44,112 +35,24 @@ function to break down the reply code and check if the first digit is a "5" or not. If true, we then use the :ref:`Summary Statistics Framework ` to keep track of the number of failed attempts. - .. code:: bro +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro + :lines: 52-60 - event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) - { - local cmd = c$ftp$cmdarg$cmd; - if ( cmd == "USER" || cmd == "PASS" ) - { - if ( FTP::parse_ftp_reply_code(code)$x == 5 ) - SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]); - } - } +Next, we use the SumStats framework to raise a notice of the attack of +the attack when the number of failed attempts exceeds the specified +threshold during the measuring interval. -Next, we use the SumStats framework to automatically print a message on -the console alerting of the attack when the number of failed attempts -exceeds the specified threshold during the measuring interval. +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro + :lines: 28-50 - .. code:: bro +Below is the final code for our script. - event bro_init() - { - local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)]; - SumStats::create([$name="ftp-detect-bruteforcing", - $epoch=bruteforce_measurement_interval, - $reducers=set(r1), - $threshold_val(key: SumStats::Key, result: SumStats::Result) = - { - return result["ftp.failed_auth"]$num+0.0; - }, - $threshold=bruteforce_threshold, - $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = - { - local r = result["ftp.failed_auth"]; - local dur = duration_to_mins_secs(r$end-r$begin); - local plural = r$unique>1 ? "s" : ""; - local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur); - }]); - } +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro -Printing a message on the console is a good start but it will be better -if we raise an alarm instead using the :ref:`Notice Framework -`. For this, we need to define a new Notice type and -trigger the alarm under the right conditions. Below is the final code -for our script. +.. btest:: ftp-bruteforce - .. code:: bro - - ##! FTP brute-forcing detector, triggering when too many rejected usernames or - ##! failed passwords have occurred from a single address. - - @load base/protocols/ftp - @load base/frameworks/sumstats - - @load base/utils/time - - module FTP; - - export { - redef enum Notice::Type += { - ## Indicates a host bruteforcing FTP logins by watching for too - ## many rejected usernames or failed passwords. - Bruteforcing - }; - - ## How many rejected usernames or passwords are required before being - ## considered to be bruteforcing. - const bruteforce_threshold: double = 20 &redef; - - ## The time period in which the threshold needs to be crossed before - ## being reset. - const bruteforce_measurement_interval = 15mins &redef; - } - - - event bro_init() - { - local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)]; - SumStats::create([$name="ftp-detect-bruteforcing", - $epoch=bruteforce_measurement_interval, - $reducers=set(r1), - $threshold_val(key: SumStats::Key, result: SumStats::Result) = - { - return result["ftp.failed_auth"]$num+0.0; - }, - $threshold=bruteforce_threshold, - $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = - { - local r = result["ftp.failed_auth"]; - local dur = duration_to_mins_secs(r$end-r$begin); - local plural = r$unique>1 ? "s" : ""; - local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur); - NOTICE([$note=FTP::Bruteforcing, - $src=key$host, - $msg=message, - $identifier=cat(key$host)]); - }]); - } - - event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) - { - local cmd = c$ftp$cmdarg$cmd; - if ( cmd == "USER" || cmd == "PASS" ) - { - if ( FTP::parse_ftp_reply_code(code)$x == 5 ) - SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]); - } - } + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro + @TEST-EXEC: btest-rst-include notice.log As a final note, the :doc:`detect-bruteforcing.bro ` script above is diff --git a/doc/httpmonitor/file_extraction.bro b/doc/httpmonitor/file_extraction.bro new file mode 100644 index 0000000000..b2318c595e --- /dev/null +++ b/doc/httpmonitor/file_extraction.bro @@ -0,0 +1,24 @@ + +global mime_to_ext: table[string] of string = { + ["application/x-dosexec"] = "exe", + ["text/plain"] = "txt", + ["image/jpeg"] = "jpg", + ["image/png"] = "png", + ["text/html"] = "html", +}; + +event file_new(f: fa_file) + { + if ( f$source != "HTTP" ) + return; + + if ( ! f?$mime_type ) + return; + + if ( f$mime_type !in mime_to_ext ) + return; + + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]); + print fmt("Extracting file %s", fname); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); + } diff --git a/doc/httpmonitor/http_proxy_01.bro b/doc/httpmonitor/http_proxy_01.bro new file mode 100644 index 0000000000..76555b6646 --- /dev/null +++ b/doc/httpmonitor/http_proxy_01.bro @@ -0,0 +1,5 @@ +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/doc/httpmonitor/http_proxy_02.bro b/doc/httpmonitor/http_proxy_02.bro new file mode 100644 index 0000000000..cdbd722619 --- /dev/null +++ b/doc/httpmonitor/http_proxy_02.bro @@ -0,0 +1,26 @@ + +module HTTP; + +export { + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/doc/httpmonitor/http_proxy_03.bro b/doc/httpmonitor/http_proxy_03.bro new file mode 100644 index 0000000000..17bfdcb95b --- /dev/null +++ b/doc/httpmonitor/http_proxy_03.bro @@ -0,0 +1,31 @@ + +@load base/utils/site + +redef Site::local_nets += { 192.168.0.0/16 }; + +module HTTP; + +export { + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( Site::is_local_addr(c$id$resp_h) && + /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/doc/httpmonitor/http_proxy_04.bro b/doc/httpmonitor/http_proxy_04.bro new file mode 100644 index 0000000000..1f11be3670 --- /dev/null +++ b/doc/httpmonitor/http_proxy_04.bro @@ -0,0 +1,40 @@ +@load base/utils/site +@load base/frameworks/notice + +redef Site::local_nets += { 192.168.0.0/16 }; + +module HTTP; + +export { + + redef enum Notice::Type += { + Open_Proxy + }; + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( Site::is_local_addr(c$id$resp_h) && + /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + NOTICE([$note=HTTP::Open_Proxy, + $msg=fmt("A local server is acting as an open proxy: %s", + c$id$resp_h), + $conn=c, + $identifier=cat(c$id$resp_h), + $suppress_for=1day]); + } diff --git a/doc/httpmonitor/index.rst b/doc/httpmonitor/index.rst index 6122421e37..f6b2f5e122 100644 --- a/doc/httpmonitor/index.rst +++ b/doc/httpmonitor/index.rst @@ -85,79 +85,37 @@ use this to identify a proxy server. We can write a basic script in Bro to handle the http_reply event and detect a reply for a ``GET http://`` request. - .. code:: bro +.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_01.bro - event http_reply(c: connection, version: string, code: count, reason: string) - { - if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 ) - { - print fmt("A local server is acting as an open proxy: ", c$id$resp_h); - } - } +.. btest:: http_proxy_01 + + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_01.bro Basically, the script is checking for a "200 OK" status code on a reply -for a request that includes "http:". In reality, the HTTP protocol -defines several success status codes other than 200, so we will extend -our basic script to also consider the additional codes. +for a request that includes "http:" (case insensitive). In reality, the +HTTP protocol defines several success status codes other than 200, so we +will extend our basic script to also consider the additional codes. - .. code:: bro +.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_02.bro - export { +.. btest:: http_proxy_02 - global success_status_codes: set[count] = { - 200, - 201, - 202, - 203, - 204, - 205, - 206, - 207, - 208, - 226, - 304 - }; - - } - - event http_reply(c: connection, version: string, code: count, reason: string) - { - if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code in success_status_codes ) - { - print fmt("A local server is acting as an open proxy: ", c$id$resp_h); - } - } + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_02.bro Next, we will make sure that the responding proxy is part of our local network. - .. code:: bro +.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_03.bro - export { +.. btest:: http_proxy_03 - global success_status_codes: set[count] = { - 200, - 201, - 202, - 203, - 204, - 205, - 206, - 207, - 208, - 226, - 304 - }; + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_03.bro - } +.. note:: - event http_reply(c: connection, version: string, code: count, reason: string) - { - if ( Site::is_local_addr(c$id$resp_h) && /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code in success_status_codes ) - { - print fmt("A local server is acting as an open proxy: ", c$id$resp_h); - } - } + The redefinition of :bro:see:`Site::local_nets` is only done inside + this script to make it a self-contained example. It's typically + redefined somewhere else. Finally, our goal should be to generate an alert when a proxy has been detected instead of printing a message on the console output. For that, @@ -166,71 +124,18 @@ we will tag the traffic accordingly and define a new ``Open_Proxy`` notification has been fired, we will further suppress it for one day. Below is the complete script. - .. code:: bro +.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_04.bro - @load base/frameworks/notice +.. btest:: http_proxy_04 - module HTTP; + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_04.bro + @TEST-EXEC: btest-rst-include notice.log - export { - - redef enum HTTP::Tags += { - OPEN_PROXY_TAG - }; - redef enum Notice::Type += { - Open_Proxy - }; - - global success_status_codes: set[count] = { - 200, - 201, - 202, - 203, - 204, - 205, - 206, - 207, - 208, - 226, - 304 - }; - - } - - redef Notice::emailed_types += { - Open_Proxy, - }; - - function open_proxy_only(rec: HTTP::Info) : bool - { - # Only write out connections with the OPEN_PROXY_TAG. - return OPEN_PROXY_TAG in rec$tags; - } - - event http_reply(c: connection, version: string, code: count, reason: string) - { - # make sure responding host is local - #if ( Site::is_local_addr(c$id$resp_h) && /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code in success_status_codes ) - { - add c$http$tags[OPEN_PROXY_TAG]; - local ident = cat(c$id$resp_h); - if ( c$http?$host ) #check if the optional host field exists in http - { - print fmt("Originator host: %s", c$id$orig_h); - NOTICE([$note=HTTP::Open_Proxy, - $msg=cat("A local server is acting as an open proxy: ", c$id$resp_h), - $conn=c, $identifier=cat(ident, c$id$resp_h), - $suppress_for=1day]); - } - } - } - - event bro_init() - { - #Creating a new filter for all open proxy logs. - local filter: Log::Filter = [$name="open_proxy", $path="open_proxy", $pred=open_proxy_only]; - Log::add_filter(HTTP::LOG, filter); - } +Note that this script only logs the presence of the proxy to +``notice.log``, but if an additional email is desired (and email +functionality is enabled), then that's done simply by redefining +:bro:see:`Notice::emailed_types` to add the ``Open_proxy`` notice type +to it. ---------------- Inspecting Files @@ -240,43 +145,19 @@ Files are often transmitted on regular HTTP conversations between a client and a server. Most of the time these files are harmless, just images and some other multimedia content, but there are also types of files, specially executable files, that can damage your system. We can -instruct Bro to create a copy of all executable files that it sees for -later analysis using the :ref:`File Analysis Framework -` (introduced with Bro 2.2) as shown in the -following script. +instruct Bro to create a copy of all files of certain types that it sees +using the :ref:`File Analysis Framework ` +(introduced with Bro 2.2): - .. code:: bro +.. btest-include:: ${DOC_ROOT}/httpmonitor/file_extraction.bro - global ext_map: table[string] of string = { - ["application/x-dosexec"] = "exe", - } &default =""; +.. btest:: file_extraction - event file_new(f: fa_file) - { - local ext = ""; + @TEST-EXEC: btest-rst-cmd -n 5 bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/httpmonitor/file_extraction.bro - if ( f?$mime_type ) - ext = ext_map[f$mime_type]; - - local fname = fmt("%s-%s.%s", f$source, f$id, ext); - Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); - } - -Bro will extract all files from the traffic and write them on a new -``extract_files/`` subdirectory and change the file name with the right -suffix (extension) based on the content of the ext_map table. So, if you -want to do the same for other extracted files besides executables you -just need to add those types to the ``ext_map`` table like this. - - .. code:: bro - - global ext_map: table[string] of string = { - ["application/x-dosexec"] = "exe", - ["text/plain"] = "txt", - ["image/jpeg"] = "jpg", - ["image/png"] = "png", - ["text/html"] = "html", - } &default =""; - -Bro will now write the appropriate suffix for text, JPEG, PNG, and HTML -files stored in the ``extract_files/`` subdirectory. +Here, the ``mime_to_ext`` table serves two purposes. It defines which +mime types to extract and also the file suffix of the extracted files. +Extracted files are written to a new ``extract_files`` subdirectory. +Also note that the first conditional in the :bro:see:`file_new` event +handler can be removed to make this behavior generic to other protocols +besides HTTP. diff --git a/doc/mimestats/index.rst b/doc/mimestats/index.rst index f81431eb8b..df17f4872f 100644 --- a/doc/mimestats/index.rst +++ b/doc/mimestats/index.rst @@ -37,125 +37,35 @@ in the MIME type, size of the file ("response_body_len") and the originator host ("orig_h"). We use the MIME type as our key and create observers for the other two values. - .. code:: bro - - export { - redef enum Log::ID += { LOG }; - type Info: record { - ## Timestamp when the log line was finished and written. - ts: time &log; - ## Time interval that the log line covers. - ts_delta: interval &log; - ## The mime type - mtype: string &log; - ## The number of unique local hosts that fetched this mime type - uniq_hosts: count &log; - ## The number of hits to the mime type - hits: count &log; - ## The total number of bytes received by this mime type - bytes: count &log; - }; - - ## The frequency of logging the stats collected by this script. - const break_interval = 5mins &redef; - } - - event HTTP::log_http(rec: HTTP::Info) - { - if(Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types) { - local mime_type = rec$resp_mime_types[0]; - SumStats::observe("mime.bytes", [$str=mime_type], [$num=rec$response_body_len]); - SumStats::observe("mime.hits", [$str=mime_type], [$str=cat(rec$id$orig_h)]); - } - } +.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro + :lines: 6-29, 54-64 Next, we create the reducers. The first one will accumulate file sizes and the second one will make sure we only store a host ID once. Below is -the partial code. +the partial code from a :bro:see:`bro_init` handler. - .. code:: bro - - local r1: SumStats::Reducer = [$stream="mime.bytes", $apply=set(SumStats::SUM)]; - local r2: SumStats::Reducer = [$stream="mime.hits", $apply=set(SumStats::UNIQUE)]; +.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro + :lines: 34-37 In our final step, we create the SumStats where we check for the observation interval and once it expires, we populate the record (defined above) with all the relevant data and write it to a log. - .. code:: bro - - SumStats::create([$name="mime-metrics", - $epoch=break_interval, - $reducers=set(r1, r2), - $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = - { - local l: Info; - l$ts = network_time(); - l$ts_delta = break_interval; - l$mtype = key$str; - l$bytes = double_to_count(floor(result["mime.bytes"]$sum)); - l$hits = result["mime.hits"]$num; - l$uniq_hosts = result["mime.hits"]$unique; - Log::write(LOG, l); - }]); +.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro + :lines: 38-51 Putting everything together we end up with the following final code for our script. - .. code:: bro +.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro - @load base/frameworks/sumstats +.. btest:: mimestats - module MimeMetrics; + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/mimestats/mimestats.bro + @TEST-EXEC: btest-rst-include mime_metrics.log - export { - redef enum Log::ID += { LOG }; - type Info: record { - ## Timestamp when the log line was finished and written. - ts: time &log; - ## Time interval that the log line covers. - ts_delta: interval &log; - ## The mime type - mtype: string &log; - ## The number of unique local hosts that fetched this mime type - uniq_hosts: count &log; - ## The number of hits to the mime type - hits: count &log; - ## The total number of bytes received by this mime type - bytes: count &log; - }; - - ## The frequency of logging the stats collected by this script. - const break_interval = 5mins &redef; - } - - event bro_init() &priority=3 - { - Log::create_stream(MimeMetrics::LOG, [$columns=Info]); - local r1: SumStats::Reducer = [$stream="mime.bytes", $apply=set(SumStats::SUM)]; - local r2: SumStats::Reducer = [$stream="mime.hits", $apply=set(SumStats::UNIQUE)]; - SumStats::create([$name="mime-metrics", - $epoch=break_interval, - $reducers=set(r1, r2), - $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = - { - local l: Info; - l$ts = network_time(); - l$ts_delta = break_interval; - l$mtype = key$str; - l$bytes = double_to_count(floor(result["mime.bytes"]$sum)); - l$hits = result["mime.hits"]$num; - l$uniq_hosts = result["mime.hits"]$unique; - Log::write(LOG, l); - }]); - } - - event HTTP::log_http(rec: HTTP::Info) - { - if(Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types) { - local mime_type = rec$resp_mime_types[0]; - SumStats::observe("mime.bytes", [$str=mime_type], [$num=rec$response_body_len]); - SumStats::observe("mime.hits", [$str=mime_type], [$str=cat(rec$id$orig_h)]); - } - } +.. note:: + The redefinition of :bro:see:`Site::local_nets` is only done inside + this script to make it a self-contained example. It's typically + redefined somewhere else. diff --git a/doc/mimestats/mimestats.bro b/doc/mimestats/mimestats.bro new file mode 100644 index 0000000000..b854b26c2d --- /dev/null +++ b/doc/mimestats/mimestats.bro @@ -0,0 +1,64 @@ +@load base/utils/site +@load base/frameworks/sumstats + +redef Site::local_nets += { 10.0.0.0/8 }; + +module MimeMetrics; + +export { + + redef enum Log::ID += { LOG }; + + type Info: record { + ## Timestamp when the log line was finished and written. + ts: time &log; + ## Time interval that the log line covers. + ts_delta: interval &log; + ## The mime type + mtype: string &log; + ## The number of unique local hosts that fetched this mime type + uniq_hosts: count &log; + ## The number of hits to the mime type + hits: count &log; + ## The total number of bytes received by this mime type + bytes: count &log; + }; + + ## The frequency of logging the stats collected by this script. + const break_interval = 5mins &redef; +} + +event bro_init() &priority=3 + { + Log::create_stream(MimeMetrics::LOG, [$columns=Info]); + local r1: SumStats::Reducer = [$stream="mime.bytes", + $apply=set(SumStats::SUM)]; + local r2: SumStats::Reducer = [$stream="mime.hits", + $apply=set(SumStats::UNIQUE)]; + SumStats::create([$name="mime-metrics", + $epoch=break_interval, + $reducers=set(r1, r2), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + local l: Info; + l$ts = network_time(); + l$ts_delta = break_interval; + l$mtype = key$str; + l$bytes = double_to_count(floor(result["mime.bytes"]$sum)); + l$hits = result["mime.hits"]$num; + l$uniq_hosts = result["mime.hits"]$unique; + Log::write(MimeMetrics::LOG, l); + }]); + } + +event HTTP::log_http(rec: HTTP::Info) + { + if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types ) + { + local mime_type = rec$resp_mime_types[0]; + SumStats::observe("mime.bytes", [$str=mime_type], + [$num=rec$response_body_len]); + SumStats::observe("mime.hits", [$str=mime_type], + [$str=cat(rec$id$orig_h)]); + } + } diff --git a/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1 b/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1 new file mode 100644 index 0000000000..312663e74e --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1 @@ -0,0 +1,14 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/bro.org.pcap file_extraction.bro + Extracting file HTTP-FiIpIB2hRQSDBOSJRg.html + Extracting file HTTP-FnaT2a3UDd093opCB9.txt + Extracting file HTTP-FsvATF146kf1Emc21j.txt + Extracting file HTTP-FkMQHg2nBr44fc5h63.txt + Extracting file HTTP-FfQGqj4Fhh3pH7nVQj.txt + [...] + diff --git a/testing/btest/Baseline/doc.sphinx.ftp-bruteforce/btest-doc.sphinx.ftp-bruteforce#1 b/testing/btest/Baseline/doc.sphinx.ftp-bruteforce/btest-doc.sphinx.ftp-bruteforce#1 new file mode 100644 index 0000000000..35bb3fca30 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.ftp-bruteforce/btest-doc.sphinx.ftp-bruteforce#1 @@ -0,0 +1,24 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro + +.. rst-class:: btest-include + + .. code-block:: guess + :linenos: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path notice + #open 2014-01-21-21-56-07 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude + #types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double + 1389721084.522861 - - - - - - - - - FTP::Bruteforcing 192.168.56.1 had 20 failed logins on 1 FTP server in 0m37s - 192.168.56.1 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - + #close 2014-01-21-21-56-07 + diff --git a/testing/btest/Baseline/doc.sphinx.http_proxy_01/btest-doc.sphinx.http_proxy_01#1 b/testing/btest/Baseline/doc.sphinx.http_proxy_01/btest-doc.sphinx.http_proxy_01#1 new file mode 100644 index 0000000000..d14ba4102a --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.http_proxy_01/btest-doc.sphinx.http_proxy_01#1 @@ -0,0 +1,9 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/proxy.pcap http_proxy_01.bro + A local server is acting as an open proxy: 192.168.56.101 + diff --git a/testing/btest/Baseline/doc.sphinx.http_proxy_02/btest-doc.sphinx.http_proxy_02#1 b/testing/btest/Baseline/doc.sphinx.http_proxy_02/btest-doc.sphinx.http_proxy_02#1 new file mode 100644 index 0000000000..48f5d8719b --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.http_proxy_02/btest-doc.sphinx.http_proxy_02#1 @@ -0,0 +1,9 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/proxy.pcap http_proxy_02.bro + A local server is acting as an open proxy: 192.168.56.101 + diff --git a/testing/btest/Baseline/doc.sphinx.http_proxy_03/btest-doc.sphinx.http_proxy_03#1 b/testing/btest/Baseline/doc.sphinx.http_proxy_03/btest-doc.sphinx.http_proxy_03#1 new file mode 100644 index 0000000000..09b2137d42 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.http_proxy_03/btest-doc.sphinx.http_proxy_03#1 @@ -0,0 +1,9 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/proxy.pcap http_proxy_03.bro + A local server is acting as an open proxy: 192.168.56.101 + diff --git a/testing/btest/Baseline/doc.sphinx.http_proxy_04/btest-doc.sphinx.http_proxy_04#1 b/testing/btest/Baseline/doc.sphinx.http_proxy_04/btest-doc.sphinx.http_proxy_04#1 new file mode 100644 index 0000000000..cfe35296cb --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.http_proxy_04/btest-doc.sphinx.http_proxy_04#1 @@ -0,0 +1,24 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/proxy.pcap http_proxy_04.bro + +.. rst-class:: btest-include + + .. code-block:: guess + :linenos: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path notice + #open 2014-01-21-20-11-20 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude + #types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double + 1389654450.449603 CXWv6p3arKYeMETxOg 192.168.56.1 52679 192.168.56.101 80 - - - tcp HTTP::Open_Proxy A local server is acting as an open proxy: 192.168.56.101 - 192.168.56.1 192.168.56.101 80 - bro Notice::ACTION_LOG 86400.000000 F - - - - - + #close 2014-01-21-20-11-20 + diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output new file mode 100644 index 0000000000..acae92f44b --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output @@ -0,0 +1,28 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_extraction.bro + + +global mime_to_ext: table[string] of string = { + ["application/x-dosexec"] = "exe", + ["text/plain"] = "txt", + ["image/jpeg"] = "jpg", + ["image/png"] = "png", + ["text/html"] = "html", +}; + +event file_new(f: fa_file) + { + if ( f$source != "HTTP" ) + return; + + if ( ! f?$mime_type ) + return; + + if ( f$mime_type !in mime_to_ext ) + return; + + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]); + print fmt("Extracting file %s", fname); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_01_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_01_bro/output new file mode 100644 index 0000000000..4e10859d98 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_01_bro/output @@ -0,0 +1,9 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_01.bro + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_02_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_02_bro/output new file mode 100644 index 0000000000..01e3822001 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_02_bro/output @@ -0,0 +1,30 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_02.bro + + +module HTTP; + +export { + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_03_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_03_bro/output new file mode 100644 index 0000000000..5139fa8c49 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_03_bro/output @@ -0,0 +1,35 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_03.bro + + +@load base/utils/site + +redef Site::local_nets += { 192.168.0.0/16 }; + +module HTTP; + +export { + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( Site::is_local_addr(c$id$resp_h) && + /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_04_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_04_bro/output new file mode 100644 index 0000000000..a8ca8e19b2 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_04_bro/output @@ -0,0 +1,44 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_04.bro + +@load base/utils/site +@load base/frameworks/notice + +redef Site::local_nets += { 192.168.0.0/16 }; + +module HTTP; + +export { + + redef enum Notice::Type += { + Open_Proxy + }; + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( Site::is_local_addr(c$id$resp_h) && + /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + NOTICE([$note=HTTP::Open_Proxy, + $msg=fmt("A local server is acting as an open proxy: %s", + c$id$resp_h), + $conn=c, + $identifier=cat(c$id$resp_h), + $suppress_for=1day]); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro/output new file mode 100644 index 0000000000..ef537b6c53 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro/output @@ -0,0 +1,39 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + +module MimeMetrics; + +export { + + redef enum Log::ID += { LOG }; + + type Info: record { + ## Timestamp when the log line was finished and written. + ts: time &log; + ## Time interval that the log line covers. + ts_delta: interval &log; + ## The mime type + mtype: string &log; + ## The number of unique local hosts that fetched this mime type + uniq_hosts: count &log; + ## The number of hits to the mime type + hits: count &log; + ## The total number of bytes received by this mime type + bytes: count &log; + }; + + ## The frequency of logging the stats collected by this script. + const break_interval = 5mins &redef; +} +event HTTP::log_http(rec: HTTP::Info) + { + if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types ) + { + local mime_type = rec$resp_mime_types[0]; + SumStats::observe("mime.bytes", [$str=mime_type], + [$num=rec$response_body_len]); + SumStats::observe("mime.hits", [$str=mime_type], + [$str=cat(rec$id$orig_h)]); + } + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@2/output b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@2/output new file mode 100644 index 0000000000..027eade4dc --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@2/output @@ -0,0 +1,8 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + + local r1: SumStats::Reducer = [$stream="mime.bytes", + $apply=set(SumStats::SUM)]; + local r2: SumStats::Reducer = [$stream="mime.hits", + $apply=set(SumStats::UNIQUE)]; diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@3/output new file mode 100644 index 0000000000..e410c6ebb9 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@3/output @@ -0,0 +1,18 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + + SumStats::create([$name="mime-metrics", + $epoch=break_interval, + $reducers=set(r1, r2), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + local l: Info; + l$ts = network_time(); + l$ts_delta = break_interval; + l$mtype = key$str; + l$bytes = double_to_count(floor(result["mime.bytes"]$sum)); + l$hits = result["mime.hits"]$num; + l$uniq_hosts = result["mime.hits"]$unique; + Log::write(MimeMetrics::LOG, l); + }]); diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@4/output b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@4/output new file mode 100644 index 0000000000..0e97a0b14e --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@4/output @@ -0,0 +1,68 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + +@load base/utils/site +@load base/frameworks/sumstats + +redef Site::local_nets += { 10.0.0.0/8 }; + +module MimeMetrics; + +export { + + redef enum Log::ID += { LOG }; + + type Info: record { + ## Timestamp when the log line was finished and written. + ts: time &log; + ## Time interval that the log line covers. + ts_delta: interval &log; + ## The mime type + mtype: string &log; + ## The number of unique local hosts that fetched this mime type + uniq_hosts: count &log; + ## The number of hits to the mime type + hits: count &log; + ## The total number of bytes received by this mime type + bytes: count &log; + }; + + ## The frequency of logging the stats collected by this script. + const break_interval = 5mins &redef; +} + +event bro_init() &priority=3 + { + Log::create_stream(MimeMetrics::LOG, [$columns=Info]); + local r1: SumStats::Reducer = [$stream="mime.bytes", + $apply=set(SumStats::SUM)]; + local r2: SumStats::Reducer = [$stream="mime.hits", + $apply=set(SumStats::UNIQUE)]; + SumStats::create([$name="mime-metrics", + $epoch=break_interval, + $reducers=set(r1, r2), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + local l: Info; + l$ts = network_time(); + l$ts_delta = break_interval; + l$mtype = key$str; + l$bytes = double_to_count(floor(result["mime.bytes"]$sum)); + l$hits = result["mime.hits"]$num; + l$uniq_hosts = result["mime.hits"]$unique; + Log::write(MimeMetrics::LOG, l); + }]); + } + +event HTTP::log_http(rec: HTTP::Info) + { + if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types ) + { + local mime_type = rec$resp_mime_types[0]; + SumStats::observe("mime.bytes", [$str=mime_type], + [$num=rec$response_body_len]); + SumStats::observe("mime.hits", [$str=mime_type], + [$str=cat(rec$id$orig_h)]); + } + } diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro/output new file mode 100644 index 0000000000..59d57223d9 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro/output @@ -0,0 +1,21 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +module FTP; + +export { + redef enum Notice::Type += { + ## Indicates a host bruteforcing FTP logins by watching for too + ## many rejected usernames or failed passwords. + Bruteforcing + }; + + ## How many rejected usernames or passwords are required before being + ## considered to be bruteforcing. + const bruteforce_threshold: double = 20 &redef; + + ## The time period in which the threshold needs to be crossed before + ## being reset. + const bruteforce_measurement_interval = 15mins &redef; +} diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2/output new file mode 100644 index 0000000000..648fe8a559 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2/output @@ -0,0 +1,13 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) + { + local cmd = c$ftp$cmdarg$cmd; + if ( cmd == "USER" || cmd == "PASS" ) + { + if ( FTP::parse_ftp_reply_code(code)$x == 5 ) + SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]); + } + } diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3/output new file mode 100644 index 0000000000..f81c9f50ba --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3/output @@ -0,0 +1,27 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +event bro_init() + { + local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)]; + SumStats::create([$name="ftp-detect-bruteforcing", + $epoch=bruteforce_measurement_interval, + $reducers=set(r1), + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["ftp.failed_auth"]$num+0.0; + }, + $threshold=bruteforce_threshold, + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + local r = result["ftp.failed_auth"]; + local dur = duration_to_mins_secs(r$end-r$begin); + local plural = r$unique>1 ? "s" : ""; + local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur); + NOTICE([$note=FTP::Bruteforcing, + $src=key$host, + $msg=message, + $identifier=cat(key$host)]); + }]); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4/output new file mode 100644 index 0000000000..bb7b0fd078 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4/output @@ -0,0 +1,64 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +##! FTP brute-forcing detector, triggering when too many rejected usernames or +##! failed passwords have occurred from a single address. + +@load base/protocols/ftp +@load base/frameworks/sumstats + +@load base/utils/time + +module FTP; + +export { + redef enum Notice::Type += { + ## Indicates a host bruteforcing FTP logins by watching for too + ## many rejected usernames or failed passwords. + Bruteforcing + }; + + ## How many rejected usernames or passwords are required before being + ## considered to be bruteforcing. + const bruteforce_threshold: double = 20 &redef; + + ## The time period in which the threshold needs to be crossed before + ## being reset. + const bruteforce_measurement_interval = 15mins &redef; +} + + +event bro_init() + { + local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)]; + SumStats::create([$name="ftp-detect-bruteforcing", + $epoch=bruteforce_measurement_interval, + $reducers=set(r1), + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["ftp.failed_auth"]$num+0.0; + }, + $threshold=bruteforce_threshold, + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + local r = result["ftp.failed_auth"]; + local dur = duration_to_mins_secs(r$end-r$begin); + local plural = r$unique>1 ? "s" : ""; + local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur); + NOTICE([$note=FTP::Bruteforcing, + $src=key$host, + $msg=message, + $identifier=cat(key$host)]); + }]); + } + +event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) + { + local cmd = c$ftp$cmdarg$cmd; + if ( cmd == "USER" || cmd == "PASS" ) + { + if ( FTP::parse_ftp_reply_code(code)$x == 5 ) + SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]); + } + } diff --git a/testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1 b/testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1 new file mode 100644 index 0000000000..8396de608d --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1 @@ -0,0 +1,31 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/bro.org.pcap mimestats.bro + +.. rst-class:: btest-include + + .. code-block:: guess + :linenos: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path mime_metrics + #open 2014-01-21-21-35-28 + #fields ts ts_delta mtype uniq_hosts hits bytes + #types time interval string count count count + 1389719059.311698 300.000000 text/html 1 4 53070 + 1389719059.311698 300.000000 image/jpeg 1 1 186859 + 1389719059.311698 300.000000 text/troff 1 1 2957 + 1389719059.311698 300.000000 application/pgp-signature 1 1 836 + 1389719059.311698 300.000000 text/plain 1 12 114205 + 1389719059.311698 300.000000 image/gif 1 1 172 + 1389719059.311698 300.000000 image/png 1 9 82176 + 1389719059.311698 300.000000 image/x-icon 1 2 2300 + #close 2014-01-21-21-35-28 + diff --git a/testing/btest/Traces/ftp/bruteforce.pcap b/testing/btest/Traces/ftp/bruteforce.pcap new file mode 100644 index 0000000000..46bdb569d7 Binary files /dev/null and b/testing/btest/Traces/ftp/bruteforce.pcap differ diff --git a/testing/btest/Traces/http/bro.org.pcap b/testing/btest/Traces/http/bro.org.pcap new file mode 100644 index 0000000000..a50be12989 Binary files /dev/null and b/testing/btest/Traces/http/bro.org.pcap differ diff --git a/testing/btest/Traces/http/proxy.pcap b/testing/btest/Traces/http/proxy.pcap new file mode 100644 index 0000000000..e0e7adf3d4 Binary files /dev/null and b/testing/btest/Traces/http/proxy.pcap differ diff --git a/testing/btest/doc/sphinx/file_extraction.btest b/testing/btest/doc/sphinx/file_extraction.btest new file mode 100644 index 0000000000..76ebd82474 --- /dev/null +++ b/testing/btest/doc/sphinx/file_extraction.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd -n 5 bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/httpmonitor/file_extraction.bro diff --git a/testing/btest/doc/sphinx/ftp-bruteforce.btest b/testing/btest/doc/sphinx/ftp-bruteforce.btest new file mode 100644 index 0000000000..0a9c89c22e --- /dev/null +++ b/testing/btest/doc/sphinx/ftp-bruteforce.btest @@ -0,0 +1,2 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro +@TEST-EXEC: btest-rst-include notice.log diff --git a/testing/btest/doc/sphinx/http_proxy_01.btest b/testing/btest/doc/sphinx/http_proxy_01.btest new file mode 100644 index 0000000000..95c212876d --- /dev/null +++ b/testing/btest/doc/sphinx/http_proxy_01.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_01.bro diff --git a/testing/btest/doc/sphinx/http_proxy_02.btest b/testing/btest/doc/sphinx/http_proxy_02.btest new file mode 100644 index 0000000000..886177a025 --- /dev/null +++ b/testing/btest/doc/sphinx/http_proxy_02.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_02.bro diff --git a/testing/btest/doc/sphinx/http_proxy_03.btest b/testing/btest/doc/sphinx/http_proxy_03.btest new file mode 100644 index 0000000000..fe1e22f58c --- /dev/null +++ b/testing/btest/doc/sphinx/http_proxy_03.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_03.bro diff --git a/testing/btest/doc/sphinx/http_proxy_04.btest b/testing/btest/doc/sphinx/http_proxy_04.btest new file mode 100644 index 0000000000..1c2dcb707e --- /dev/null +++ b/testing/btest/doc/sphinx/http_proxy_04.btest @@ -0,0 +1,2 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_04.bro +@TEST-EXEC: btest-rst-include notice.log diff --git a/testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest b/testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest new file mode 100644 index 0000000000..acae92f44b --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest @@ -0,0 +1,28 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_extraction.bro + + +global mime_to_ext: table[string] of string = { + ["application/x-dosexec"] = "exe", + ["text/plain"] = "txt", + ["image/jpeg"] = "jpg", + ["image/png"] = "png", + ["text/html"] = "html", +}; + +event file_new(f: fa_file) + { + if ( f$source != "HTTP" ) + return; + + if ( ! f?$mime_type ) + return; + + if ( f$mime_type !in mime_to_ext ) + return; + + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]); + print fmt("Extracting file %s", fname); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); + } diff --git a/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_01_bro.btest b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_01_bro.btest new file mode 100644 index 0000000000..4e10859d98 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_01_bro.btest @@ -0,0 +1,9 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_01.bro + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_02_bro.btest b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_02_bro.btest new file mode 100644 index 0000000000..01e3822001 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_02_bro.btest @@ -0,0 +1,30 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_02.bro + + +module HTTP; + +export { + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_03_bro.btest b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_03_bro.btest new file mode 100644 index 0000000000..5139fa8c49 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_03_bro.btest @@ -0,0 +1,35 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_03.bro + + +@load base/utils/site + +redef Site::local_nets += { 192.168.0.0/16 }; + +module HTTP; + +export { + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( Site::is_local_addr(c$id$resp_h) && + /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + print fmt("A local server is acting as an open proxy: %s", c$id$resp_h); + } diff --git a/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_04_bro.btest b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_04_bro.btest new file mode 100644 index 0000000000..a8ca8e19b2 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_04_bro.btest @@ -0,0 +1,44 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +http_proxy_04.bro + +@load base/utils/site +@load base/frameworks/notice + +redef Site::local_nets += { 192.168.0.0/16 }; + +module HTTP; + +export { + + redef enum Notice::Type += { + Open_Proxy + }; + + global success_status_codes: set[count] = { + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 226, + 304 + }; +} + +event http_reply(c: connection, version: string, code: count, reason: string) + { + if ( Site::is_local_addr(c$id$resp_h) && + /^[hH][tT][tT][pP]:/ in c$http$uri && + c$http$status_code in HTTP::success_status_codes ) + NOTICE([$note=HTTP::Open_Proxy, + $msg=fmt("A local server is acting as an open proxy: %s", + c$id$resp_h), + $conn=c, + $identifier=cat(c$id$resp_h), + $suppress_for=1day]); + } diff --git a/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro.btest b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro.btest new file mode 100644 index 0000000000..ef537b6c53 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro.btest @@ -0,0 +1,39 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + +module MimeMetrics; + +export { + + redef enum Log::ID += { LOG }; + + type Info: record { + ## Timestamp when the log line was finished and written. + ts: time &log; + ## Time interval that the log line covers. + ts_delta: interval &log; + ## The mime type + mtype: string &log; + ## The number of unique local hosts that fetched this mime type + uniq_hosts: count &log; + ## The number of hits to the mime type + hits: count &log; + ## The total number of bytes received by this mime type + bytes: count &log; + }; + + ## The frequency of logging the stats collected by this script. + const break_interval = 5mins &redef; +} +event HTTP::log_http(rec: HTTP::Info) + { + if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types ) + { + local mime_type = rec$resp_mime_types[0]; + SumStats::observe("mime.bytes", [$str=mime_type], + [$num=rec$response_body_len]); + SumStats::observe("mime.hits", [$str=mime_type], + [$str=cat(rec$id$orig_h)]); + } + } diff --git a/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@2.btest b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@2.btest new file mode 100644 index 0000000000..027eade4dc --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@2.btest @@ -0,0 +1,8 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + + local r1: SumStats::Reducer = [$stream="mime.bytes", + $apply=set(SumStats::SUM)]; + local r2: SumStats::Reducer = [$stream="mime.hits", + $apply=set(SumStats::UNIQUE)]; diff --git a/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@3.btest b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@3.btest new file mode 100644 index 0000000000..e410c6ebb9 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@3.btest @@ -0,0 +1,18 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + + SumStats::create([$name="mime-metrics", + $epoch=break_interval, + $reducers=set(r1, r2), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + local l: Info; + l$ts = network_time(); + l$ts_delta = break_interval; + l$mtype = key$str; + l$bytes = double_to_count(floor(result["mime.bytes"]$sum)); + l$hits = result["mime.hits"]$num; + l$uniq_hosts = result["mime.hits"]$unique; + Log::write(MimeMetrics::LOG, l); + }]); diff --git a/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@4.btest b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@4.btest new file mode 100644 index 0000000000..0e97a0b14e --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@4.btest @@ -0,0 +1,68 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +mimestats.bro + +@load base/utils/site +@load base/frameworks/sumstats + +redef Site::local_nets += { 10.0.0.0/8 }; + +module MimeMetrics; + +export { + + redef enum Log::ID += { LOG }; + + type Info: record { + ## Timestamp when the log line was finished and written. + ts: time &log; + ## Time interval that the log line covers. + ts_delta: interval &log; + ## The mime type + mtype: string &log; + ## The number of unique local hosts that fetched this mime type + uniq_hosts: count &log; + ## The number of hits to the mime type + hits: count &log; + ## The total number of bytes received by this mime type + bytes: count &log; + }; + + ## The frequency of logging the stats collected by this script. + const break_interval = 5mins &redef; +} + +event bro_init() &priority=3 + { + Log::create_stream(MimeMetrics::LOG, [$columns=Info]); + local r1: SumStats::Reducer = [$stream="mime.bytes", + $apply=set(SumStats::SUM)]; + local r2: SumStats::Reducer = [$stream="mime.hits", + $apply=set(SumStats::UNIQUE)]; + SumStats::create([$name="mime-metrics", + $epoch=break_interval, + $reducers=set(r1, r2), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + local l: Info; + l$ts = network_time(); + l$ts_delta = break_interval; + l$mtype = key$str; + l$bytes = double_to_count(floor(result["mime.bytes"]$sum)); + l$hits = result["mime.hits"]$num; + l$uniq_hosts = result["mime.hits"]$unique; + Log::write(MimeMetrics::LOG, l); + }]); + } + +event HTTP::log_http(rec: HTTP::Info) + { + if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types ) + { + local mime_type = rec$resp_mime_types[0]; + SumStats::observe("mime.bytes", [$str=mime_type], + [$num=rec$response_body_len]); + SumStats::observe("mime.hits", [$str=mime_type], + [$str=cat(rec$id$orig_h)]); + } + } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro.btest b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro.btest new file mode 100644 index 0000000000..59d57223d9 --- /dev/null +++ b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro.btest @@ -0,0 +1,21 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +module FTP; + +export { + redef enum Notice::Type += { + ## Indicates a host bruteforcing FTP logins by watching for too + ## many rejected usernames or failed passwords. + Bruteforcing + }; + + ## How many rejected usernames or passwords are required before being + ## considered to be bruteforcing. + const bruteforce_threshold: double = 20 &redef; + + ## The time period in which the threshold needs to be crossed before + ## being reset. + const bruteforce_measurement_interval = 15mins &redef; +} diff --git a/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2.btest b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2.btest new file mode 100644 index 0000000000..648fe8a559 --- /dev/null +++ b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2.btest @@ -0,0 +1,13 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) + { + local cmd = c$ftp$cmdarg$cmd; + if ( cmd == "USER" || cmd == "PASS" ) + { + if ( FTP::parse_ftp_reply_code(code)$x == 5 ) + SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]); + } + } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3.btest b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3.btest new file mode 100644 index 0000000000..f81c9f50ba --- /dev/null +++ b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3.btest @@ -0,0 +1,27 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +event bro_init() + { + local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)]; + SumStats::create([$name="ftp-detect-bruteforcing", + $epoch=bruteforce_measurement_interval, + $reducers=set(r1), + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["ftp.failed_auth"]$num+0.0; + }, + $threshold=bruteforce_threshold, + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + local r = result["ftp.failed_auth"]; + local dur = duration_to_mins_secs(r$end-r$begin); + local plural = r$unique>1 ? "s" : ""; + local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur); + NOTICE([$note=FTP::Bruteforcing, + $src=key$host, + $msg=message, + $identifier=cat(key$host)]); + }]); + } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4.btest b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4.btest new file mode 100644 index 0000000000..bb7b0fd078 --- /dev/null +++ b/testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4.btest @@ -0,0 +1,64 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +detect-bruteforcing.bro + +##! FTP brute-forcing detector, triggering when too many rejected usernames or +##! failed passwords have occurred from a single address. + +@load base/protocols/ftp +@load base/frameworks/sumstats + +@load base/utils/time + +module FTP; + +export { + redef enum Notice::Type += { + ## Indicates a host bruteforcing FTP logins by watching for too + ## many rejected usernames or failed passwords. + Bruteforcing + }; + + ## How many rejected usernames or passwords are required before being + ## considered to be bruteforcing. + const bruteforce_threshold: double = 20 &redef; + + ## The time period in which the threshold needs to be crossed before + ## being reset. + const bruteforce_measurement_interval = 15mins &redef; +} + + +event bro_init() + { + local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)]; + SumStats::create([$name="ftp-detect-bruteforcing", + $epoch=bruteforce_measurement_interval, + $reducers=set(r1), + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["ftp.failed_auth"]$num+0.0; + }, + $threshold=bruteforce_threshold, + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + local r = result["ftp.failed_auth"]; + local dur = duration_to_mins_secs(r$end-r$begin); + local plural = r$unique>1 ? "s" : ""; + local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur); + NOTICE([$note=FTP::Bruteforcing, + $src=key$host, + $msg=message, + $identifier=cat(key$host)]); + }]); + } + +event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) + { + local cmd = c$ftp$cmdarg$cmd; + if ( cmd == "USER" || cmd == "PASS" ) + { + if ( FTP::parse_ftp_reply_code(code)$x == 5 ) + SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]); + } + } diff --git a/testing/btest/doc/sphinx/mimestats.btest b/testing/btest/doc/sphinx/mimestats.btest new file mode 100644 index 0000000000..06e47ea888 --- /dev/null +++ b/testing/btest/doc/sphinx/mimestats.btest @@ -0,0 +1,2 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/mimestats/mimestats.bro +@TEST-EXEC: btest-rst-include mime_metrics.log