Add unit tests for new Bro Manual docs.

doc/broids/index.rst

@@ -22,19 +22,10 @@ Detecting an FTP Bruteforce attack and notifying
 For the purpose of this exercise, we define FTP bruteforcing as too many
 rejected usernames and passwords occurring from a single address.  We
 start by defining a threshold for the number of attempts and a
-monitoring interval in minutes.
+monitoring interval in minutes as well as a new notice type.
 
-  .. code:: bro
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
-
+    :lines: 9-25
-	export {
-		## How many rejected usernames or passwords are required before being
-		## considered to be bruteforcing.
-		const bruteforce_threshold: double = 20 &redef;
-
-		## The time period in which the threshold needs to be crossed before
-		## being reset.
-		const bruteforce_measurement_interval = 15mins &redef;
-	}
 
 Now, using the ftp_reply event, we check for error codes from the `500
 series <http://en.wikipedia.org/wiki/List_of_FTP_server_return_codes>`_
@@ -44,112 +35,24 @@ function to break down the reply code and check if the first digit is a
 "5" or not. If true, we then use the :ref:`Summary Statistics Framework
 <sumstats-framework>` to keep track of the number of failed attempts.
 
-  .. code:: bro
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+    :lines: 52-60
 
-	event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+Next, we use the SumStats framework to raise a notice of the attack of
-		{
+the attack when the number of failed attempts exceeds the specified
-		local cmd = c$ftp$cmdarg$cmd;
+threshold during the measuring interval.
-		if ( cmd == "USER" || cmd == "PASS" )
-			{
-			if ( FTP::parse_ftp_reply_code(code)$x == 5 )
-				SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
-			}
-		}
 
-Next, we use the SumStats framework to automatically print a message on
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
-the console alerting of the attack when the number of failed attempts
+    :lines: 28-50
-exceeds the specified threshold during the measuring interval.
 
-  .. code:: bro
+Below is the final code for our script.
 
-	event bro_init()
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
-		{
-		local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)];
-		SumStats::create([$name="ftp-detect-bruteforcing",
-			          $epoch=bruteforce_measurement_interval,
-			          $reducers=set(r1),
-			          $threshold_val(key: SumStats::Key, result: SumStats::Result) =
-			          	{
-			          	return result["ftp.failed_auth"]$num+0.0;
-			          	},
-			          $threshold=bruteforce_threshold,
-			          $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
-			          	{
-			          	local r = result["ftp.failed_auth"];
-			          	local dur = duration_to_mins_secs(r$end-r$begin);
-			          	local plural = r$unique>1 ? "s" : "";
-			          	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
-			          	}]);
-		}
 
-Printing a message on the console is a good start but it will be better
+.. btest:: ftp-bruteforce
-if we raise an alarm instead using the :ref:`Notice Framework
-<notice-framework>`. For this, we need to define a new Notice type and
-trigger the alarm under the right conditions. Below is the final code
-for our script.
 
-  .. code:: bro
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro
-
+    @TEST-EXEC: btest-rst-include notice.log
-	##! FTP brute-forcing detector, triggering when too many rejected usernames or
-	##! failed passwords have occurred from a single address.
-
-	@load base/protocols/ftp
-	@load base/frameworks/sumstats
-
-	@load base/utils/time
-
-	module FTP;
-
-	export {
-		redef enum Notice::Type += {
-			## Indicates a host bruteforcing FTP logins by watching for too
-			## many rejected usernames or failed passwords.
-			Bruteforcing
-		};
-
-		## How many rejected usernames or passwords are required before being
-		## considered to be bruteforcing.
-		const bruteforce_threshold: double = 20 &redef;
-
-		## The time period in which the threshold needs to be crossed before
-		## being reset.
-		const bruteforce_measurement_interval = 15mins &redef;
-	}
-
-
-	event bro_init()
-		{
-		local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)];
-		SumStats::create([$name="ftp-detect-bruteforcing",
-			          $epoch=bruteforce_measurement_interval,
-			          $reducers=set(r1),
-			          $threshold_val(key: SumStats::Key, result: SumStats::Result) =
-			          	{
-			          	return result["ftp.failed_auth"]$num+0.0;
-			          	},
-			          $threshold=bruteforce_threshold,
-			          $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
-			          	{
-			          	local r = result["ftp.failed_auth"];
-			          	local dur = duration_to_mins_secs(r$end-r$begin);
-			          	local plural = r$unique>1 ? "s" : "";
-			          	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
-			          	NOTICE([$note=FTP::Bruteforcing,
-			          	        $src=key$host,
-			          	        $msg=message,
-			          	        $identifier=cat(key$host)]);
-			          	}]);
-		}
-
-	event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
-		{
-		local cmd = c$ftp$cmdarg$cmd;
-		if ( cmd == "USER" || cmd == "PASS" )
-			{
-			if ( FTP::parse_ftp_reply_code(code)$x == 5 )
-				SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
-			}
-		}
 
 As a final note, the :doc:`detect-bruteforcing.bro
 </scripts/policy/protocols/ftp/detect-bruteforcing.bro>` script above is

doc/httpmonitor/file_extraction.bro

@@ -0,0 +1,24 @@
+
+global mime_to_ext: table[string] of string = {
+	["application/x-dosexec"] = "exe",
+	["text/plain"] = "txt",
+	["image/jpeg"] = "jpg",
+	["image/png"] = "png",
+	["text/html"] = "html",
+};
+
+event file_new(f: fa_file)
+	{
+	if ( f$source != "HTTP" )
+		return;
+
+	if ( ! f?$mime_type )
+		return;
+
+	if ( f$mime_type !in mime_to_ext )
+		return;
+
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+	print fmt("Extracting file %s", fname);
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+	}

doc/httpmonitor/http_proxy_01.bro

@@ -0,0 +1,5 @@
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

doc/httpmonitor/http_proxy_02.bro

@@ -0,0 +1,26 @@
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

doc/httpmonitor/http_proxy_03.bro

@@ -0,0 +1,31 @@
+
+@load base/utils/site
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

doc/httpmonitor/http_proxy_04.bro

@@ -0,0 +1,40 @@
+@load base/utils/site
+@load base/frameworks/notice
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	redef enum Notice::Type += {
+		Open_Proxy
+	};
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		NOTICE([$note=HTTP::Open_Proxy,
+		        $msg=fmt("A local server is acting as an open proxy: %s",
+		                 c$id$resp_h),
+		        $conn=c,
+		        $identifier=cat(c$id$resp_h),
+		        $suppress_for=1day]);
+	}

doc/httpmonitor/index.rst

@@ -85,79 +85,37 @@ use this to identify a proxy server.
 We can write a basic script in Bro to handle the http_reply event and
 detect a reply for a ``GET http://`` request.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
 
-	event http_reply(c: connection, version: string, code: count, reason: string)
+.. btest:: http_proxy_01
-		{
+
-			if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
-		                {			
-				print fmt("A local server is acting as an open proxy: ", c$id$resp_h);
-		                }
-		}
 
 Basically, the script is checking for a "200 OK" status code on a reply
-for a request that includes "http:". In reality, the HTTP protocol
+for a request that includes "http:" (case insensitive). In reality, the
-defines several success status codes other than 200, so we will extend
+HTTP protocol defines several success status codes other than 200, so we
-our basic script to also consider the additional codes.
+will extend our basic script to also consider the additional codes.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
 
-	export {
+.. btest:: http_proxy_02
 
-		global success_status_codes: set[count] = {
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
-		        200,
-		        201,
-		        202,
-		        203,
-		        204,
-		        205,
-		        206,
-		        207,
-		        208,
-		        226,
-			304
-		        };
-
-	}
-
-	event http_reply(c: connection, version: string, code: count, reason: string)
-		{
-			if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code in success_status_codes )
-		                {			
-				print fmt("A local server is acting as an open proxy: ", c$id$resp_h);
-		                }
-		}
 
 Next, we will make sure that the responding proxy is part of our local
 network.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
 
-	export {
+.. btest:: http_proxy_03
 
-		global success_status_codes: set[count] = {
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
-		        200,
-		        201,
-		        202,
-		        203,
-		        204,
-		        205,
-		        206,
-		        207,
-		        208,
-		        226,
-			304
-		        };
 
-	}
+.. note::
 
-	event http_reply(c: connection, version: string, code: count, reason: string)
+    The redefinition of :bro:see:`Site::local_nets` is only done inside
-		{
+    this script to make it a self-contained example.  It's typically
-			if ( Site::is_local_addr(c$id$resp_h) && /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code in success_status_codes )
+    redefined somewhere else.
-		                {			
-				print fmt("A local server is acting as an open proxy: ", c$id$resp_h);
-		                }
-		}
 
 Finally, our goal should be to generate an alert when a proxy has been
 detected instead of printing a message on the console output.  For that,
@@ -166,71 +124,18 @@ we will tag the traffic accordingly and define a new ``Open_Proxy``
 notification has been fired, we will further suppress it for one day.
 Below is the complete script.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
 
-	@load base/frameworks/notice
+.. btest:: http_proxy_04
 
-	module HTTP;
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+    @TEST-EXEC: btest-rst-include notice.log
 
-	export {
+Note that this script only logs the presence of the proxy to
-
+``notice.log``, but if an additional email is desired (and email
-		redef enum HTTP::Tags += {
+functionality is enabled), then that's done simply by redefining
-		        OPEN_PROXY_TAG
+:bro:see:`Notice::emailed_types` to add the ``Open_proxy`` notice type
-		};
+to it.
-		redef enum Notice::Type += {
-		       Open_Proxy
-		};
-
-		global success_status_codes: set[count] = {
-		        200,
-		        201,
-		        202,
-		        203,
-		        204,
-		        205,
-		        206,
-		        207,
-		        208,
-		        226,
-			304
-		        };
-
-	}
-
-	redef Notice::emailed_types += {
-		Open_Proxy,
-	};
-
-	function open_proxy_only(rec: HTTP::Info) : bool
-		{
-		# Only write out connections with the OPEN_PROXY_TAG.
-		return OPEN_PROXY_TAG in rec$tags;
-		}
-
-	event http_reply(c: connection, version: string, code: count, reason: string)
-		{
-		        # make sure responding host is local
-		        #if ( Site::is_local_addr(c$id$resp_h) && /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code in success_status_codes )
-		                {			
-		                add c$http$tags[OPEN_PROXY_TAG];
-				local ident = cat(c$id$resp_h);
-		                if ( c$http?$host ) #check if the optional host field exists in http
-					{
-					print fmt("Originator host: %s", c$id$orig_h);
-		                        NOTICE([$note=HTTP::Open_Proxy,
-		                                $msg=cat("A local server is acting as an open proxy: ", c$id$resp_h),
-		                                $conn=c, $identifier=cat(ident, c$id$resp_h),
-		                                $suppress_for=1day]);
-					}
-		                }
-		}
-
-	event bro_init()
-		{
-		#Creating a new filter for all open proxy logs.
-		local filter: Log::Filter = [$name="open_proxy", $path="open_proxy", $pred=open_proxy_only];
-		Log::add_filter(HTTP::LOG, filter);
-		}
 
 ----------------
 Inspecting Files
@@ -240,43 +145,19 @@ Files are often transmitted on regular HTTP conversations between a
 client and a server. Most of the time these files are harmless, just
 images and some other multimedia content, but there are also types of
 files, specially executable files, that can damage your system. We can
-instruct Bro to create a copy of all executable files that it sees for
+instruct Bro to create a copy of all files of certain types that it sees
-later analysis using the :ref:`File Analysis Framework
+using the :ref:`File Analysis Framework <file-analysis-framework>`
-<file-analysis-framework>` (introduced with Bro 2.2) as shown in the
+(introduced with Bro 2.2):
-following script.
 
-    .. code:: bro
+.. btest-include:: ${DOC_ROOT}/httpmonitor/file_extraction.bro
 
-        global ext_map: table[string] of string = {
+.. btest:: file_extraction
-            ["application/x-dosexec"] = "exe",
-        } &default ="";
 
-        event file_new(f: fa_file)
+    @TEST-EXEC: btest-rst-cmd -n 5 bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/httpmonitor/file_extraction.bro
-            {
-            local ext = "";
 
-            if ( f?$mime_type )
+Here, the ``mime_to_ext`` table serves two purposes.  It defines which
-                ext = ext_map[f$mime_type];
+mime types to extract and also the file suffix of the extracted files.
-
+Extracted files are written to a new ``extract_files`` subdirectory.
-            local fname = fmt("%s-%s.%s", f$source, f$id, ext);
+Also note that the first conditional in the :bro:see:`file_new` event
-            Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+handler can be removed to make this behavior generic to other protocols
-            }
+besides HTTP.
-
-Bro will extract all files from the traffic and write them on a new
-``extract_files/`` subdirectory and change the file name with the right
-suffix (extension) based on the content of the ext_map table. So, if you
-want to do the same for other extracted files besides executables you
-just need to add those types to the ``ext_map`` table like this.
-
-    .. code:: bro
-
-        global ext_map: table[string] of string = {
-            ["application/x-dosexec"] = "exe",
-            ["text/plain"] = "txt",
-            ["image/jpeg"] = "jpg",
-            ["image/png"] = "png",
-            ["text/html"] = "html",
-        } &default ="";
-
-Bro will now write the appropriate suffix for text, JPEG, PNG, and HTML
-files stored in the ``extract_files/`` subdirectory.

doc/mimestats/index.rst

@@ -37,125 +37,35 @@ in the MIME type, size of the file ("response_body_len") and the
 originator host ("orig_h"). We use the MIME type as our key and create
 observers for the other two values.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
-
+    :lines: 6-29, 54-64
-	export {
-	    redef enum Log::ID += { LOG };
-		type Info: record {
-		        ## Timestamp when the log line was finished and written.
-		        ts:         time   &log;
-		        ## Time interval that the log line covers.
-		        ts_delta:   interval &log;
-		        ## The mime type
-		        mtype:        string &log;
-		        ## The number of unique local hosts that fetched this mime type
-		        uniq_hosts: count  &log;
-		        ## The number of hits to the mime type 
-		        hits:       count  &log;
-		        ## The total number of bytes received by this mime type
-		        bytes:      count  &log;
-		};
-
-		## The frequency of logging the stats collected by this script.
-		const break_interval = 5mins &redef;
-	}
-
-	event HTTP::log_http(rec: HTTP::Info)
-	{
-	    if(Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types) {
-		local mime_type = rec$resp_mime_types[0];
-		SumStats::observe("mime.bytes", [$str=mime_type], [$num=rec$response_body_len]);
-		SumStats::observe("mime.hits",  [$str=mime_type], [$str=cat(rec$id$orig_h)]);
-	    }
-	}
 
 Next, we create the reducers. The first one will accumulate file sizes
 and the second one will make sure we only store a host ID once. Below is
-the partial code.
+the partial code from a :bro:see:`bro_init` handler.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
-
+    :lines: 34-37
-        local r1: SumStats::Reducer = [$stream="mime.bytes", $apply=set(SumStats::SUM)];
-        local r2: SumStats::Reducer = [$stream="mime.hits",  $apply=set(SumStats::UNIQUE)];
 
 In our final step, we create the SumStats where we check for the
 observation interval and once it expires, we populate the record
 (defined above) with all the relevant data and write it to a log.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
-
+    :lines: 38-51
-        SumStats::create([$name="mime-metrics",
-                          $epoch=break_interval,
-                          $reducers=set(r1, r2),
-                          $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
-                                {
-                                local l: Info;
-                                l$ts         = network_time();
-                                l$ts_delta   = break_interval;
-                                l$mtype      = key$str;
-                                l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
-                                l$hits       = result["mime.hits"]$num;
-                                l$uniq_hosts = result["mime.hits"]$unique;
-                                Log::write(LOG, l);
-                                }]);
 
 Putting everything together we end up with the following final code for
 our script.
 
-  .. code:: bro
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
 
-	@load base/frameworks/sumstats
+.. btest:: mimestats
 
-	module MimeMetrics;
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/mimestats/mimestats.bro
+    @TEST-EXEC: btest-rst-include mime_metrics.log
 
-	export {
+.. note::
-	    redef enum Log::ID += { LOG };
-		type Info: record {
-		        ## Timestamp when the log line was finished and written.
-		        ts:         time   &log;
-		        ## Time interval that the log line covers.
-		        ts_delta:   interval &log;
-		        ## The mime type
-		        mtype:        string &log;
-		        ## The number of unique local hosts that fetched this mime type
-		        uniq_hosts: count  &log;
-		        ## The number of hits to the mime type 
-		        hits:       count  &log;
-		        ## The total number of bytes received by this mime type
-		        bytes:      count  &log;
-		};
-
-		## The frequency of logging the stats collected by this script.
-		const break_interval = 5mins &redef;
-	}
-
-	event bro_init() &priority=3
-	{
-	    Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
-		local r1: SumStats::Reducer = [$stream="mime.bytes", $apply=set(SumStats::SUM)];
-		local r2: SumStats::Reducer = [$stream="mime.hits",  $apply=set(SumStats::UNIQUE)];
-		SumStats::create([$name="mime-metrics",
-		                  $epoch=break_interval,
-		                  $reducers=set(r1, r2),
-		                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
-		                        {
-		                        local l: Info;
-		                        l$ts         = network_time();
-		                        l$ts_delta   = break_interval;
-		                        l$mtype      = key$str;
-		                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
-		                        l$hits       = result["mime.hits"]$num;
-		                        l$uniq_hosts = result["mime.hits"]$unique;
-		                        Log::write(LOG, l);
-		                        }]);
-	}
-
-	event HTTP::log_http(rec: HTTP::Info)
-	{
-	    if(Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types) {
-		local mime_type = rec$resp_mime_types[0];
-		SumStats::observe("mime.bytes", [$str=mime_type], [$num=rec$response_body_len]);
-		SumStats::observe("mime.hits",  [$str=mime_type], [$str=cat(rec$id$orig_h)]);
-	    }
-	}
 
+    The redefinition of :bro:see:`Site::local_nets` is only done inside
+    this script to make it a self-contained example.  It's typically
+    redefined somewhere else.

doc/mimestats/mimestats.bro

@@ -0,0 +1,64 @@
+@load base/utils/site
+@load base/frameworks/sumstats
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+module MimeMetrics;
+
+export {
+
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The mime type
+		mtype:        string &log;
+		## The number of unique local hosts that fetched this mime type
+		uniq_hosts: count  &log;
+		## The number of hits to the mime type
+		hits:       count  &log;
+		## The total number of bytes received by this mime type
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 5mins &redef;
+}
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+	local r1: SumStats::Reducer = [$stream="mime.bytes",
+	                               $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="mime.hits", 
+	                               $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$name="mime-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                        {
+	                        local l: Info;
+	                        l$ts         = network_time();
+	                        l$ts_delta   = break_interval;
+	                        l$mtype      = key$str;
+	                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
+	                        l$hits       = result["mime.hits"]$num;
+	                        l$uniq_hosts = result["mime.hits"]$unique;
+	                        Log::write(MimeMetrics::LOG, l);
+	                        }]);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+		{
+		local mime_type = rec$resp_mime_types[0];
+		SumStats::observe("mime.bytes", [$str=mime_type],
+		                  [$num=rec$response_body_len]);
+		SumStats::observe("mime.hits",  [$str=mime_type],
+		                  [$str=cat(rec$id$orig_h)]);
+		}
+	}

testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1

@@ -0,0 +1,14 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/bro.org.pcap file_extraction.bro
+      Extracting file HTTP-FiIpIB2hRQSDBOSJRg.html
+      Extracting file HTTP-FnaT2a3UDd093opCB9.txt
+      Extracting file HTTP-FsvATF146kf1Emc21j.txt
+      Extracting file HTTP-FkMQHg2nBr44fc5h63.txt
+      Extracting file HTTP-FfQGqj4Fhh3pH7nVQj.txt
+      [...]
+

testing/btest/Baseline/doc.sphinx.ftp-bruteforce/btest-doc.sphinx.ftp-bruteforce#1

@@ -0,0 +1,24 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro
+
+.. rst-class:: btest-include
+
+    .. code-block:: guess
+      :linenos:
+
+      #separator \x09
+      #set_separator	,
+      #empty_field	(empty)
+      #unset_field	-
+      #path	notice
+      #open	2014-01-21-21-56-07
+      #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+      #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+      1389721084.522861	-	-	-	-	-	-	-	-	-	FTP::Bruteforcing	192.168.56.1 had 20 failed logins on 1 FTP server in 0m37s	-	192.168.56.1	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+      #close	2014-01-21-21-56-07
+

testing/btest/Baseline/doc.sphinx.http_proxy_01/btest-doc.sphinx.http_proxy_01#1

@@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_01.bro
+      A local server is acting as an open proxy: 192.168.56.101
+

testing/btest/Baseline/doc.sphinx.http_proxy_02/btest-doc.sphinx.http_proxy_02#1

@@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_02.bro
+      A local server is acting as an open proxy: 192.168.56.101
+

testing/btest/Baseline/doc.sphinx.http_proxy_03/btest-doc.sphinx.http_proxy_03#1

@@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_03.bro
+      A local server is acting as an open proxy: 192.168.56.101
+

testing/btest/Baseline/doc.sphinx.http_proxy_04/btest-doc.sphinx.http_proxy_04#1

@@ -0,0 +1,24 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_04.bro
+
+.. rst-class:: btest-include
+
+    .. code-block:: guess
+      :linenos:
+
+      #separator \x09
+      #set_separator	,
+      #empty_field	(empty)
+      #unset_field	-
+      #path	notice
+      #open	2014-01-21-20-11-20
+      #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+      #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+      1389654450.449603	CXWv6p3arKYeMETxOg	192.168.56.1	52679	192.168.56.101	80	-	-	-	tcp	HTTP::Open_Proxy	A local server is acting as an open proxy: 192.168.56.101	-	192.168.56.1	192.168.56.101	80	-	bro	Notice::ACTION_LOG	86400.000000	F	-	-	-	-	-
+      #close	2014-01-21-20-11-20
+

testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output

@@ -0,0 +1,28 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+file_extraction.bro
+
+
+global mime_to_ext: table[string] of string = {
+	["application/x-dosexec"] = "exe",
+	["text/plain"] = "txt",
+	["image/jpeg"] = "jpg",
+	["image/png"] = "png",
+	["text/html"] = "html",
+};
+
+event file_new(f: fa_file)
+	{
+	if ( f$source != "HTTP" )
+		return;
+
+	if ( ! f?$mime_type )
+		return;
+
+	if ( f$mime_type !in mime_to_ext )
+		return;
+
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+	print fmt("Extracting file %s", fname);
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+	}

testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_01_bro/output

@@ -0,0 +1,9 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_01.bro
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_02_bro/output

@@ -0,0 +1,30 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_02.bro
+
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_03_bro/output

@@ -0,0 +1,35 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_03.bro
+
+
+@load base/utils/site
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_04_bro/output

@@ -0,0 +1,44 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_04.bro
+
+@load base/utils/site
+@load base/frameworks/notice
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	redef enum Notice::Type += {
+		Open_Proxy
+	};
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		NOTICE([$note=HTTP::Open_Proxy,
+		        $msg=fmt("A local server is acting as an open proxy: %s",
+		                 c$id$resp_h),
+		        $conn=c,
+		        $identifier=cat(c$id$resp_h),
+		        $suppress_for=1day]);
+	}

testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro/output

@@ -0,0 +1,39 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+module MimeMetrics;
+
+export {
+
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The mime type
+		mtype:        string &log;
+		## The number of unique local hosts that fetched this mime type
+		uniq_hosts: count  &log;
+		## The number of hits to the mime type
+		hits:       count  &log;
+		## The total number of bytes received by this mime type
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 5mins &redef;
+}
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+		{
+		local mime_type = rec$resp_mime_types[0];
+		SumStats::observe("mime.bytes", [$str=mime_type],
+		                  [$num=rec$response_body_len]);
+		SumStats::observe("mime.hits",  [$str=mime_type],
+		                  [$str=cat(rec$id$orig_h)]);
+		}
+	}

testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@2/output

@@ -0,0 +1,8 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+	local r1: SumStats::Reducer = [$stream="mime.bytes",
+	                               $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="mime.hits", 
+	                               $apply=set(SumStats::UNIQUE)];

testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@3/output

@@ -0,0 +1,18 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+	SumStats::create([$name="mime-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                        {
+	                        local l: Info;
+	                        l$ts         = network_time();
+	                        l$ts_delta   = break_interval;
+	                        l$mtype      = key$str;
+	                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
+	                        l$hits       = result["mime.hits"]$num;
+	                        l$uniq_hosts = result["mime.hits"]$unique;
+	                        Log::write(MimeMetrics::LOG, l);
+	                        }]);

testing/btest/Baseline/doc.sphinx.include-doc_mimestats_mimestats_bro@4/output

@@ -0,0 +1,68 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+@load base/utils/site
+@load base/frameworks/sumstats
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+module MimeMetrics;
+
+export {
+
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The mime type
+		mtype:        string &log;
+		## The number of unique local hosts that fetched this mime type
+		uniq_hosts: count  &log;
+		## The number of hits to the mime type
+		hits:       count  &log;
+		## The total number of bytes received by this mime type
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 5mins &redef;
+}
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+	local r1: SumStats::Reducer = [$stream="mime.bytes",
+	                               $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="mime.hits", 
+	                               $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$name="mime-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                        {
+	                        local l: Info;
+	                        l$ts         = network_time();
+	                        l$ts_delta   = break_interval;
+	                        l$mtype      = key$str;
+	                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
+	                        l$hits       = result["mime.hits"]$num;
+	                        l$uniq_hosts = result["mime.hits"]$unique;
+	                        Log::write(MimeMetrics::LOG, l);
+	                        }]);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+		{
+		local mime_type = rec$resp_mime_types[0];
+		SumStats::observe("mime.bytes", [$str=mime_type],
+		                  [$num=rec$response_body_len]);
+		SumStats::observe("mime.hits",  [$str=mime_type],
+		                  [$str=cat(rec$id$orig_h)]);
+		}
+	}

testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro/output

@@ -0,0 +1,21 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+module FTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates a host bruteforcing FTP logins by watching for too
+		## many rejected usernames or failed passwords.
+		Bruteforcing
+	};
+
+	## How many rejected usernames or passwords are required before being
+	## considered to be bruteforcing.
+	const bruteforce_threshold: double = 20 &redef;
+
+	## The time period in which the threshold needs to be crossed before
+	## being reset.
+	const bruteforce_measurement_interval = 15mins &redef;
+}

testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2/output

@@ -0,0 +1,13 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+	{
+	local cmd = c$ftp$cmdarg$cmd;
+	if ( cmd == "USER" || cmd == "PASS" )
+		{
+		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
+			SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
+		}
+	}

testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3/output

@@ -0,0 +1,27 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+event bro_init()
+	{
+	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)];
+	SumStats::create([$name="ftp-detect-bruteforcing",
+	                  $epoch=bruteforce_measurement_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["ftp.failed_auth"]$num+0.0;
+	                  	},
+	                  $threshold=bruteforce_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["ftp.failed_auth"];
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local plural = r$unique>1 ? "s" : "";
+	                  	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
+	                  	NOTICE([$note=FTP::Bruteforcing,
+	                  	        $src=key$host,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
+	}

testing/btest/Baseline/doc.sphinx.include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4/output

@@ -0,0 +1,64 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+##! FTP brute-forcing detector, triggering when too many rejected usernames or
+##! failed passwords have occurred from a single address.
+
+@load base/protocols/ftp
+@load base/frameworks/sumstats
+
+@load base/utils/time
+
+module FTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates a host bruteforcing FTP logins by watching for too
+		## many rejected usernames or failed passwords.
+		Bruteforcing
+	};
+
+	## How many rejected usernames or passwords are required before being
+	## considered to be bruteforcing.
+	const bruteforce_threshold: double = 20 &redef;
+
+	## The time period in which the threshold needs to be crossed before
+	## being reset.
+	const bruteforce_measurement_interval = 15mins &redef;
+}
+
+
+event bro_init()
+	{
+	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)];
+	SumStats::create([$name="ftp-detect-bruteforcing",
+	                  $epoch=bruteforce_measurement_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["ftp.failed_auth"]$num+0.0;
+	                  	},
+	                  $threshold=bruteforce_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["ftp.failed_auth"];
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local plural = r$unique>1 ? "s" : "";
+	                  	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
+	                  	NOTICE([$note=FTP::Bruteforcing,
+	                  	        $src=key$host,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
+	}
+
+event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+	{
+	local cmd = c$ftp$cmdarg$cmd;
+	if ( cmd == "USER" || cmd == "PASS" )
+		{
+		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
+			SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
+		}
+	}

testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1

@@ -0,0 +1,31 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/bro.org.pcap mimestats.bro
+
+.. rst-class:: btest-include
+
+    .. code-block:: guess
+      :linenos:
+
+      #separator \x09
+      #set_separator	,
+      #empty_field	(empty)
+      #unset_field	-
+      #path	mime_metrics
+      #open	2014-01-21-21-35-28
+      #fields	ts	ts_delta	mtype	uniq_hosts	hits	bytes
+      #types	time	interval	string	count	count	count
+      1389719059.311698	300.000000	text/html	1	4	53070
+      1389719059.311698	300.000000	image/jpeg	1	1	186859
+      1389719059.311698	300.000000	text/troff	1	1	2957
+      1389719059.311698	300.000000	application/pgp-signature	1	1	836
+      1389719059.311698	300.000000	text/plain	1	12	114205
+      1389719059.311698	300.000000	image/gif	1	1	172
+      1389719059.311698	300.000000	image/png	1	9	82176
+      1389719059.311698	300.000000	image/x-icon	1	2	2300
+      #close	2014-01-21-21-35-28
+

testing/btest/doc/sphinx/file_extraction.btest

@@ -0,0 +1 @@
+@TEST-EXEC: btest-rst-cmd -n 5 bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/httpmonitor/file_extraction.bro

testing/btest/doc/sphinx/ftp-bruteforce.btest

@@ -0,0 +1,2 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro
+@TEST-EXEC: btest-rst-include notice.log

testing/btest/doc/sphinx/http_proxy_01.btest

@@ -0,0 +1 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_01.bro

testing/btest/doc/sphinx/http_proxy_02.btest

@@ -0,0 +1 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_02.bro

testing/btest/doc/sphinx/http_proxy_03.btest

@@ -0,0 +1 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_03.bro

testing/btest/doc/sphinx/http_proxy_04.btest

@@ -0,0 +1,2 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+@TEST-EXEC: btest-rst-include notice.log

testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest

@@ -0,0 +1,28 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+file_extraction.bro
+
+
+global mime_to_ext: table[string] of string = {
+	["application/x-dosexec"] = "exe",
+	["text/plain"] = "txt",
+	["image/jpeg"] = "jpg",
+	["image/png"] = "png",
+	["text/html"] = "html",
+};
+
+event file_new(f: fa_file)
+	{
+	if ( f$source != "HTTP" )
+		return;
+
+	if ( ! f?$mime_type )
+		return;
+
+	if ( f$mime_type !in mime_to_ext )
+		return;
+
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+	print fmt("Extracting file %s", fname);
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+	}

testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_01_bro.btest

@@ -0,0 +1,9 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_01.bro
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_02_bro.btest

@@ -0,0 +1,30 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_02.bro
+
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_03_bro.btest

@@ -0,0 +1,35 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_03.bro
+
+
+@load base/utils/site
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

testing/btest/doc/sphinx/include-doc_httpmonitor_http_proxy_04_bro.btest

@@ -0,0 +1,44 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_04.bro
+
+@load base/utils/site
+@load base/frameworks/notice
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	redef enum Notice::Type += {
+		Open_Proxy
+	};
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		NOTICE([$note=HTTP::Open_Proxy,
+		        $msg=fmt("A local server is acting as an open proxy: %s",
+		                 c$id$resp_h),
+		        $conn=c,
+		        $identifier=cat(c$id$resp_h),
+		        $suppress_for=1day]);
+	}

testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro.btest

@@ -0,0 +1,39 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+module MimeMetrics;
+
+export {
+
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The mime type
+		mtype:        string &log;
+		## The number of unique local hosts that fetched this mime type
+		uniq_hosts: count  &log;
+		## The number of hits to the mime type
+		hits:       count  &log;
+		## The total number of bytes received by this mime type
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 5mins &redef;
+}
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+		{
+		local mime_type = rec$resp_mime_types[0];
+		SumStats::observe("mime.bytes", [$str=mime_type],
+		                  [$num=rec$response_body_len]);
+		SumStats::observe("mime.hits",  [$str=mime_type],
+		                  [$str=cat(rec$id$orig_h)]);
+		}
+	}

testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@2.btest

@@ -0,0 +1,8 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+	local r1: SumStats::Reducer = [$stream="mime.bytes",
+	                               $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="mime.hits", 
+	                               $apply=set(SumStats::UNIQUE)];

testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@3.btest

@@ -0,0 +1,18 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+	SumStats::create([$name="mime-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                        {
+	                        local l: Info;
+	                        l$ts         = network_time();
+	                        l$ts_delta   = break_interval;
+	                        l$mtype      = key$str;
+	                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
+	                        l$hits       = result["mime.hits"]$num;
+	                        l$uniq_hosts = result["mime.hits"]$unique;
+	                        Log::write(MimeMetrics::LOG, l);
+	                        }]);

testing/btest/doc/sphinx/include-doc_mimestats_mimestats_bro@4.btest

@@ -0,0 +1,68 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+mimestats.bro
+
+@load base/utils/site
+@load base/frameworks/sumstats
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+module MimeMetrics;
+
+export {
+
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The mime type
+		mtype:        string &log;
+		## The number of unique local hosts that fetched this mime type
+		uniq_hosts: count  &log;
+		## The number of hits to the mime type
+		hits:       count  &log;
+		## The total number of bytes received by this mime type
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 5mins &redef;
+}
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+	local r1: SumStats::Reducer = [$stream="mime.bytes",
+	                               $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="mime.hits", 
+	                               $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$name="mime-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                        {
+	                        local l: Info;
+	                        l$ts         = network_time();
+	                        l$ts_delta   = break_interval;
+	                        l$mtype      = key$str;
+	                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
+	                        l$hits       = result["mime.hits"]$num;
+	                        l$uniq_hosts = result["mime.hits"]$unique;
+	                        Log::write(MimeMetrics::LOG, l);
+	                        }]);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+		{
+		local mime_type = rec$resp_mime_types[0];
+		SumStats::observe("mime.bytes", [$str=mime_type],
+		                  [$num=rec$response_body_len]);
+		SumStats::observe("mime.hits",  [$str=mime_type],
+		                  [$str=cat(rec$id$orig_h)]);
+		}
+	}

testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro.btest

@@ -0,0 +1,21 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+module FTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates a host bruteforcing FTP logins by watching for too
+		## many rejected usernames or failed passwords.
+		Bruteforcing
+	};
+
+	## How many rejected usernames or passwords are required before being
+	## considered to be bruteforcing.
+	const bruteforce_threshold: double = 20 &redef;
+
+	## The time period in which the threshold needs to be crossed before
+	## being reset.
+	const bruteforce_measurement_interval = 15mins &redef;
+}

testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@2.btest

@@ -0,0 +1,13 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+	{
+	local cmd = c$ftp$cmdarg$cmd;
+	if ( cmd == "USER" || cmd == "PASS" )
+		{
+		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
+			SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
+		}
+	}

testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@3.btest

@@ -0,0 +1,27 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+event bro_init()
+	{
+	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)];
+	SumStats::create([$name="ftp-detect-bruteforcing",
+	                  $epoch=bruteforce_measurement_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["ftp.failed_auth"]$num+0.0;
+	                  	},
+	                  $threshold=bruteforce_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["ftp.failed_auth"];
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local plural = r$unique>1 ? "s" : "";
+	                  	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
+	                  	NOTICE([$note=FTP::Bruteforcing,
+	                  	        $src=key$host,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
+	}

testing/btest/doc/sphinx/include-scripts_policy_protocols_ftp_detect-bruteforcing_bro@4.btest

@@ -0,0 +1,64 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+detect-bruteforcing.bro
+
+##! FTP brute-forcing detector, triggering when too many rejected usernames or
+##! failed passwords have occurred from a single address.
+
+@load base/protocols/ftp
+@load base/frameworks/sumstats
+
+@load base/utils/time
+
+module FTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates a host bruteforcing FTP logins by watching for too
+		## many rejected usernames or failed passwords.
+		Bruteforcing
+	};
+
+	## How many rejected usernames or passwords are required before being
+	## considered to be bruteforcing.
+	const bruteforce_threshold: double = 20 &redef;
+
+	## The time period in which the threshold needs to be crossed before
+	## being reset.
+	const bruteforce_measurement_interval = 15mins &redef;
+}
+
+
+event bro_init()
+	{
+	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE), $unique_max=double_to_count(bruteforce_threshold+2)];
+	SumStats::create([$name="ftp-detect-bruteforcing",
+	                  $epoch=bruteforce_measurement_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["ftp.failed_auth"]$num+0.0;
+	                  	},
+	                  $threshold=bruteforce_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["ftp.failed_auth"];
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local plural = r$unique>1 ? "s" : "";
+	                  	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
+	                  	NOTICE([$note=FTP::Bruteforcing,
+	                  	        $src=key$host,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
+	}
+
+event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+	{
+	local cmd = c$ftp$cmdarg$cmd;
+	if ( cmd == "USER" || cmd == "PASS" )
+		{
+		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
+			SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
+		}
+	}

testing/btest/doc/sphinx/mimestats.btest

@@ -0,0 +1,2 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/mimestats/mimestats.bro
+@TEST-EXEC: btest-rst-include mime_metrics.log