Merge branch 'master' of ssh://git.bro-ids.org/bro

This commit is contained in:
Seth Hall 2011-09-29 13:07:37 -04:00
commit 936e6ad746
36 changed files with 321 additions and 134 deletions

58
CHANGES
View file

@ -1,4 +1,62 @@
1.6-dev-1316 | 2011-09-28 16:50:05 -0700
* Unit test cleanup. Updated README and collected coverage-related
tests in a common dir. (Jon Siwek)
* Fixes for known-services. (Seth Hall)
* Ported and 2.0ized the capture-loss script. (Seth Hall)
* Communication fix and extension.(Robin Sommer)
- Removing unnecessary log flushing. Closes #498.
- Adding new BiF disconnect() that shuts a connection to a peer down.
- terminate_connection() now first flushes any still buffered log
messages.
* Fix for high SSL memory usage by adding &transient attribute to
top-level SSL pac array type. Closes #574. (Robin Sommer)
* Fix a small bug in the metrics framework. (Seth Hall)
* Temporarily removing scripts that aren't ready to be included.
Will return before next release. (Seth Hall)
* New SSL policy scripts. (Seth Hall)
- protocols/ssl/expiring-certs uses time based information from
certificates to determine if they will expire soon, have already
expired, or haven't yet become valid.
- protocols/ssl/extract-certs-pem is a script for taking certs off
the line and converting them to PEM certificates with the openssl
command line tool then dumping them to a file.
* Notice::type_suppression_intervals: table[Notice::Type] of
interval can be used to modify the suppression intervals for
entire types of notices. (Seth Hall)
* EOF SSL protocol violations are only generated a single time now.
(Seth Hall)
* Script level fixes. (Seth Hall)
- Fixed a type name conflict in the Known namespace.
- Fixed a DPD framework bug that was causing Reporter messages.
- Fixed the notice_policy log.
- Predicate functions are now logged.
- Predicate functions are now optional. If not given, it's assumed that
the result should always apply. (Seth Hall)
- Fix a problem with accidental and mistaken HTTP log lines.
1.6-dev-1293 | 2011-09-22 19:44:37 -0700
* Smaller script tweaks. (Seth Hall)

View file

@ -1 +1 @@
1.6-dev-1293
1.6-dev-1316

@ -1 +1 @@
Subproject commit 01720883d2ba5584817964f6c30bef88b865726e
Subproject commit f90d3eded266b4effbdd607f76768dd010c7f3b5

View file

@ -42,6 +42,7 @@ rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
rest_target(${psd} base/frameworks/notice/actions/drop.bro)
rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
rest_target(${psd} base/frameworks/notice/actions/page.bro)
rest_target(${psd} base/frameworks/notice/cluster.bro)
rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
rest_target(${psd} base/frameworks/notice/main.bro)
rest_target(${psd} base/frameworks/notice/weird.bro)
@ -125,6 +126,8 @@ rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
rest_target(${psd} policy/protocols/ssh/geo-data.bro)
rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
rest_target(${psd} policy/protocols/ssh/software.bro)
rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
rest_target(${psd} policy/protocols/ssl/known-certs.bro)
rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)

View file

@ -1,3 +1,3 @@
@load ./main
@load ./postprocessors
@load ./writers/ascii

View file

@ -0,0 +1 @@
@load ./scp

View file

@ -17,4 +17,4 @@
@if ( Cluster::is_enabled() )
@load ./cluster
@endif
@endif

View file

@ -48,6 +48,10 @@ export {
status_code: count &log &optional;
## The status message returned by the server.
status_msg: string &log &optional;
## The last 1xx informational reply code returned by the server.
info_code: count &log &optional;
## The last 1xx informational reply message returned by the server.
info_msg: string &log &optional;
## The filename given in the Content-Disposition header
## sent by the server.
filename: string &log &optional;
@ -111,6 +115,11 @@ redef capture_filters += {
["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)"
};
function code_in_range(c: count, min: count, max: count) : bool
{
return c >= min && c <= max;
}
function new_http_session(c: connection): Info
{
local tmp: Info;
@ -163,12 +172,21 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
local s: State;
c$http_state = s;
}
++c$http_state$current_response;
# If the last response was an informational 1xx, we're still expecting
# the real response to the request, so don't create a new Info record yet.
if ( c$http_state$current_response !in c$http_state$pending ||
! code_in_range(c$http_state$pending[c$http_state$current_response]$status_code, 100, 199) )
++c$http_state$current_response;
set_state(c, F, F);
c$http$status_code = code;
c$http$status_msg = reason;
if ( code_in_range(code, 100, 199) )
{
c$http$info_code = code;
c$http$info_msg = reason;
}
}
event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5
@ -245,8 +263,13 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
# The reply body is done so we're ready to log.
if ( ! is_orig )
{
Log::write(HTTP::LOG, c$http);
delete c$http_state$pending[c$http_state$current_response];
# If the response was an informational 1xx, we're still expecting
# the real response later, so we'll continue using the same record.
if ( ! code_in_range(c$http$status_code, 100, 199) )
{
Log::write(HTTP::LOG, c$http);
delete c$http_state$pending[c$http_state$current_response];
}
}
}

View file

@ -5,6 +5,7 @@
@load base/protocols/ssl
@load base/frameworks/notice
@load base/utils/directions-and-hosts
module SSL;
@ -58,5 +59,3 @@ event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: co
$conn=c, $suppress_for=1day,
$identifier=fmt("%s:%d-%s", c$id$resp_h, c$id$resp_p, md5_hash(der_cert))]);
}

View file

@ -14,6 +14,7 @@
##!
@load base/protocols/ssl
@load base/utils/directions-and-hosts
module SSL;
@ -45,4 +46,4 @@ event ssl_established(c: connection)
local side = Site::is_local_addr(c$id$resp_h) ? "local" : "remote";
local cmd = fmt("%s x509 -inform DER -outform PEM >> certs-%s.pem", openssl_util, side);
piped_exec(cmd, c$ssl$cert);
}
}

View file

@ -49,6 +49,8 @@
@load protocols/ssh/geo-data.bro
@load protocols/ssh/interesting-hostnames.bro
@load protocols/ssh/software.bro
@load protocols/ssl/expiring-certs.bro
@load protocols/ssl/extract-certs-pem.bro
@load protocols/ssl/known-certs.bro
@load protocols/ssl/validate-certs.bro
@load tuning/__load__.bro

View file

@ -5,6 +5,8 @@
#include <ctype.h>
#include <math.h>
#include <stdlib.h>
#include <string>
#include <algorithm>
#include "NetVar.h"
#include "HTTP.h"
@ -310,6 +312,67 @@ void HTTP_Entity::SubmitHeader(MIME_Header* h)
}
}
// Figure out content-length for HTTP 206 Partial Content response
// that uses multipart/byteranges content-type.
else if ( strcasecmp_n(h->get_name(), "content-range") == 0 && Parent() &&
Parent()->MIMEContentType() == CONTENT_TYPE_MULTIPART &&
http_message->MyHTTP_Analyzer()->HTTP_ReplyCode() == 206 )
{
data_chunk_t vt = h->get_value_token();
string byte_unit(vt.data, vt.length);
vt = h->get_value_after_token();
string byte_range(vt.data, vt.length);
byte_range.erase(remove(byte_range.begin(), byte_range.end(), ' '),
byte_range.end());
if ( byte_unit != "bytes" )
{
http_message->Weird("HTTP_content_range_unknown_byte_unit");
return;
}
size_t p = byte_range.find("/");
if ( p == string::npos )
{
http_message->Weird("HTTP_content_range_cannot_parse");
return;
}
string byte_range_resp_spec = byte_range.substr(0, p);
string instance_length = byte_range.substr(p + 1);
p = byte_range_resp_spec.find("-");
if ( p == string::npos )
{
http_message->Weird("HTTP_content_range_cannot_parse");
return;
}
string first_byte_pos = byte_range_resp_spec.substr(0, p);
string last_byte_pos = byte_range_resp_spec.substr(p + 1);
if ( DEBUG_http )
DEBUG_MSG("Parsed Content-Range: %s %s-%s/%s\n", byte_unit.c_str(),
first_byte_pos.c_str(), last_byte_pos.c_str(),
instance_length.c_str());
int64_t f, l;
atoi_n(first_byte_pos.size(), first_byte_pos.c_str(), 0, 10, f);
atoi_n(last_byte_pos.size(), last_byte_pos.c_str(), 0, 10, l);
int64_t len = l - f + 1;
if ( DEBUG_http )
DEBUG_MSG("Content-Range length = %"PRId64"\n", len);
if ( len > 0 )
content_length = len;
else
{
http_message->Weird("HTTP_non_positive_content_range");
return;
}
}
else if ( strcasecmp_n(h->get_name(), "transfer-encoding") == 0 )
{
data_chunk_t vt = h->get_value_token();
@ -1305,7 +1368,9 @@ void HTTP_Analyzer::ReplyMade(const int interrupted, const char* msg)
if ( reply_message )
reply_message->Done(interrupted, msg);
if ( ! unanswered_requests.empty() )
// 1xx replies do not indicate the final response to a request,
// so don't pop an unanswered request in that case.
if ( (reply_code < 100 || reply_code >= 200) && ! unanswered_requests.empty() )
{
Unref(unanswered_requests.front());
unanswered_requests.pop();

View file

@ -163,6 +163,9 @@ public:
void SkipEntityData(int is_orig);
int IsConnectionClose() { return connection_close; }
int HTTP_ReplyCode() const { return reply_code; };
// Overriden from Analyzer.
virtual void Done();
virtual void DeliverStream(int len, const u_char* data, bool orig);
@ -183,8 +186,6 @@ public:
http_content_type || http_entity_data || http_message_done ||
http_event || http_stats) && !FLAGS_use_binpac; }
int IsConnectionClose() { return connection_close; }
protected:
void GenStats();

View file

@ -95,6 +95,7 @@ public:
virtual void EndOfData();
MIME_Entity* Parent() const { return parent; }
int MIMEContentType() const { return content_type; }
StringVal* ContentType() const { return content_type_str; }
StringVal* ContentSubType() const { return content_subtype_str; }
int ContentTransferEncoding() const { return content_encoding; }

View file

@ -12,5 +12,7 @@
1 scripts/base/frameworks/logging/__load__.bro
2 scripts/base/frameworks/logging/./main.bro
3 build/src/base/logging.bif.bro
2 scripts/base/frameworks/logging/./postprocessors/__load__.bro
3 scripts/base/frameworks/logging/./postprocessors/./scp.bro
2 scripts/base/frameworks/logging/./writers/ascii.bro
0 scripts/policy/misc/loaded-scripts.bro

View file

@ -12,6 +12,8 @@
1 scripts/base/frameworks/logging/__load__.bro
2 scripts/base/frameworks/logging/./main.bro
3 build/src/base/logging.bif.bro
2 scripts/base/frameworks/logging/./postprocessors/__load__.bro
3 scripts/base/frameworks/logging/./postprocessors/./scp.bro
2 scripts/base/frameworks/logging/./writers/ascii.bro
0 scripts/base/init-default.bro
1 scripts/base/utils/site.bro

View file

@ -0,0 +1,6 @@
-./frameworks/cluster/nodes/manager.bro
-./frameworks/cluster/nodes/proxy.bro
-./frameworks/cluster/nodes/worker.bro
-./frameworks/cluster/setup-connections.bro
-./frameworks/metrics/cluster.bro
-./frameworks/notice/cluster.bro

View file

@ -1,5 +1,5 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file
1316124231.969273 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file
1317149787.593092 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - -

View file

@ -1,5 +1,5 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file
1316124231.969273 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file
1317149787.593092 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - -

View file

@ -1,5 +1,5 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file
1316124240.720195 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file
1317149750.648989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - -

View file

@ -1,5 +1,5 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file
1316124240.720195 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file
1317149750.648989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - -

View file

@ -1,5 +1,5 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file
1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 F 346 F 404 Not Found - - - - - text/html - -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file
1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 F 346 F 404 Not Found - - - - - - - text/html - -

View file

@ -0,0 +1,5 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file
1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 F 60731 F 200 OK 100 Continue - - - - - text/html - -

View file

@ -1,9 +1,9 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file
1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - FAKE_MIME - -
1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - FAKE_MIME - -
1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - FAKE_MIME - -
1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - image/png e0029eea80812e9a8e57b8d05d52938a -
1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - image/png 30aa926344f58019d047e85ba049ca1e -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file
1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - - - FAKE_MIME - -
1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - - - FAKE_MIME - -
1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - - - FAKE_MIME - -
1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - - - image/png e0029eea80812e9a8e57b8d05d52938a -
1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - - - image/png 30aa926344f58019d047e85ba049ca1e -

View file

@ -1,9 +1,9 @@
#separator \x09
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string string table string string table string file
1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - - -
1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - - -
1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - - -
1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - - -
1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - - -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file
#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string file
1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - - - - -
1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - - - - -
1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - - - - -
1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - - - - -
1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - - - - -

View file

@ -1,97 +1,85 @@
BTest is simple framework for writing unit tests. Each test consists of a set
of command lines that will be executed, and success is determined based on
their exit codes. In addition, output can optionally be compared against a
previously established baseline.
This a test suite of small "unit tests" that verify individual pieces of Bro
functionality. They all utilize BTest, a simple framework/driver for
writing unit tests. More information about BTest can be found at
http://www.bro-ids.org/development/btest.html
More information about BTest can be found at http://www.icir.org/robin/btest/
The test suite's BTest configuration is handled through the
``btest.cfg`` file. Of particular interest is the "TestDirs" settings,
which specifies which directories BTest will recursively search for
test files.
Significant Subdirectories
==========================
This README contains the following sections:
* Contents of the testing/btest/ directory
* Running tests
* Adding tests
* Baseline/
Validated baselines for comparison against the output of each
test on future runs. If the new output differs from the Baseline
output, then the test fails.
Contents of the testing/btest/ directory:
Baseline/*/
The validated baselines for comparison against the output of each test on
future runs. If the new output differs from the Baseline output, then the
test fails.
Scripts/
Shell scripts invoked by BTest to support testing.
Traces/
* Traces/
Packet captures utilized by the various BTest tests.
logging/
Tests to validate the logging framework.
* scripts/
This hierarchy of tests emulates the hierarchy of the Bro scripts/
directory.
policy/
Tests of the functionality of Bro's bundled policy scripts.
* coverage/
This collection of tests relates to checking whether we're covering
everything we want to in terms of tests, documentation, and which
scripts get loaded in different Bro configurations. These tests are
more prone to fail as new Bro scripts are developed and added to the
distribution -- checking the individual test's comments is the best
place to check for more details on what exactly the test is checking
and hints on how to fix it when it fails.
software/
Tests to validate Bro software not tested elsewhere.
Running Tests
=============
btest.cfg
Configuration file that specifies run-time settings for BTest. Of particular
interest is the "TestDirs" settings, which specifies which directories BTest
will recursively search for test files.
Either use the ``make all`` or ``make brief`` ``Makefile`` targets, or
run ``btest`` directly with desired options/arguments. Examples:
* btest <no arguments>
If you simply execute btest in this directory with no arguments,
then all directories listed as "TestDirs" in btest.cfg will be
searched recursively for test files.
Running tests:
btest <no arguments>
If you simply execute btest in this directory with no arguments, then all
directories listed as "TestDirs" in btest.cfg will be searched recursively
for test files. This is how the NMI automated build & test environment
invokes BTest to run all tests.
* btest <btest options> test_directory
You can specify a directory on the command line to run just the
tests contained in that directory. This is useful if you wish to
run all of a given type of test, without running all the tests
there are. For example, "btest scripts" will run all of the Bro
script unit tests.
btest test_directory
You can specify a directory on the command line to run just the tests
contained in that directory. This is useful if you wish to run all of a
given type of test, without running all the tests there are. For example,
"btest policy" will run all of the tests for Bro's bundled policy scripts.
btest test_directory/test_file
You can specify a single test file to run just that test. This is useful
when testing a single aspect of Bro functionality, and also when developing
* btest <btest options> test_directory/test_file
You can specify a single test file to run just that test. This
is useful when testing a single failing test or when developing
a new test.
Adding Tests
=============
See either the `BTest documentation
<http://www.bro-ids.org/development/btest.html>`_ or the existing unit
tests for examples of what they actually look like. The essential
components of a new test include:
Adding tests:
* A test file in one of the subdirectories listed in the ``TestDirs``
of the ``btest.cfg`` file.
See the documentation at http://www.icir.org/robin/btest/ for information on
what BTests actually look like.
* If the unit test requires a known-good baseline output against which
future tests will be compared (via ``btest-diff``), then that baseline
output will need to live in the ``Baseline`` directory. Manually
adding that is possible, but it's easier to just use the ``-u`` or
``-U`` options of ``btest`` to do it for you (using ``btest -d`` on a
test for which no baseline exists will show you the output so it can
be verified first before adding/updating the baseline output).
The essential components of a new test include:
* A test file in a subdirectory of /testing/btest. This can be a sub-sub-
directory, as the search for test files is recursive from the directories
listed as "TestDirs" in btest.cfg
* A baseline for the output of your test. Although the baseline will be stored
in testing/btest/Baseline/ you should allow btest to copy the correct files
to that location, rather than copying them manually (see below).
If you create a new top-level testing directory for collecting related
tests, then you'll need to add it to the list of ``TestDirs`` in
``btest.cfg``. Do this only if your test really doesn't fit logically in
any of the extant directories.
If you create a new subdirectory from testing/btest you'll need to add it to the
list of "TestDirs" in btest.cfg. Do this only if your test really doesn't fit
logically in any of the extant directories.
While developing your test, you can specify the "-t" command-line option to make
BTest preserve the testing/btest/.tmp directory. This directory holds the output
from your test run; you can inspect it in place to ensure it is correct and as
expected.
Once you are satisfied with the results in testing/btest/.tmp you can make BTest
store this output as the Baseline for the test by specifying the "-U" command-
line option.
When you are ready to commit your test to git, be sure the testing/btest/.tmp
directory is deleted, and use "git status" to ensure you correctly identify all
of the files that should be committed to the repository.
Note that any new test you add this way will automatically be included in the
testing done in the NMI automated build & test environment.
Note that any new test you add this way will automatically be included
in the testing done in the NMI automated build & test environment.

Binary file not shown.

View file

@ -1,5 +1,5 @@
[btest]
TestDirs = doc bifs language core scripts istate
TestDirs = doc bifs language core scripts istate coverage
TmpDir = %(testbase)s/.tmp
BaselineDir = %(testbase)s/Baseline
IgnoreDirs = .svn CVS .tmp

View file

@ -1,5 +1,7 @@
# This test is meant to cover whether the set of scripts that get loaded by
# default in bare mode matches a baseline of known defaults.
# default in bare mode matches a baseline of known defaults. The baseline
# should only need updating if something new is @load'd from init-bare.bro
# (or from an @load'd descendent of it).
#
# As the output has absolute paths in it, we need to remove the common
# prefix to make the test work everywhere. That's what the sed magic
@ -7,6 +9,6 @@
# @TEST-EXEC: bro -b misc/loaded-scripts
# @TEST-EXEC: test -e loaded_scripts.log
# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
# @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
# @TEST-EXEC: btest-diff canonified_loaded_scripts.log

View file

@ -1,6 +1,9 @@
# Makes sure any given policy script in the scripts/ tree can be loaded in
# bare mode. btest-bg-run/btest-bg-wait are used to kill off scripts that
# block after loading, e.g. start listening on a socket.
# Makes sure any given bro script in the scripts/ tree can be loaded in
# bare mode without error. btest-bg-run/btest-bg-wait are used to kill off
# scripts that block after loading, e.g. start listening on a socket.
#
# Commonly, this test may fail if one forgets to @load some base/ scripts
# when writing a new bro scripts.
#
# @TEST-EXEC: test -d $DIST/scripts
# @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro`; do echo $script;if [[ "$script" =~ listen-clear|listen-ssl|controllee ]]; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0

View file

@ -1,5 +1,7 @@
# This test is meant to cover whether the set of scripts that get loaded by
# default matches a baseline of known defaults.
# default matches a baseline of known defaults. When new scripts are
# added to the scripts/base/ directory, the baseline will usually just need
# to be updated.
#
# As the output has absolute paths in it, we need to remove the common
# prefix to make the test work everywhere. That's what the sed magic
@ -7,6 +9,6 @@
# @TEST-EXEC: bro misc/loaded-scripts
# @TEST-EXEC: test -e loaded_scripts.log
# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
# @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
# @TEST-EXEC: btest-diff canonified_loaded_scripts.log

View file

@ -1,5 +1,5 @@
# This tests that we're generating policy script documentation for all the
# available policy scripts. If this fails, then the genDocSources.sh needs
# This tests that we're generating bro script documentation for all the
# available bro scripts. If this fails, then the genDocSources.sh needs
# to be run to produce a new DocSourcesList.cmake or genDocSources.sh needs
# to be updated to blacklist undesired scripts.
#

View file

@ -1,11 +1,18 @@
# Makes sure that all base/* scripts are loaded by default via init-default.bro;
# and that all scripts loaded there in there actually exist.
#
# This test will fail if a new bro script is added under the scripts/base/
# directory and it is not also added as an @load in base/init-default.bro.
# In some cases, a script in base is loaded based on the bro configuration
# (e.g. cluster operation), and in such cases, the missing_loads baseline
# can be adjusted to tolerate that.
#@TEST-EXEC: test -d $DIST/scripts/base
#@TEST-EXEC: test -e $DIST/scripts/base/init-default.bro
#@TEST-EXEC: ( cd $DIST/scripts/base && find . -name '*.bro' ) | sort >"all scripts found"
#@TEST-EXEC: bro misc/loaded-scripts
#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro|#' | awk 'NR>1{print $2}' | sed 's#/./#/#g' >loaded_scripts.log.tmp
#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro|#' | awk 'NR>0{print $2}' | sed 's#/./#/#g' >loaded_scripts.log.tmp
#@TEST-EXEC: cat loaded_scripts.log.tmp | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
#@TEST-EXEC: cat loaded_scripts.log.tmp | sed "s#`cat prefix`#./#g" | sort >init-default.bro
#@TEST-EXEC: diff -u "all scripts found" init-default.bro 1>&2
#@TEST-EXEC: diff -u "all scripts found" init-default.bro | egrep "^-[^-]" > missing_loads
#@TEST-EXEC: btest-diff missing_loads

View file

@ -1,5 +1,9 @@
# Makes sure that all policy/* scripts are loaded in test-all-policy.bro; and that
# all scripts loaded there actually exist.
# Makes sure that all policy/* scripts are loaded in
# scripts/test-all-policy.bro and that all scripts loaded there actually exist.
#
# This test will fail if new bro scripts are added to the scripts/policy/
# directory. Correcting that just involves updating scripts/test-all-policy.bro
# to @load the new bro scripts.
@TEST-EXEC: test -e $DIST/scripts/test-all-policy.bro
@TEST-EXEC: test -d $DIST/scripts

View file

@ -0,0 +1,12 @@
# This tests that the HTTP analyzer does not generate an unmatched_HTTP_reply
# weird as a result of seeing both a 1xx response and the real response to
# a given request. The http scripts should also be able log such replies
# in a way that correlates the final response with the request.
#
# @TEST-EXEC: bro -r $TRACES/http-100-continue.trace %INPUT
# @TEST-EXEC: grep -q unmatched_HTTP_reply weird.log && exit 1 || exit 0
# @TEST-EXEC: btest-diff http.log
# The base analysis scripts are loaded by default.
#@load base/protocols/http