Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch 'master' into topic/jsiwek/broxygen

Browse source
This commit is contained in:
Jon Siwek 2013-11-21 15:46:55 -06:00
commit 96ba5c82a3
41 changed files with 717 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

44
CHANGES
View file

@ -1,4 +1,48 @@
2.2-9 | 2013-11-18 14:03:21 -0800
* Update local.bro for Bro >= 2.2. The commented out Notice::policy
example didn't work anymore. (Daniel Thayer)
2.2-6 | 2013-11-15 07:05:15 -0800
* Make "install-example-configs" target use DESTDIR. (Jon Siwek)
2.2-5 | 2013-11-11 13:47:54 -0800
* Fix the irc_reply event for certain server message types. (Seth
Hall)
* Fixed Segmentation fault in SQLite Writer. (Jon Crussell)
2.2 | 2013-11-07 10:25:50 -0800
* Release 2.2.
* Removing location information from ssh.log in external tests.
(Robin Sommer)
2.2-beta-199 | 2013-11-07 00:36:46 -0800
* Fixing warnings during doc build. (Robin Sommer)
2.2-beta-198 | 2013-11-06 22:54:30 -0800
* Update docs and tests for a recent change to detect-MHR.bro
(Daniel Thayer)
* Update tests and baselines for sumstats docs. (Daniel Thayer)
2.2-beta-194 | 2013-11-06 14:39:50 -0500
* Remove resp_size from the ssh log. Refactor when we write out to
the log a bit. Geodata now works reliably. (Vlad Grigorescu)
* Update VirusTotal URL to work with changes to their website and
changed it to a redef. (Vlad Grigorescu)
* Added a document for the SumStats framework. (Seth Hall)
2.2-beta-184 | 2013-11-03 22:53:42 -0800
* Remove swig-ruby from required packages section of install doc.

4
NEWS
View file

@ -4,8 +4,8 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file
(note that submodules, such as BroControl and Broccoli, come with
their own ``CHANGES``.)
Bro 2.2 Beta
============
Bro 2.2
=======
New Functionality
-----------------

2
VERSION
View file

@ -1 +1 @@
2.2-beta-184
2.2-9

2
aux/binpac

@ -1 +1 @@
Subproject commit 0f20a50afacb68154b4035b6da63164d154093e4
Subproject commit 54b321009b750268526419bdbd841f421c839313

2
aux/bro-aux

@ -1 +1 @@
Subproject commit ce8a9733c4f8c4c8e8b2e9e440acfcf985f39cd8
Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8

2
aux/broccoli

@ -1 +1 @@
Subproject commit 02e5c1e1f993ef0fea3e2a59c34df9f40839e398
Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d

2
aux/broctl

@ -1 +1 @@
Subproject commit aeb8501a50dcf33c53e7fe776b6e333327c11861
Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f

2
aux/btest

@ -1 +1 @@
Subproject commit cfc8fe7ddf5ba3a9f957d1d5a98e9cfe1e9692ac
Subproject commit 26c3136d56493017bc33c5a2f22ae393d585c2d9

2
cmake

@ -1 +1 @@
Subproject commit d902e23fd14624eb9caf0b4a0e693014bf5bd684
Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20

1
doc/frameworks/index.rst
View file

@ -13,4 +13,5 @@ Frameworks
logging
notice
signatures
sumstats

36
doc/frameworks/sumstats-countconns.bro Normal file
View file

@ -0,0 +1,36 @@
@load base/frameworks/sumstats
event connection_established(c: connection)
{
# Make an observation!
# This observation is global so the key is empty.
# Each established connection counts as one so the observation is always 1.
SumStats::observe("conn established",
SumStats::Key(),
SumStats::Observation($num=1));
}
event bro_init()
{
# Create the reducer.
# The reducer attaches to the "conn established" observation stream
# and uses the summing calculation on the observations.
local r1 = SumStats::Reducer($stream="conn established",
$apply=set(SumStats::SUM));
# Create the final sumstat.
# We give it an arbitrary name and make it collect data every minute.
# The reducer is then attached and a $epoch_result callback is given
# to finally do something with the data collected.
SumStats::create([$name = "counting connections",
$epoch = 1min,
$reducers = set(r1),
$epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
{
# This is the body of the callback that is called when a single
# result has been collected. We are just printing the total number
# of connections that were seen. The $sum field is provided as a
# double type value so we need to use %f as the format specifier.
print fmt("Number of connections established: %.0f", result["conn established"]$sum);
}]);
}

45
doc/frameworks/sumstats-toy-scan.bro Normal file
View file

@ -0,0 +1,45 @@
@load base/frameworks/sumstats
# We use the connection_attempt event to limit our observations to those
# which were attempted and not successful.
event connection_attempt(c: connection)
{
# Make an observation!
# This observation is about the host attempting the connection.
# Each established connection counts as one so the observation is always 1.
SumStats::observe("conn attempted",
SumStats::Key($host=c$id$orig_h),
SumStats::Observation($num=1));
}
event bro_init()
{
# Create the reducer.
# The reducer attaches to the "conn attempted" observation stream
# and uses the summing calculation on the observations. Keep
# in mind that there will be one result per key (connection originator).
local r1 = SumStats::Reducer($stream="conn attempted",
$apply=set(SumStats::SUM));
# Create the final sumstat.
# This is slightly different from the last example since we're providing
# a callback to calculate a value to check against the threshold with
# $threshold_val. The actual threshold itself is provided with $threshold.
# Another callback is provided for when a key crosses the threshold.
SumStats::create([$name = "finding scanners",
$epoch = 5min,
$reducers = set(r1),
# Provide a threshold.
$threshold = 5.0,
# Provide a callback to calculate a value from the result
# to check against the threshold field.
$threshold_val(key: SumStats::Key, result: SumStats::Result) =
{
return result["conn attempted"]$sum;
},
# Provide a callback for when a key crosses the threshold.
$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
{
print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum);
}]);
}

102
doc/frameworks/sumstats.rst Normal file
View file

@ -0,0 +1,102 @@
==================
Summary Statistics
==================
.. rst-class:: opening
Measuring aspects of network traffic is an extremely common task in Bro.
Bro provides data structures which make this very easy as well in
simplistic cases such as size limited trace file processing. In
real-world deployments though, there are difficulties that arise from
clusterization (many processes sniffing traffic) and unbounded data sets
(traffic never stops). The Summary Statistics (otherwise referred to as
SumStats) framework aims to define a mechanism for consuming unbounded
data sets and making them measurable in practice on large clustered and
non-clustered Bro deployments.
.. contents::
Overview
========
The Sumstat processing flow is broken into three pieces. Observations, where
some aspect of an event is observed and fed into the Sumstats framework.
Reducers, where observations are collected and measured, typically by taking
some sort of summary statistic measurement like average or variance (among
others). Sumstats, where reducers have an epoch (time interval) that their
measurements are performed over along with callbacks for monitoring thresholds
or viewing the collected and measured data.
Terminology
===========
Observation
A single point of data. Observations have a few components of their
own. They are part of an arbitrarily named observation stream, they
have a key that is something the observation is about, and the actual
observation itself.
Reducer
Calculations are applied to an observation stream here to reduce the
full unbounded set of observations down to a smaller representation.
Results are collected within each reducer per-key so care must be
taken to keep the total number of keys tracked down to a reasonable
level.
Sumstat
The final definition of a Sumstat where one or more reducers is
collected over an interval, also known as an epoch. Thresholding can
be applied here along with a callback in the event that a threshold is
crossed. Additionally, a callback can be provided to access each
result (per-key) at the end of each epoch.
Examples
========
These examples may seem very simple to an experienced Bro script developer and
they're intended to look that way. Keep in mind that these scripts will work
on small single process Bro instances as well as large many-worker clusters.
The complications from dealing with flow based load balancing can be ignored
by developers writing scripts that use Sumstats due to its built-in cluster
transparency.
Printing the number of connections
----------------------------------
Sumstats provides a simple way of approaching the problem of trying to count
the number of connections over a given time interval. Here is a script with
inline documentation that does this with the Sumstats framework:
.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-countconns.bro
When run on a sample PCAP file from the Bro test suite, the following output
is created:
.. btest:: sumstats-countconns
@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro
Toy scan detection
------------------
Taking the previous example even further, we can implement a simple detection
to demonstrate the thresholding functionality. This example is a toy to
demonstrate how thresholding works in Sumstats and is not meant to be a
real-world functional example, that is left to the
:doc:`/scripts/policy/misc/scan` script that is included with Bro.
.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro
Let's see if there are any hosts that crossed the threshold in a PCAP file
containing a host running nmap:
.. btest:: sumstats-toy-scan
@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro
It seems the host running nmap was detected!

4
doc/scripting/index.rst
View file

@ -66,7 +66,7 @@ are ensuring the Files framework, the Notice framework and the script to hash al
been loaded by Bro.
.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
:lines: 10-31
:lines: 10-36
The export section redefines an enumerable constant that describes the
type of notice we will generate with the Notice framework. Bro
@ -87,7 +87,7 @@ Up until this point, the script has merely done some basic setup. With the next
the script starts to define instructions to take in a given event.
.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
:lines: 33-57
:lines: 38-62
The workhorse of the script is contained in the event handler for
``file_hash``. The :bro:see:`file_hash` event allows scripts to access

8
doc/scripts/notices.rst Normal file
View file

@ -0,0 +1,8 @@
.. Not nice but I don't find a way to link to the notice index
.. directly from the upper level TOC tree.
Notices
=======
See the `Bro Notice Index <../bro-noticeindex.html>`_.

3
scripts/base/frameworks/packet-filter/cluster.bro
View file

@ -1,4 +1,7 @@
@load base/frameworks/cluster
@load ./main
module PacketFilter;
event remote_connection_handshake_done(p: event_peer) &priority=3

32
scripts/base/protocols/ssh/main.bro
View file

@ -37,12 +37,6 @@ export {
client: string &log &optional;
## Software string from the server.
server: string &log &optional;
## Amount of data returned from the server. This is currently
## the only measure of the success heuristic and it is logged to
## assist analysts looking at the logs to make their own
## determination about the success on a case-by-case basis.
resp_size: count &log &default=0;
## Indicate if the SSH session is done being watched.
done: bool &default=F;
};
@ -107,10 +101,10 @@ function check_ssh_connection(c: connection, done: bool)
# this matches the conditions for a failed login. Failed
# logins are only detected at connection state removal.
if ( # Require originators to have sent at least 50 bytes.
c$orig$size > 50 &&
if ( # Require originators and responders to have sent at least 50 bytes.
c$orig$size > 50 && c$resp$size > 50 &&
# Responders must be below 4000 bytes.
c$resp$size < 4000 &&
c$resp$size < authentication_data_size &&
# Responder must have sent fewer than 40 packets.
c$resp$num_pkts < 40 &&
# If there was a content gap we can't reliably do this heuristic.
@ -122,7 +116,7 @@ function check_ssh_connection(c: connection, done: bool)
event SSH::heuristic_failed_login(c);
}
if ( c$resp$size > authentication_data_size )
if ( c$resp$size >= authentication_data_size )
{
c$ssh$status = "success";
event SSH::heuristic_successful_login(c);
@ -132,7 +126,7 @@ function check_ssh_connection(c: connection, done: bool)
{
# If this connection is still being tracked, then it's possible
# to watch for it to be a successful connection.
if ( c$resp$size > authentication_data_size )
if ( c$resp$size >= authentication_data_size )
{
c$ssh$status = "success";
event SSH::heuristic_successful_login(c);
@ -150,8 +144,6 @@ function check_ssh_connection(c: connection, done: bool)
# after detection is done.
c$ssh$done=T;
Log::write(SSH::LOG, c$ssh);
if ( skip_processing_after_detection )
{
# Stop watching this connection, we don't care about it anymore.
@ -161,10 +153,24 @@ function check_ssh_connection(c: connection, done: bool)
}
event heuristic_successful_login(c: connection) &priority=-5
{
Log::write(SSH::LOG, c$ssh);
}
event heuristic_failed_login(c: connection) &priority=-5
{
Log::write(SSH::LOG, c$ssh);
}
event connection_state_remove(c: connection) &priority=-5
{
if ( c?$ssh )
{
check_ssh_connection(c, T);
if ( c$ssh$status == "undetermined" )
Log::write(SSH::LOG, c$ssh);
}
}
event ssh_watcher(c: connection)

7
scripts/policy/frameworks/files/detect-MHR.bro
View file

@ -23,6 +23,11 @@ export {
/application\/jar/ |
/video\/mp4/ &redef;
## The Match notice has a sub message with a URL where you can get more
## information about the file. The %s will be replaced with the SHA-1
## hash of the file.
const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
## The malware hash registry runs each malware sample through several
## A/V engines. Team Cymru returns a percentage to indicate how
## many A/V engines flagged the sample as malicious. This threshold
@ -48,7 +53,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
if ( mhr_detect_rate >= notice_threshold )
{
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
local virustotal_url = fmt(match_sub_url, hash);
NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
}
}

22
scripts/policy/protocols/ssh/geo-data.bro
View file

@ -24,21 +24,29 @@ export {
const watched_countries: set[string] = {"RO"} &redef;
}
function get_location(c: connection): geo_location
{
local lookup_ip = (c$ssh$direction == OUTBOUND) ? c$id$resp_h : c$id$orig_h;
return lookup_location(lookup_ip);
}
event SSH::heuristic_successful_login(c: connection) &priority=5
{
local location: geo_location;
location = (c$ssh$direction == OUTBOUND) ?
lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
# Add the location data to the SSH record.
c$ssh$remote_location = location;
c$ssh$remote_location = get_location(c);
if ( location?$country_code && location$country_code in watched_countries )
if ( c$ssh$remote_location?$country_code && c$ssh$remote_location$country_code in watched_countries )
{
NOTICE([$note=Watched_Country_Login,
$conn=c,
$msg=fmt("SSH login %s watched country: %s",
(c$ssh$direction == OUTBOUND) ? "to" : "from",
location$country_code)]);
c$ssh$remote_location$country_code)]);
}
}
event SSH::heuristic_failed_login(c: connection) &priority=5
{
# Add the location data to the SSH record.
c$ssh$remote_location = get_location(c);
}

4
scripts/site/local.bro
View file

@ -30,10 +30,6 @@
# This adds signatures to detect cleartext forward and reverse windows shells.
@load-sigs frameworks/signatures/detect-windows-shells
# Uncomment the following line to begin receiving (by default hourly) emails
# containing all of your notices.
# redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 0] };
# Load all of the scripts that detect software in various protocols.
@load protocols/ftp/software
@load protocols/smtp/software

3
src/analyzer/protocol/irc/IRC.cc
View file

@ -164,7 +164,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig)
if ( code > 0 )
{
switch ( code ) {
// Ignore unimportant messages.
/*
case 1: // RPL_WELCOME
case 2: // RPL_YOURHOST
case 3: // RPL_CREATED
@ -183,6 +183,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig)
case 376: // RPL_ENDOFMOTD
case 331: // RPL_NOTOPIC
break;
*/
// Count of users, services and servers in whole network.
case 251:

4
src/logging/writers/SQLite.cc
View file

@ -308,7 +308,7 @@ int SQLite::AddParams(Value* val, int pos)
if ( j > 0 )
desc.AddRaw(set_separator);
io->Describe(&desc, val->val.set_val.vals[j], fields[pos]->name);
io->Describe(&desc, val->val.set_val.vals[j], fields[pos-1]->name);
}
desc.RemoveEscapeSequence(set_separator);
@ -330,7 +330,7 @@ int SQLite::AddParams(Value* val, int pos)
if ( j > 0 )
desc.AddRaw(set_separator);
io->Describe(&desc, val->val.vector_val.vals[j], fields[pos]->name);
io->Describe(&desc, val->val.vector_val.vals[j], fields[pos-1]->name);
}
desc.RemoveEscapeSequence(set_separator);

40
testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output Normal file
View file

@ -0,0 +1,40 @@
# @TEST-EXEC: cat %INPUT >output && btest-diff output
sumstats-countconns.bro
@load base/frameworks/sumstats
event connection_established(c: connection)
{
# Make an observation!
# This observation is global so the key is empty.
# Each established connection counts as one so the observation is always 1.
SumStats::observe("conn established",
SumStats::Key(),
SumStats::Observation($num=1));
}
event bro_init()
{
# Create the reducer.
# The reducer attaches to the "conn established" observation stream
# and uses the summing calculation on the observations.
local r1 = SumStats::Reducer($stream="conn established",
$apply=set(SumStats::SUM));
# Create the final sumstat.
# We give it an arbitrary name and make it collect data every minute.
# The reducer is then attached and a $epoch_result callback is given
# to finally do something with the data collected.
SumStats::create([$name = "counting connections",
$epoch = 1min,
$reducers = set(r1),
$epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
{
# This is the body of the callback that is called when a single
# result has been collected. We are just printing the total number
# of connections that were seen. The $sum field is provided as a
# double type value so we need to use %f as the format specifier.
print fmt("Number of connections established: %.0f", result["conn established"]$sum);
}]);
}

49
testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output Normal file
View file

@ -0,0 +1,49 @@
# @TEST-EXEC: cat %INPUT >output && btest-diff output
sumstats-toy-scan.bro
@load base/frameworks/sumstats
# We use the connection_attempt event to limit our observations to those
# which were attempted and not successful.
event connection_attempt(c: connection)
{
# Make an observation!
# This observation is about the host attempting the connection.
# Each established connection counts as one so the observation is always 1.
SumStats::observe("conn attempted",
SumStats::Key($host=c$id$orig_h),
SumStats::Observation($num=1));
}
event bro_init()
{
# Create the reducer.
# The reducer attaches to the "conn attempted" observation stream
# and uses the summing calculation on the observations. Keep
# in mind that there will be one result per key (connection originator).
local r1 = SumStats::Reducer($stream="conn attempted",
$apply=set(SumStats::SUM));
# Create the final sumstat.
# This is slightly different from the last example since we're providing
# a callback to calculate a value to check against the threshold with
# $threshold_val. The actual threshold itself is provided with $threshold.
# Another callback is provided for when a key crosses the threshold.
SumStats::create([$name = "finding scanners",
$epoch = 5min,
$reducers = set(r1),
# Provide a threshold.
$threshold = 5.0,
# Provide a callback to calculate a value from the result
# to check against the threshold field.
$threshold_val(key: SumStats::Key, result: SumStats::Result) =
{
return result["conn attempted"]$sum;
},
# Provide a callback for when a key crosses the threshold.
$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
{
print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum);
}]);
}

7
testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output
View file

@ -27,6 +27,11 @@ export {
/application\/jar/ |
/video\/mp4/ &redef;
## The Match notice has a sub message with a URL where you can get more
## information about the file. The %s will be replaced with the SHA-1
## hash of the file.
const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
## The malware hash registry runs each malware sample through several
## A/V engines. Team Cymru returns a percentage to indicate how
## many A/V engines flagged the sample as malicious. This threshold
@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
if ( mhr_detect_rate >= notice_threshold )
{
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
local virustotal_url = fmt(match_sub_url, hash);
NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
}
}

5
testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output
View file

@ -18,6 +18,11 @@ export {
/application\/jar/ |
/video\/mp4/ &redef;
## The Match notice has a sub message with a URL where you can get more
## information about the file. The %s will be replaced with the SHA-1
## hash of the file.
const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
## The malware hash registry runs each malware sample through several
## A/V engines. Team Cymru returns a percentage to indicate how
## many A/V engines flagged the sample as malicious. This threshold

2
testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output
View file

@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
if ( mhr_detect_rate >= notice_threshold )
{
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
local virustotal_url = fmt(match_sub_url, hash);
NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
}
}

9
testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 Normal file
View file

@ -0,0 +1,9 @@
.. rst-class:: btest-cmd
.. code-block:: none
:linenos:
:emphasize-lines: 1,1
# bro -r workshop_2011_browse.trace sumstats-countconns.bro
Number of connections established: 6

9
testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 Normal file
View file

@ -0,0 +1,9 @@
.. rst-class:: btest-cmd
.. code-block:: none
:linenos:
:emphasize-lines: 1,1
# bro -r nmap-vsn.trace sumstats-toy-scan.bro
192.168.1.71 attempted 5 or more connections

1
testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select Normal file
View file

@ -0,0 +1 @@
CC,AA,BB

BIN
testing/btest/Traces/nmap-vsn.trace Normal file
View file

Binary file not shown.

40
testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest Normal file
View file

@ -0,0 +1,40 @@
# @TEST-EXEC: cat %INPUT >output && btest-diff output
sumstats-countconns.bro
@load base/frameworks/sumstats
event connection_established(c: connection)
{
# Make an observation!
# This observation is global so the key is empty.
# Each established connection counts as one so the observation is always 1.
SumStats::observe("conn established",
SumStats::Key(),
SumStats::Observation($num=1));
}
event bro_init()
{
# Create the reducer.
# The reducer attaches to the "conn established" observation stream
# and uses the summing calculation on the observations.
local r1 = SumStats::Reducer($stream="conn established",
$apply=set(SumStats::SUM));
# Create the final sumstat.
# We give it an arbitrary name and make it collect data every minute.
# The reducer is then attached and a $epoch_result callback is given
# to finally do something with the data collected.
SumStats::create([$name = "counting connections",
$epoch = 1min,
$reducers = set(r1),
$epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
{
# This is the body of the callback that is called when a single
# result has been collected. We are just printing the total number
# of connections that were seen. The $sum field is provided as a
# double type value so we need to use %f as the format specifier.
print fmt("Number of connections established: %.0f", result["conn established"]$sum);
}]);
}

49
testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest Normal file
View file

@ -0,0 +1,49 @@
# @TEST-EXEC: cat %INPUT >output && btest-diff output
sumstats-toy-scan.bro
@load base/frameworks/sumstats
# We use the connection_attempt event to limit our observations to those
# which were attempted and not successful.
event connection_attempt(c: connection)
{
# Make an observation!
# This observation is about the host attempting the connection.
# Each established connection counts as one so the observation is always 1.
SumStats::observe("conn attempted",
SumStats::Key($host=c$id$orig_h),
SumStats::Observation($num=1));
}
event bro_init()
{
# Create the reducer.
# The reducer attaches to the "conn attempted" observation stream
# and uses the summing calculation on the observations. Keep
# in mind that there will be one result per key (connection originator).
local r1 = SumStats::Reducer($stream="conn attempted",
$apply=set(SumStats::SUM));
# Create the final sumstat.
# This is slightly different from the last example since we're providing
# a callback to calculate a value to check against the threshold with
# $threshold_val. The actual threshold itself is provided with $threshold.
# Another callback is provided for when a key crosses the threshold.
SumStats::create([$name = "finding scanners",
$epoch = 5min,
$reducers = set(r1),
# Provide a threshold.
$threshold = 5.0,
# Provide a callback to calculate a value from the result
# to check against the threshold field.
$threshold_val(key: SumStats::Key, result: SumStats::Result) =
{
return result["conn attempted"]$sum;
},
# Provide a callback for when a key crosses the threshold.
$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
{
print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum);
}]);
}

7
testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest
View file

@ -27,6 +27,11 @@ export {
/application\/jar/ |
/video\/mp4/ &redef;
## The Match notice has a sub message with a URL where you can get more
## information about the file. The %s will be replaced with the SHA-1
## hash of the file.
const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
## The malware hash registry runs each malware sample through several
## A/V engines. Team Cymru returns a percentage to indicate how
## many A/V engines flagged the sample as malicious. This threshold
@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
if ( mhr_detect_rate >= notice_threshold )
{
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
local virustotal_url = fmt(match_sub_url, hash);
NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
}
}

5
testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest
View file

@ -18,6 +18,11 @@ export {
/application\/jar/ |
/video\/mp4/ &redef;
## The Match notice has a sub message with a URL where you can get more
## information about the file. The %s will be replaced with the SHA-1
## hash of the file.
const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
## The malware hash registry runs each malware sample through several
## A/V engines. Team Cymru returns a percentage to indicate how
## many A/V engines flagged the sample as malicious. This threshold

2
testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest
View file

@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
if ( mhr_detect_rate >= notice_threshold )
{
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
local virustotal_url = fmt(match_sub_url, hash);
NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
}
}

1
testing/btest/doc/sphinx/sumstats-countconns.btest Normal file
View file

@ -0,0 +1 @@
@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro

1
testing/btest/doc/sphinx/sumstats-toy-scan.btest Normal file
View file

@ -0,0 +1 @@
@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro

50
testing/btest/scripts/base/frameworks/logging/sqlite/set.bro Normal file
View file

@ -0,0 +1,50 @@
#
# Check if set works in last position (the describe call in sqlite.cc has a good
# chance of being off by one if someone changes it).
#
# @TEST-REQUIRES: which sqlite3
# @TEST-REQUIRES: has-writer SQLite
# @TEST-GROUP: sqlite
#
# @TEST-EXEC: bro -b %INPUT
# @TEST-EXEC: sqlite3 ssh.sqlite 'select * from ssh' > ssh.select
# @TEST-EXEC: btest-diff ssh.select
#
# Testing all possible types.
redef LogSQLite::unset_field = "(unset)";
module SSH;
export {
redef enum Log::ID += { LOG };
type Log: record {
ss: set[string];
} &log;
}
function foo(i : count) : string
{
if ( i > 0 )
return "Foo";
else
return "Bar";
}
event bro_init()
{
Log::create_stream(SSH::LOG, [$columns=Log]);
Log::remove_filter(SSH::LOG, "default");
local filter: Log::Filter = [$name="sqlite", $path="ssh", $writer=Log::WRITER_SQLITE];
Log::add_filter(SSH::LOG, filter);
local empty_set: set[string];
local empty_vector: vector of string;
Log::write(SSH::LOG, [
$ss=set("AA", "BB", "CC")
]);
}

4
testing/scripts/diff-canonifier-external
View file

@ -8,6 +8,10 @@ if [ "$1" == "capture_loss.log" ]; then
addl="`dirname $0`/diff-remove-fractions"
fi
if [ "$1" == "ssh.log" ]; then
addl="`dirname $0`/diff-remove-fields remote_location"
fi
`dirname $0`/diff-remove-timestamps \
| `dirname $0`/diff-remove-uids \
| `dirname $0`/diff-remove-file-ids \

34
testing/scripts/diff-remove-fields Executable file
View file

@ -0,0 +1,34 @@
#! /usr/bin/env bash
#
# A diff canonifier that removes all fields starting with a given
# prefix.
if [ $# != 1 ]; then
echo "usage: `basename $0` <field prefix>"
exit 1
fi
awk -v "PREFIX=$1" '
BEGIN { FS="\t"; OFS="\t"; }
/^#fields/ {
for ( i = 2; i <= NF; ++i )
{
if ( index($i, PREFIX) == 1 )
rem[i-1] = 1;
}
print;
next;
}
{
for ( i in rem )
# Mark that it iss set, but ignore content.
$i = "+";
print;
}
'