mirror of
https://github.com/zeek/zeek.git
synced 2025-10-07 17:18:20 +00:00
Merge branch 'master' into topic/jsiwek/faf-updates
Conflicts: testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
This commit is contained in:
Jon Siwek
commit
9bd7a65071
91 changed files with 14058 additions and 402 deletions
|
|
@ -10,13 +10,14 @@ module Intel;
|
|||
export {
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
## String data needs to be further categoried since it could represent
|
||||
## and number of types of data.
|
||||
type StrType: enum {
|
||||
## Enum type to represent various types of intelligence data.
|
||||
type Type: enum {
|
||||
## An IP address.
|
||||
ADDR,
|
||||
## A complete URL without the prefix "http://".
|
||||
URL,
|
||||
## User-Agent string, typically HTTP or mail message body.
|
||||
USER_AGENT,
|
||||
## Software name.
|
||||
SOFTWARE,
|
||||
## Email address.
|
||||
EMAIL,
|
||||
## DNS domain name.
|
||||
|
|
@ -44,18 +45,15 @@ export {
|
|||
|
||||
## Represents a piece of intelligence.
|
||||
type Item: record {
|
||||
## The IP address if the intelligence is about an IP address.
|
||||
host: addr &optional;
|
||||
## The network if the intelligence is about a CIDR block.
|
||||
net: subnet &optional;
|
||||
## The string if the intelligence is about a string.
|
||||
str: string &optional;
|
||||
## The type of data that is in the string if the $str field is set.
|
||||
str_type: StrType &optional;
|
||||
## The intelligence indicator.
|
||||
indicator: string;
|
||||
|
||||
## The type of data that the indicator field represents.
|
||||
indicator_type: Type;
|
||||
|
||||
## Metadata for the item. Typically represents more deeply \
|
||||
## Metadata for the item. Typically represents more deeply
|
||||
## descriptive data for a piece of intelligence.
|
||||
meta: MetaData;
|
||||
meta: MetaData;
|
||||
};
|
||||
|
||||
## Enum to represent where data came from when it was discovered.
|
||||
|
|
@ -65,23 +63,23 @@ export {
|
|||
IN_ANYWHERE,
|
||||
};
|
||||
|
||||
## The $host field and combination of $str and $str_type fields are mutually
|
||||
## exclusive. These records *must* represent either an IP address being
|
||||
## seen or a string being seen.
|
||||
type Seen: record {
|
||||
## The IP address if the data seen is an IP address.
|
||||
host: addr &log &optional;
|
||||
## The string if the data is about a string.
|
||||
str: string &log &optional;
|
||||
## The type of data that is in the string if the $str field is set.
|
||||
str_type: StrType &log &optional;
|
||||
indicator: string &log &optional;
|
||||
|
||||
## The type of data that the indicator represents.
|
||||
indicator_type: Type &log &optional;
|
||||
|
||||
## If the indicator type was :bro:enum:`Intel::ADDR`, then this
|
||||
## field will be present.
|
||||
host: addr &optional;
|
||||
|
||||
## Where the data was discovered.
|
||||
where: Where &log;
|
||||
where: Where &log;
|
||||
|
||||
## If the data was discovered within a connection, the
|
||||
## connection record should go into get to give context to the data.
|
||||
conn: connection &optional;
|
||||
conn: connection &optional;
|
||||
};
|
||||
|
||||
## Record used for the logging framework representing a positive
|
||||
|
|
@ -100,7 +98,7 @@ export {
|
|||
## Where the data was seen.
|
||||
seen: Seen &log;
|
||||
## Sources which supplied data that resulted in this match.
|
||||
sources: set[string] &log;
|
||||
sources: set[string] &log &default=string_set();
|
||||
};
|
||||
|
||||
## Intelligence data manipulation functions.
|
||||
|
|
@ -135,8 +133,8 @@ const have_full_data = T &redef;
|
|||
|
||||
# The in memory data structure for holding intelligence.
|
||||
type DataStore: record {
|
||||
net_data: table[subnet] of set[MetaData];
|
||||
string_data: table[string, StrType] of set[MetaData];
|
||||
host_data: table[addr] of set[MetaData];
|
||||
string_data: table[string, Type] of set[MetaData];
|
||||
};
|
||||
global data_store: DataStore &redef;
|
||||
|
||||
|
|
@ -144,8 +142,8 @@ global data_store: DataStore &redef;
|
|||
# This is primarily for workers to do the initial quick matches and store
|
||||
# a minimal amount of data for the full match to happen on the manager.
|
||||
type MinDataStore: record {
|
||||
net_data: set[subnet];
|
||||
string_data: set[string, StrType];
|
||||
host_data: set[addr];
|
||||
string_data: set[string, Type];
|
||||
};
|
||||
global min_data_store: MinDataStore &redef;
|
||||
|
||||
|
|
@ -157,15 +155,13 @@ event bro_init() &priority=5
|
|||
|
||||
function find(s: Seen): bool
|
||||
{
|
||||
if ( s?$host &&
|
||||
((have_full_data && s$host in data_store$net_data) ||
|
||||
(s$host in min_data_store$net_data)))
|
||||
if ( s?$host )
|
||||
{
|
||||
return T;
|
||||
return ((s$host in min_data_store$host_data) ||
|
||||
(have_full_data && s$host in data_store$host_data));
|
||||
}
|
||||
else if ( s?$str && s?$str_type &&
|
||||
((have_full_data && [s$str, s$str_type] in data_store$string_data) ||
|
||||
([s$str, s$str_type] in min_data_store$string_data)))
|
||||
else if ( ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
|
||||
(have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data) )
|
||||
{
|
||||
return T;
|
||||
}
|
||||
|
|
@ -177,8 +173,7 @@ function find(s: Seen): bool
|
|||
|
||||
function get_items(s: Seen): set[Item]
|
||||
{
|
||||
local item: Item;
|
||||
local return_data: set[Item] = set();
|
||||
local return_data: set[Item];
|
||||
|
||||
if ( ! have_full_data )
|
||||
{
|
||||
|
|
@ -191,26 +186,23 @@ function get_items(s: Seen): set[Item]
|
|||
if ( s?$host )
|
||||
{
|
||||
# See if the host is known about and it has meta values
|
||||
if ( s$host in data_store$net_data )
|
||||
if ( s$host in data_store$host_data )
|
||||
{
|
||||
for ( m in data_store$net_data[s$host] )
|
||||
for ( m in data_store$host_data[s$host] )
|
||||
{
|
||||
# TODO: the lookup should be finding all and not just most specific
|
||||
# and $host/$net should have the correct value.
|
||||
item = [$host=s$host, $meta=m];
|
||||
add return_data[item];
|
||||
add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=m)];
|
||||
}
|
||||
}
|
||||
}
|
||||
else if ( s?$str && s?$str_type )
|
||||
else
|
||||
{
|
||||
local lower_indicator = to_lower(s$indicator);
|
||||
# See if the string is known about and it has meta values
|
||||
if ( [s$str, s$str_type] in data_store$string_data )
|
||||
if ( [lower_indicator, s$indicator_type] in data_store$string_data )
|
||||
{
|
||||
for ( m in data_store$string_data[s$str, s$str_type] )
|
||||
for ( m in data_store$string_data[lower_indicator, s$indicator_type] )
|
||||
{
|
||||
item = [$str=s$str, $str_type=s$str_type, $meta=m];
|
||||
add return_data[item];
|
||||
add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=m)];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -222,6 +214,12 @@ function Intel::seen(s: Seen)
|
|||
{
|
||||
if ( find(s) )
|
||||
{
|
||||
if ( s?$host )
|
||||
{
|
||||
s$indicator = cat(s$host);
|
||||
s$indicator_type = Intel::ADDR;
|
||||
}
|
||||
|
||||
if ( have_full_data )
|
||||
{
|
||||
local items = get_items(s);
|
||||
|
|
@ -250,8 +248,7 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
|
|||
|
||||
event Intel::match(s: Seen, items: set[Item]) &priority=5
|
||||
{
|
||||
local empty_set: set[string] = set();
|
||||
local info: Info = [$ts=network_time(), $seen=s, $sources=empty_set];
|
||||
local info: Info = [$ts=network_time(), $seen=s];
|
||||
|
||||
if ( s?$conn )
|
||||
{
|
||||
|
|
@ -267,52 +264,37 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
|
|||
|
||||
function insert(item: Item)
|
||||
{
|
||||
if ( item?$str && !item?$str_type )
|
||||
{
|
||||
event reporter_warning(network_time(), fmt("You must provide a str_type for strings or this item doesn't make sense. Item: %s", item), "");
|
||||
return;
|
||||
}
|
||||
|
||||
# Create and fill out the meta data item.
|
||||
local meta = item$meta;
|
||||
local metas: set[MetaData];
|
||||
|
||||
if ( item?$host )
|
||||
# All intelligence is case insensitive at the moment.
|
||||
local lower_indicator = to_lower(item$indicator);
|
||||
|
||||
if ( item$indicator_type == ADDR )
|
||||
{
|
||||
local host = mask_addr(item$host, is_v4_addr(item$host) ? 32 : 128);
|
||||
local host = to_addr(item$indicator);
|
||||
if ( have_full_data )
|
||||
{
|
||||
if ( host !in data_store$net_data )
|
||||
data_store$net_data[host] = set();
|
||||
if ( host !in data_store$host_data )
|
||||
data_store$host_data[host] = set();
|
||||
|
||||
metas = data_store$net_data[host];
|
||||
metas = data_store$host_data[host];
|
||||
}
|
||||
|
||||
add min_data_store$net_data[host];
|
||||
add min_data_store$host_data[host];
|
||||
}
|
||||
else if ( item?$net )
|
||||
else
|
||||
{
|
||||
if ( have_full_data )
|
||||
{
|
||||
if ( item$net !in data_store$net_data )
|
||||
data_store$net_data[item$net] = set();
|
||||
if ( [lower_indicator, item$indicator_type] !in data_store$string_data )
|
||||
data_store$string_data[lower_indicator, item$indicator_type] = set();
|
||||
|
||||
metas = data_store$net_data[item$net];
|
||||
metas = data_store$string_data[lower_indicator, item$indicator_type];
|
||||
}
|
||||
|
||||
add min_data_store$net_data[item$net];
|
||||
}
|
||||
else if ( item?$str )
|
||||
{
|
||||
if ( have_full_data )
|
||||
{
|
||||
if ( [item$str, item$str_type] !in data_store$string_data )
|
||||
data_store$string_data[item$str, item$str_type] = set();
|
||||
|
||||
metas = data_store$string_data[item$str, item$str_type];
|
||||
}
|
||||
|
||||
add min_data_store$string_data[item$str, item$str_type];
|
||||
add min_data_store$string_data[lower_indicator, item$indicator_type];
|
||||
}
|
||||
|
||||
local updated = F;
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ export {
|
|||
|
||||
## Frequently files can be "described" to give a bit more context.
|
||||
## This field will typically be automatically filled out from an
|
||||
## fa_file record. For example, if a notice was related to a
|
||||
## fa_file record. For example, if a notice was related to a
|
||||
## file over HTTP, the URL of the request would be shown.
|
||||
file_desc: string &log &optional;
|
||||
|
||||
|
|
@ -483,12 +483,12 @@ function apply_policy(n: Notice::Info)
|
|||
{
|
||||
if ( ! n?$fuid )
|
||||
n$fuid = n$f$id;
|
||||
|
||||
|
||||
if ( ! n?$file_mime_type && n$f?$mime_type )
|
||||
n$file_mime_type = n$f$mime_type;
|
||||
|
||||
n$file_desc = Files::describe(n$f);
|
||||
|
||||
|
||||
if ( n$f?$conns && |n$f$conns| == 1 )
|
||||
{
|
||||
for ( id in n$f$conns )
|
||||
|
|
@ -550,8 +550,8 @@ function apply_policy(n: Notice::Info)
|
|||
if ( ! n?$suppress_for )
|
||||
n$suppress_for = default_suppression_interval;
|
||||
|
||||
# Delete the connection and file records if they're there so we
|
||||
# aren't sending that to remote machines. It can cause problems
|
||||
# Delete the connection and file records if they're there so we
|
||||
# aren't sending that to remote machines. It can cause problems
|
||||
# due to the size of those records.
|
||||
if ( n?$conn )
|
||||
delete n$conn;
|
||||
|
|
|
|||
|
|
@ -702,6 +702,7 @@ type entropy_test_result: record {
|
|||
@load base/bif/strings.bif
|
||||
@load base/bif/bro.bif
|
||||
@load base/bif/reporter.bif
|
||||
@load base/bif/bloom-filter.bif
|
||||
|
||||
## Deprecated. This is superseded by the new logging framework.
|
||||
global log_file_name: function(tag: string): string &redef;
|
||||
|
|
@ -3047,3 +3048,5 @@ const snaplen = 8192 &redef;
|
|||
@load base/frameworks/input
|
||||
@load base/frameworks/analyzer
|
||||
@load base/frameworks/files
|
||||
|
||||
@load base/bif
|
||||
|
|
|
|||
|
|
@ -5,9 +5,12 @@
|
|||
##! you actually want.
|
||||
|
||||
@load base/utils/site
|
||||
@load base/utils/active-http
|
||||
@load base/utils/addrs
|
||||
@load base/utils/conn-ids
|
||||
@load base/utils/dir
|
||||
@load base/utils/directions-and-hosts
|
||||
@load base/utils/exec
|
||||
@load base/utils/files
|
||||
@load base/utils/numbers
|
||||
@load base/utils/paths
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ function ftp_message(s: Info)
|
|||
s$arg = s$cmdarg$arg;
|
||||
if ( s$cmdarg$cmd in file_cmds )
|
||||
s$arg = build_url_ftp(s);
|
||||
|
||||
|
||||
if ( s$arg == "" )
|
||||
delete s$arg;
|
||||
|
||||
|
|
@ -142,7 +142,7 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel)
|
|||
s$passive = chan$passive;
|
||||
s$data_channel = chan;
|
||||
ftp_data_expected[chan$resp_h, chan$resp_p] = s;
|
||||
Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p,
|
||||
Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p,
|
||||
Analyzer::ANALYZER_FTP_DATA,
|
||||
5mins);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ export {
|
|||
##
|
||||
## Returns: A URL, not prefixed by "ftp://".
|
||||
global build_url: function(rec: Info): string;
|
||||
|
||||
|
||||
## Creates a URL from an :bro:type:`FTP::Info` record.
|
||||
##
|
||||
## rec: An :bro:type:`FTP::Info` record.
|
||||
|
|
@ -36,7 +36,7 @@ function build_url(rec: Info): string
|
|||
|
||||
return fmt("%s%s", addr_to_uri(rec$id$resp_h), comp_path);
|
||||
}
|
||||
|
||||
|
||||
function build_url_ftp(rec: Info): string
|
||||
{
|
||||
return fmt("ftp://%s", build_url(rec));
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ export {
|
|||
|
||||
function get_file_handle(c: connection, is_orig: bool): string
|
||||
{
|
||||
if ( ! c?$http )
|
||||
if ( ! c?$http )
|
||||
return "";
|
||||
|
||||
if ( c$http$range_request && ! is_orig )
|
||||
|
|
@ -29,7 +29,7 @@ function get_file_handle(c: connection, is_orig: bool): string
|
|||
else
|
||||
{
|
||||
local mime_depth = is_orig ? c$http$orig_mime_depth : c$http$resp_mime_depth;
|
||||
return cat(Analyzer::ANALYZER_HTTP, c$start_time, is_orig,
|
||||
return cat(Analyzer::ANALYZER_HTTP, c$start_time, is_orig,
|
||||
c$http$trans_depth, mime_depth, id_string(c$id));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
##! Implements base functionality for HTTP analysis. The logging model is
|
||||
##! to log request/response pairs and all relevant metadata together in
|
||||
##! Implements base functionality for HTTP analysis. The logging model is
|
||||
##! to log request/response pairs and all relevant metadata together in
|
||||
##! a single record.
|
||||
|
||||
@load base/utils/numbers
|
||||
|
|
@ -15,10 +15,10 @@ export {
|
|||
## Placeholder.
|
||||
EMPTY
|
||||
};
|
||||
|
||||
|
||||
## This setting changes if passwords used in Basic-Auth are captured or not.
|
||||
const default_capture_password = F &redef;
|
||||
|
||||
|
||||
type Info: record {
|
||||
## Timestamp for when the request happened.
|
||||
ts: time &log;
|
||||
|
|
@ -26,7 +26,7 @@ export {
|
|||
uid: string &log;
|
||||
## The connection's 4-tuple of endpoint addresses/ports.
|
||||
id: conn_id &log;
|
||||
## Represents the pipelined depth into the connection of this
|
||||
## Represents the pipelined depth into the connection of this
|
||||
## request/response transaction.
|
||||
trans_depth: count &log;
|
||||
## Verb used in the HTTP request (GET, POST, HEAD, etc.).
|
||||
|
|
@ -60,24 +60,24 @@ export {
|
|||
## A set of indicators of various attributes discovered and
|
||||
## related to a particular request/response pair.
|
||||
tags: set[Tags] &log;
|
||||
|
||||
|
||||
## Username if basic-auth is performed for the request.
|
||||
username: string &log &optional;
|
||||
## Password if basic-auth is performed for the request.
|
||||
password: string &log &optional;
|
||||
|
||||
|
||||
## Determines if the password will be captured for this request.
|
||||
capture_password: bool &default=default_capture_password;
|
||||
|
||||
|
||||
## All of the headers that may indicate if the request was proxied.
|
||||
proxied: set[string] &log &optional;
|
||||
|
||||
|
||||
## Indicates if this request can assume 206 partial content in
|
||||
## response.
|
||||
range_request: bool &default=F;
|
||||
};
|
||||
|
||||
## Structure to maintain state for an HTTP connection with multiple
|
||||
|
||||
## Structure to maintain state for an HTTP connection with multiple
|
||||
## requests and responses.
|
||||
type State: record {
|
||||
## Pending requests.
|
||||
|
|
@ -87,7 +87,7 @@ export {
|
|||
## Current response in the pending queue.
|
||||
current_response: count &default=0;
|
||||
};
|
||||
|
||||
|
||||
## A list of HTTP headers typically used to indicate proxied requests.
|
||||
const proxy_headers: set[string] = {
|
||||
"FORWARDED",
|
||||
|
|
@ -111,8 +111,8 @@ export {
|
|||
"POLL", "REPORT", "SUBSCRIBE", "BMOVE",
|
||||
"SEARCH"
|
||||
} &redef;
|
||||
|
||||
## Event that can be handled to access the HTTP record as it is sent on
|
||||
|
||||
## Event that can be handled to access the HTTP record as it is sent on
|
||||
## to the logging framework.
|
||||
global log_http: event(rec: Info);
|
||||
}
|
||||
|
|
@ -147,12 +147,12 @@ function new_http_session(c: connection): Info
|
|||
tmp$ts=network_time();
|
||||
tmp$uid=c$uid;
|
||||
tmp$id=c$id;
|
||||
# $current_request is set prior to the Info record creation so we
|
||||
# $current_request is set prior to the Info record creation so we
|
||||
# can use the value directly here.
|
||||
tmp$trans_depth = c$http_state$current_request;
|
||||
return tmp;
|
||||
}
|
||||
|
||||
|
||||
function set_state(c: connection, request: bool, is_orig: bool)
|
||||
{
|
||||
if ( ! c?$http_state )
|
||||
|
|
@ -160,19 +160,19 @@ function set_state(c: connection, request: bool, is_orig: bool)
|
|||
local s: State;
|
||||
c$http_state = s;
|
||||
}
|
||||
|
||||
|
||||
# These deal with new requests and responses.
|
||||
if ( request || c$http_state$current_request !in c$http_state$pending )
|
||||
c$http_state$pending[c$http_state$current_request] = new_http_session(c);
|
||||
if ( ! is_orig && c$http_state$current_response !in c$http_state$pending )
|
||||
c$http_state$pending[c$http_state$current_response] = new_http_session(c);
|
||||
|
||||
|
||||
if ( is_orig )
|
||||
c$http = c$http_state$pending[c$http_state$current_request];
|
||||
else
|
||||
c$http = c$http_state$pending[c$http_state$current_response];
|
||||
}
|
||||
|
||||
|
||||
event http_request(c: connection, method: string, original_URI: string,
|
||||
unescaped_URI: string, version: string) &priority=5
|
||||
{
|
||||
|
|
@ -181,17 +181,17 @@ event http_request(c: connection, method: string, original_URI: string,
|
|||
local s: State;
|
||||
c$http_state = s;
|
||||
}
|
||||
|
||||
|
||||
++c$http_state$current_request;
|
||||
set_state(c, T, T);
|
||||
|
||||
|
||||
c$http$method = method;
|
||||
c$http$uri = unescaped_URI;
|
||||
|
||||
if ( method !in http_methods )
|
||||
event conn_weird("unknown_HTTP_method", c, method);
|
||||
}
|
||||
|
||||
|
||||
event http_reply(c: connection, version: string, code: count, reason: string) &priority=5
|
||||
{
|
||||
if ( ! c?$http_state )
|
||||
|
|
@ -199,7 +199,7 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
|
|||
local s: State;
|
||||
c$http_state = s;
|
||||
}
|
||||
|
||||
|
||||
# If the last response was an informational 1xx, we're still expecting
|
||||
# the real response to the request, so don't create a new Info record yet.
|
||||
if ( c$http_state$current_response !in c$http_state$pending ||
|
||||
|
|
@ -207,7 +207,7 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
|
|||
! code_in_range(c$http_state$pending[c$http_state$current_response]$status_code, 100, 199)) )
|
||||
++c$http_state$current_response;
|
||||
set_state(c, F, F);
|
||||
|
||||
|
||||
c$http$status_code = code;
|
||||
c$http$status_msg = reason;
|
||||
if ( code_in_range(code, 100, 199) )
|
||||
|
|
@ -216,33 +216,33 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
|
|||
c$http$info_msg = reason;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5
|
||||
{
|
||||
set_state(c, F, is_orig);
|
||||
|
||||
|
||||
if ( is_orig ) # client headers
|
||||
{
|
||||
if ( name == "REFERER" )
|
||||
c$http$referrer = value;
|
||||
|
||||
|
||||
else if ( name == "HOST" )
|
||||
# The split is done to remove the occasional port value that shows up here.
|
||||
c$http$host = split1(value, /:/)[1];
|
||||
|
||||
else if ( name == "RANGE" )
|
||||
c$http$range_request = T;
|
||||
|
||||
|
||||
else if ( name == "USER-AGENT" )
|
||||
c$http$user_agent = value;
|
||||
|
||||
|
||||
else if ( name in proxy_headers )
|
||||
{
|
||||
if ( ! c$http?$proxied )
|
||||
c$http$proxied = set();
|
||||
add c$http$proxied[fmt("%s -> %s", name, value)];
|
||||
}
|
||||
|
||||
|
||||
else if ( name == "AUTHORIZATION" )
|
||||
{
|
||||
if ( /^[bB][aA][sS][iI][cC] / in value )
|
||||
|
|
@ -266,17 +266,17 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5
|
||||
{
|
||||
set_state(c, F, is_orig);
|
||||
|
||||
|
||||
if ( is_orig )
|
||||
c$http$request_body_len = stat$body_length;
|
||||
else
|
||||
c$http$response_body_len = stat$body_length;
|
||||
}
|
||||
|
||||
|
||||
event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = -5
|
||||
{
|
||||
# The reply body is done so we're ready to log.
|
||||
|
|
@ -305,4 +305,4 @@ event connection_state_remove(c: connection) &priority=-5
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
##!
|
||||
##! There is a major problem with this script in the cluster context because
|
||||
##! we might see A send B a message that a DCC connection is to be expected,
|
||||
##! but that connection will actually be between B and C which could be
|
||||
##! but that connection will actually be between B and C which could be
|
||||
##! analyzed on a different worker.
|
||||
##!
|
||||
|
||||
|
|
@ -44,7 +44,7 @@ function log_dcc(f: fa_file)
|
|||
Log::write(IRC::LOG, irc);
|
||||
irc$command = tmp;
|
||||
|
||||
# Delete these values in case another DCC transfer
|
||||
# Delete these values in case another DCC transfer
|
||||
# happens during the IRC session.
|
||||
delete irc$dcc_file_name;
|
||||
delete irc$dcc_file_size;
|
||||
|
|
@ -57,7 +57,7 @@ function log_dcc(f: fa_file)
|
|||
|
||||
event file_new(f: fa_file) &priority=-5
|
||||
{
|
||||
if ( f$source == "IRC_DATA" )
|
||||
if ( f$source == "IRC_DATA" )
|
||||
log_dcc(f);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ export {
|
|||
};
|
||||
|
||||
redef record State += {
|
||||
## Track the number of MIME encoded files transferred
|
||||
## Track the number of MIME encoded files transferred
|
||||
## during a session.
|
||||
mime_depth: count &default=0;
|
||||
};
|
||||
|
|
@ -33,7 +33,7 @@ event mime_begin_entity(c: connection) &priority=10
|
|||
|
||||
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
|
||||
{
|
||||
if ( f$source == "SMTP" && c?$smtp )
|
||||
if ( f$source == "SMTP" && c?$smtp )
|
||||
{
|
||||
if ( c$smtp?$entity && c$smtp$entity?$filename )
|
||||
f$info$filename = c$smtp$entity$filename;
|
||||
|
|
@ -57,6 +57,6 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5
|
|||
|
||||
event mime_end_entity(c: connection) &priority=5
|
||||
{
|
||||
if ( c?$smtp && c$smtp?$entity )
|
||||
if ( c?$smtp && c$smtp?$entity )
|
||||
delete c$smtp$entity;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -226,7 +226,10 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5
|
|||
{
|
||||
if ( ! c$smtp?$to )
|
||||
c$smtp$to = set();
|
||||
add c$smtp$to[h$value];
|
||||
|
||||
local to_parts = split(h$value, /[[:blank:]]*,[[:blank:]]*/);
|
||||
for ( i in to_parts )
|
||||
add c$smtp$to[to_parts[i]];
|
||||
}
|
||||
|
||||
else if ( h$name == "X-ORIGINATING-IP" )
|
||||
|
|
@ -296,4 +299,4 @@ function describe(rec: Info): string
|
|||
(abbrev_subject != "" ? fmt(": %s", abbrev_subject) : ""));
|
||||
}
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
123
scripts/base/utils/active-http.bro
Normal file
123
scripts/base/utils/active-http.bro
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
##! A module for performing active HTTP requests and
|
||||
##! getting the reply at runtime.
|
||||
|
||||
@load ./exec
|
||||
|
||||
module ActiveHTTP;
|
||||
|
||||
export {
|
||||
## The default timeout for HTTP requests.
|
||||
const default_max_time = 1min &redef;
|
||||
|
||||
## The default HTTP method/verb to use for requests.
|
||||
const default_method = "GET" &redef;
|
||||
|
||||
type Response: record {
|
||||
## Numeric response code from the server.
|
||||
code: count;
|
||||
## String response message from the server.
|
||||
msg: string;
|
||||
## Full body of the response.
|
||||
body: string &optional;
|
||||
## All headers returned by the server.
|
||||
headers: table[string] of string &optional;
|
||||
};
|
||||
|
||||
type Request: record {
|
||||
## The URL being requested.
|
||||
url: string;
|
||||
## The HTTP method/verb to use for the request.
|
||||
method: string &default=default_method;
|
||||
## Data to send to the server in the client body. Keep in
|
||||
## mind that you will probably need to set the *method* field
|
||||
## to "POST" or "PUT".
|
||||
client_data: string &optional;
|
||||
## Arbitrary headers to pass to the server. Some headers
|
||||
## will be included by libCurl.
|
||||
#custom_headers: table[string] of string &optional;
|
||||
## Timeout for the request.
|
||||
max_time: interval &default=default_max_time;
|
||||
## Additional curl command line arguments. Be very careful
|
||||
## with this option since shell injection could take place
|
||||
## if careful handling of untrusted data is not applied.
|
||||
addl_curl_args: string &optional;
|
||||
};
|
||||
|
||||
## Perform an HTTP request according to the :bro:type:`Request` record.
|
||||
## This is an asynchronous function and must be called within a "when"
|
||||
## statement.
|
||||
##
|
||||
## req: A record instance representing all options for an HTTP request.
|
||||
##
|
||||
## Returns: A record with the full response message.
|
||||
global request: function(req: ActiveHTTP::Request): ActiveHTTP::Response;
|
||||
}
|
||||
|
||||
function request2curl(r: Request, bodyfile: string, headersfile: string): string
|
||||
{
|
||||
local cmd = fmt("curl -s -g -o \"%s\" -D \"%s\" -X \"%s\"",
|
||||
str_shell_escape(bodyfile),
|
||||
str_shell_escape(headersfile),
|
||||
str_shell_escape(r$method));
|
||||
|
||||
cmd = fmt("%s -m %.0f", cmd, r$max_time);
|
||||
|
||||
if ( r?$client_data )
|
||||
cmd = fmt("%s -d -", cmd);
|
||||
|
||||
if ( r?$addl_curl_args )
|
||||
cmd = fmt("%s %s", cmd, r$addl_curl_args);
|
||||
|
||||
cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url));
|
||||
return cmd;
|
||||
}
|
||||
|
||||
function request(req: Request): ActiveHTTP::Response
|
||||
{
|
||||
local tmpfile = "/tmp/bro-activehttp-" + unique_id("");
|
||||
local bodyfile = fmt("%s_body", tmpfile);
|
||||
local headersfile = fmt("%s_headers", tmpfile);
|
||||
|
||||
local cmd = request2curl(req, bodyfile, headersfile);
|
||||
local stdin_data = req?$client_data ? req$client_data : "";
|
||||
|
||||
local resp: Response;
|
||||
resp$code = 0;
|
||||
resp$msg = "";
|
||||
resp$body = "";
|
||||
resp$headers = table();
|
||||
return when ( local result = Exec::run([$cmd=cmd, $stdin=stdin_data, $read_files=set(bodyfile, headersfile)]) )
|
||||
{
|
||||
# If there is no response line then nothing else will work either.
|
||||
if ( ! (result?$files && headersfile in result$files) )
|
||||
{
|
||||
Reporter::error(fmt("There was a failure when requesting \"%s\" with ActiveHTTP.", req$url));
|
||||
return resp;
|
||||
}
|
||||
|
||||
local headers = result$files[headersfile];
|
||||
for ( i in headers )
|
||||
{
|
||||
# The reply is the first line.
|
||||
if ( i == 0 )
|
||||
{
|
||||
local response_line = split_n(headers[0], /[[:blank:]]+/, F, 2);
|
||||
if ( |response_line| != 3 )
|
||||
return resp;
|
||||
|
||||
resp$code = to_count(response_line[2]);
|
||||
resp$msg = response_line[3];
|
||||
resp$body = join_string_vec(result$files[bodyfile], "");
|
||||
}
|
||||
else
|
||||
{
|
||||
local line = headers[i];
|
||||
local h = split1(line, /:/);
|
||||
if ( |h| != 2 )
|
||||
next;
|
||||
resp$headers[h[1]] = sub_bytes(h[2], 0, |h[2]|-1);
|
||||
}
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
}
|
||||
66
scripts/base/utils/dir.bro
Normal file
66
scripts/base/utils/dir.bro
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
@load base/utils/exec
|
||||
@load base/frameworks/reporter
|
||||
@load base/utils/paths
|
||||
|
||||
module Dir;
|
||||
|
||||
export {
|
||||
## The default interval this module checks for files in directories when
|
||||
## using the :bro:see:`Dir::monitor` function.
|
||||
const polling_interval = 30sec &redef;
|
||||
|
||||
## Register a directory to monitor with a callback that is called
|
||||
## every time a previously unseen file is seen. If a file is deleted
|
||||
## and seen to be gone, the file is available for being seen again in
|
||||
## the future.
|
||||
##
|
||||
## dir: The directory to monitor for files.
|
||||
##
|
||||
## callback: Callback that gets executed with each file name
|
||||
## that is found. Filenames are provided with the full path.
|
||||
##
|
||||
## poll_interval: An interval at which to check for new files.
|
||||
global monitor: function(dir: string, callback: function(fname: string),
|
||||
poll_interval: interval &default=polling_interval);
|
||||
}
|
||||
|
||||
event Dir::monitor_ev(dir: string, last_files: set[string],
|
||||
callback: function(fname: string),
|
||||
poll_interval: interval)
|
||||
{
|
||||
when ( local result = Exec::run([$cmd=fmt("ls -i \"%s/\"", str_shell_escape(dir))]) )
|
||||
{
|
||||
if ( result$exit_code != 0 )
|
||||
{
|
||||
Reporter::warning(fmt("Requested monitoring of non-existent directory (%s).", dir));
|
||||
return;
|
||||
}
|
||||
|
||||
local current_files: set[string] = set();
|
||||
local files: vector of string = vector();
|
||||
|
||||
if ( result?$stdout )
|
||||
files = result$stdout;
|
||||
|
||||
for ( i in files )
|
||||
{
|
||||
local parts = split1(files[i], / /);
|
||||
if ( parts[1] !in last_files )
|
||||
callback(build_path_compressed(dir, parts[2]));
|
||||
add current_files[parts[1]];
|
||||
}
|
||||
|
||||
schedule poll_interval
|
||||
{
|
||||
Dir::monitor_ev(dir, current_files, callback, poll_interval)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function monitor(dir: string, callback: function(fname: string),
|
||||
poll_interval: interval &default=polling_interval)
|
||||
{
|
||||
event Dir::monitor_ev(dir, set(), callback, poll_interval);
|
||||
}
|
||||
|
||||
|
||||
185
scripts/base/utils/exec.bro
Normal file
185
scripts/base/utils/exec.bro
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
##! A module for executing external command line programs.
|
||||
|
||||
@load base/frameworks/input
|
||||
|
||||
module Exec;
|
||||
|
||||
export {
|
||||
type Command: record {
|
||||
## The command line to execute. Use care to avoid injection attacks.
|
||||
## I.e. if the command uses untrusted/variable data, sanitize
|
||||
## it with str_shell_escape().
|
||||
cmd: string;
|
||||
## Provide standard in to the program as a string.
|
||||
stdin: string &default="";
|
||||
## If additional files are required to be read in as part of the output
|
||||
## of the command they can be defined here.
|
||||
read_files: set[string] &optional;
|
||||
# The unique id for tracking executors.
|
||||
uid: string &default=unique_id("");
|
||||
};
|
||||
|
||||
type Result: record {
|
||||
## Exit code from the program.
|
||||
exit_code: count &default=0;
|
||||
## True if the command was terminated with a signal.
|
||||
signal_exit: bool &default=F;
|
||||
## Each line of standard out.
|
||||
stdout: vector of string &optional;
|
||||
## Each line of standard error.
|
||||
stderr: vector of string &optional;
|
||||
## If additional files were requested to be read in
|
||||
## the content of the files will be available here.
|
||||
files: table[string] of string_vec &optional;
|
||||
};
|
||||
|
||||
## Function for running command line programs and getting
|
||||
## output. This is an asynchronous function which is meant
|
||||
## to be run with the `when` statement.
|
||||
##
|
||||
## cmd: The command to run. Use care to avoid injection attacks!
|
||||
##
|
||||
## returns: A record representing the full results from the
|
||||
## external program execution.
|
||||
global run: function(cmd: Command): Result;
|
||||
|
||||
## The system directory for temp files.
|
||||
const tmp_dir = "/tmp" &redef;
|
||||
}
|
||||
|
||||
# Indexed by command uid.
|
||||
global results: table[string] of Result;
|
||||
global pending_commands: set[string];
|
||||
global pending_files: table[string] of set[string];
|
||||
|
||||
type OneLine: record {
|
||||
s: string;
|
||||
is_stderr: bool;
|
||||
};
|
||||
|
||||
type FileLine: record {
|
||||
s: string;
|
||||
};
|
||||
|
||||
event Exec::line(description: Input::EventDescription, tpe: Input::Event, s: string, is_stderr: bool)
|
||||
{
|
||||
local result = results[description$name];
|
||||
if ( is_stderr )
|
||||
{
|
||||
if ( ! result?$stderr )
|
||||
result$stderr = vector(s);
|
||||
else
|
||||
result$stderr[|result$stderr|] = s;
|
||||
}
|
||||
else
|
||||
{
|
||||
if ( ! result?$stdout )
|
||||
result$stdout = vector(s);
|
||||
else
|
||||
result$stdout[|result$stdout|] = s;
|
||||
}
|
||||
}
|
||||
|
||||
event Exec::file_line(description: Input::EventDescription, tpe: Input::Event, s: string)
|
||||
{
|
||||
local parts = split1(description$name, /_/);
|
||||
local name = parts[1];
|
||||
local track_file = parts[2];
|
||||
|
||||
local result = results[name];
|
||||
if ( ! result?$files )
|
||||
result$files = table();
|
||||
|
||||
if ( track_file !in result$files )
|
||||
result$files[track_file] = vector(s);
|
||||
else
|
||||
result$files[track_file][|result$files[track_file]|] = s;
|
||||
}
|
||||
|
||||
event Input::end_of_data(name: string, source:string)
|
||||
{
|
||||
local parts = split1(name, /_/);
|
||||
name = parts[1];
|
||||
|
||||
if ( name !in pending_commands || |parts| < 2 )
|
||||
return;
|
||||
|
||||
local track_file = parts[2];
|
||||
|
||||
Input::remove(name);
|
||||
|
||||
if ( name !in pending_files )
|
||||
delete pending_commands[name];
|
||||
else
|
||||
{
|
||||
delete pending_files[name][track_file];
|
||||
if ( |pending_files[name]| == 0 )
|
||||
delete pending_commands[name];
|
||||
system(fmt("rm \"%s\"", str_shell_escape(track_file)));
|
||||
}
|
||||
}
|
||||
|
||||
event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool)
|
||||
{
|
||||
if ( name !in pending_commands )
|
||||
return;
|
||||
|
||||
Input::remove(name);
|
||||
results[name]$exit_code = exit_code;
|
||||
results[name]$signal_exit = signal_exit;
|
||||
|
||||
if ( name !in pending_files || |pending_files[name]| == 0 )
|
||||
# No extra files to read, command is done.
|
||||
delete pending_commands[name];
|
||||
else
|
||||
for ( read_file in pending_files[name] )
|
||||
Input::add_event([$source=fmt("%s", read_file),
|
||||
$name=fmt("%s_%s", name, read_file),
|
||||
$reader=Input::READER_RAW,
|
||||
$want_record=F,
|
||||
$fields=FileLine,
|
||||
$ev=Exec::file_line]);
|
||||
}
|
||||
|
||||
function run(cmd: Command): Result
|
||||
{
|
||||
add pending_commands[cmd$uid];
|
||||
results[cmd$uid] = [];
|
||||
|
||||
if ( cmd?$read_files )
|
||||
{
|
||||
for ( read_file in cmd$read_files )
|
||||
{
|
||||
if ( cmd$uid !in pending_files )
|
||||
pending_files[cmd$uid] = set();
|
||||
add pending_files[cmd$uid][read_file];
|
||||
}
|
||||
}
|
||||
|
||||
local config_strings: table[string] of string = {
|
||||
["stdin"] = cmd$stdin,
|
||||
["read_stderr"] = "1",
|
||||
};
|
||||
Input::add_event([$name=cmd$uid,
|
||||
$source=fmt("%s |", cmd$cmd),
|
||||
$reader=Input::READER_RAW,
|
||||
$fields=Exec::OneLine,
|
||||
$ev=Exec::line,
|
||||
$want_record=F,
|
||||
$config=config_strings]);
|
||||
|
||||
return when ( cmd$uid !in pending_commands )
|
||||
{
|
||||
local result = results[cmd$uid];
|
||||
delete results[cmd$uid];
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
event bro_done()
|
||||
{
|
||||
# We are punting here and just deleting any unprocessed files.
|
||||
for ( uid in pending_files )
|
||||
for ( fname in pending_files[uid] )
|
||||
system(fmt("rm \"%s\"", str_shell_escape(fname)));
|
||||
}
|
||||
|
|
@ -6,21 +6,21 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str
|
|||
{
|
||||
local conn_info = fmt("%s:%d-%s:%d", addr_to_uri(c$id$orig_h), c$id$orig_p,
|
||||
addr_to_uri(c$id$resp_h), c$id$resp_p);
|
||||
|
||||
|
||||
if ( prefix != "" )
|
||||
conn_info = fmt("%s_%s", prefix, conn_info);
|
||||
if ( suffix != "" )
|
||||
conn_info = fmt("%s_%s", conn_info, suffix);
|
||||
|
||||
|
||||
return conn_info;
|
||||
}
|
||||
|
||||
## For CONTENT-DISPOSITION headers, this function can be used to extract
|
||||
|
||||
## For CONTENT-DISPOSITION headers, this function can be used to extract
|
||||
## the filename.
|
||||
function extract_filename_from_content_disposition(data: string): string
|
||||
{
|
||||
local filename = sub(data, /^.*[nN][aA][mM][eE][[:blank:]]*\*?=[[:blank:]]*/, "");
|
||||
|
||||
|
||||
# Remove quotes around the filename if they are there.
|
||||
if ( /^\"/ in filename )
|
||||
filename = split_n(filename, /\"/, F, 2)[2];
|
||||
|
|
@ -30,4 +30,4 @@ function extract_filename_from_content_disposition(data: string): string
|
|||
filename = sub(filename, /^.+'.*'/, "");
|
||||
|
||||
return unescape_URI(filename);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue