Merge remote-tracking branch 'origin/master' into topic/seth/faf-updates

Conflicts:
	scripts/base/frameworks/files/main.bro
	scripts/base/init-bare.bro
	scripts/base/protocols/ftp/file-analysis.bro
	scripts/base/protocols/http/file-analysis.bro
	scripts/base/protocols/irc/file-analysis.bro
	scripts/base/protocols/smtp/file-analysis.bro
	src/const.bif
	src/event.bif
	src/file_analysis/Analyzer.h
	src/file_analysis/file_analysis.bif
This commit is contained in:
Seth Hall 2013-07-05 02:13:27 -04:00
commit 58d133e764
555 changed files with 16982 additions and 13190 deletions

View file

@ -6,9 +6,9 @@ module Conn;
export {
## Define inactivity timeouts by the service detected being used over
## the connection.
const analyzer_inactivity_timeouts: table[AnalyzerTag] of interval = {
const analyzer_inactivity_timeouts: table[Analyzer::Tag] of interval = {
# For interactive services, allow longer periods of inactivity.
[[ANALYZER_SSH, ANALYZER_FTP]] = 1 hrs,
[[Analyzer::ANALYZER_SSH, Analyzer::ANALYZER_FTP]] = 1 hrs,
} &redef;
## Define inactivity timeouts based on common protocol ports.
@ -18,7 +18,7 @@ export {
}
event protocol_confirmation(c: connection, atype: count, aid: count)
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count)
{
if ( atype in analyzer_inactivity_timeouts )
set_inactivity_timeout(c$id, analyzer_inactivity_timeouts[atype]);

View file

@ -130,19 +130,13 @@ redef capture_filters += {
["netbios-ns"] = "udp port 137",
};
const dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
redef dpd_config += { [ANALYZER_DNS] = [$ports = dns_ports] };
const dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp };
const dns_tcp_ports = { 53/tcp };
redef dpd_config += { [ANALYZER_DNS_UDP_BINPAC] = [$ports = dns_udp_ports] };
redef dpd_config += { [ANALYZER_DNS_TCP_BINPAC] = [$ports = dns_tcp_ports] };
redef likely_server_ports += { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
const ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
redef likely_server_ports += { ports };
event bro_init() &priority=5
{
Log::create_stream(DNS::LOG, [$columns=Info, $ev=log_dns]);
Analyzer::register_for_ports(Analyzer::ANALYZER_DNS, ports);
}
function new_session(c: connection, trans_id: count): Info

View file

@ -1,6 +1,6 @@
##! The logging this script does is primarily focused on logging FTP commands
##! along with metadata. For example, if files are transferred, the argument
##! will take on the full path that the client is at along with the requested
##! will take on the full path that the client is at along with the requested
##! file name.
@load ./utils-commands
@ -13,16 +13,16 @@ module FTP;
export {
## The FTP protocol logging stream identifier.
redef enum Log::ID += { LOG };
## List of commands that should have their command/response pairs logged.
const logged_commands = {
"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT", "PORT", "PASV", "EPRT",
"EPSV"
} &redef;
## This setting changes if passwords used in FTP sessions are captured or not.
const default_capture_password = F &redef;
## User IDs that can be considered "anonymous".
const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef;
@ -37,7 +37,7 @@ export {
## The port at which the acceptor is listening for the data connection.
resp_p: port &log;
};
type Info: record {
## Time when the command was sent.
ts: time &log;
@ -53,12 +53,12 @@ export {
command: string &log &optional;
## Argument for the command if one is given.
arg: string &log &optional;
## Libmagic "sniffed" file type if the command indicates a file transfer.
mime_type: string &log &optional;
## Size of the file if the command indicates a file transfer.
file_size: count &log &optional;
## Reply code from the server in response to the command.
reply_code: count &log &optional;
## Reply message from the server in response to the command.
@ -74,31 +74,31 @@ export {
## more concrete is discovered that the existing but unknown
## directory is ok to use.
cwd: string &default=".";
## Command that is currently waiting for a response.
cmdarg: CmdArg &optional;
## Queue for commands that have been sent but not yet responded to
## Queue for commands that have been sent but not yet responded to
## are tracked here.
pending_commands: PendingCmds;
## Indicates if the session is in active or passive mode.
passive: bool &default=F;
## Determines if the password will be captured for this request.
capture_password: bool &default=default_capture_password;
};
## This record is to hold a parsed FTP reply code. For example, for the
## This record is to hold a parsed FTP reply code. For example, for the
## 201 status code, the digits would be parsed as: x->2, y->0, z=>1.
type ReplyCode: record {
x: count;
y: count;
z: count;
};
## Parse FTP reply codes into the three constituent single digit values.
global parse_ftp_reply_code: function(code: count): ReplyCode;
## Event that can be handled to access the :bro:type:`FTP::Info`
## record as it is sent on to the logging framework.
global log_ftp: event(rec: Info);
@ -111,11 +111,10 @@ redef record connection += {
};
# Configure DPD
const ports = { 21/tcp, 2811/tcp } &redef; # 2811/tcp is GridFTP.
redef capture_filters += { ["ftp"] = "port 21 and port 2811" };
redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
redef likely_server_ports += { 21/tcp, 2811/tcp };
const ports = { 21/tcp, 2811/tcp };
redef likely_server_ports += { ports };
# Establish the variable for tracking expected connections.
global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
@ -123,6 +122,7 @@ global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
event bro_init() &priority=5
{
Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]);
Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, ports);
}
## A set of commands where the argument can be expected to refer
@ -166,7 +166,7 @@ function set_ftp_session(c: connection)
s$uid=c$uid;
s$id=c$id;
c$ftp=s;
# Add a shim command so the server can respond with some init response.
add_pending_cmd(c$ftp$pending_commands, "<init>", "");
}
@ -178,13 +178,13 @@ function ftp_message(s: Info)
# or it's a deliberately logged command.
if ( |s$tags| > 0 || (s?$cmdarg && s$cmdarg$cmd in logged_commands) )
{
if ( s?$password &&
! s$capture_password &&
if ( s?$password &&
! s$capture_password &&
to_lower(s$user) !in guest_ids )
{
s$password = "<hidden>";
}
local arg = s$cmdarg$arg;
if ( s$cmdarg$cmd in file_cmds )
{
@ -194,7 +194,7 @@ function ftp_message(s: Info)
arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path);
}
s$ts=s$cmdarg$ts;
s$command=s$cmdarg$cmd;
if ( arg == "" )
@ -204,9 +204,9 @@ function ftp_message(s: Info)
Log::write(FTP::LOG, s);
}
# The MIME and file_size fields are specific to file transfer commands
# and may not be used in all commands so they need reset to "blank"
# The MIME and file_size fields are specific to file transfer commands
# and may not be used in all commands so they need reset to "blank"
# values after logging.
delete s$mime_type;
delete s$file_size;
@ -221,8 +221,8 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel)
s$passive = chan$passive;
s$data_channel = chan;
ftp_data_expected[chan$resp_h, chan$resp_p] = s;
expect_connection(chan$orig_h, chan$resp_h, chan$resp_p, ANALYZER_FTP_DATA,
5mins);
Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, Analyzer::ANALYZER_FTP_DATA,
5mins);
}
event ftp_request(c: connection, command: string, arg: string) &priority=5
@ -237,19 +237,19 @@ event ftp_request(c: connection, command: string, arg: string) &priority=5
remove_pending_cmd(c$ftp$pending_commands, c$ftp$cmdarg);
ftp_message(c$ftp);
}
local id = c$id;
set_ftp_session(c);
# Queue up the new command and argument
add_pending_cmd(c$ftp$pending_commands, command, arg);
if ( command == "USER" )
c$ftp$user = arg;
else if ( command == "PASS" )
c$ftp$password = arg;
else if ( command == "PORT" || command == "EPRT" )
{
local data = (command == "PORT") ?
@ -277,7 +277,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
# TODO: figure out what to do with continued FTP response (not used much)
if ( cont_resp ) return;
# TODO: do some sort of generic clear text login processing here.
local response_xyz = parse_ftp_reply_code(code);
#if ( response_xyz$x == 2 && # successful
@ -293,17 +293,17 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
# if that's given as well which would be more correct.
c$ftp$file_size = extract_count(msg);
}
# PASV and EPSV processing
else if ( (code == 227 || code == 229) &&
(c$ftp$cmdarg$cmd == "PASV" || c$ftp$cmdarg$cmd == "EPSV") )
{
local data = (code == 227) ? parse_ftp_pasv(msg) : parse_ftp_epsv(msg);
if ( data$valid )
{
c$ftp$passive=T;
if ( code == 229 && data$h == [::] )
data$h = c$id$resp_h;
@ -327,9 +327,9 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
else if ( c$ftp$cmdarg$cmd == "PWD" || c$ftp$cmdarg$cmd == "XPWD" )
c$ftp$cwd = extract_path(msg);
}
# In case there are multiple commands queued, go ahead and remove the
# command here and log because we can't do the normal processing pipeline
# command here and log because we can't do the normal processing pipeline
# to wait for a new command before logging the command/response pair.
if ( |c$ftp$pending_commands| > 1 )
{
@ -338,7 +338,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
}
}
event expected_connection_seen(c: connection, a: count) &priority=10
event scheduled_analyzer_applied(c: connection, a: Analyzer::Tag) &priority=10
{
local id = c$id;
if ( [id$resp_h, id$resp_p] in ftp_data_expected )
@ -361,7 +361,7 @@ event connection_reused(c: connection) &priority=5
if ( "ftp-data" in c$service )
c$ftp_data_reuse = T;
}
event connection_state_remove(c: connection) &priority=-5
{
if ( c$ftp_data_reuse ) return;

View file

@ -127,29 +127,26 @@ redef record connection += {
http_state: State &optional;
};
# Initialize the HTTP logging stream.
event bro_init() &priority=5
{
Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http]);
}
# DPD configuration.
const ports = {
80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp,
8000/tcp, 8080/tcp, 8888/tcp,
};
redef dpd_config += {
[[ANALYZER_HTTP, ANALYZER_HTTP_BINPAC]] = [$ports = ports],
};
redef capture_filters += {
["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)"
};
redef likely_server_ports += {
80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp,
const ports = {
80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp,
8000/tcp, 8080/tcp, 8888/tcp,
};
redef likely_server_ports += { ports };
# Initialize the HTTP logging stream and ports.
event bro_init() &priority=5
{
Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http]);
Analyzer::register_for_ports(Analyzer::ANALYZER_HTTP, ports);
}
function code_in_range(c: count, min: count, max: count) : bool
{
return c >= min && c <= max;

View file

@ -70,11 +70,11 @@ event irc_dcc_message(c: connection, is_orig: bool,
c$irc$dcc_file_name = argument;
c$irc$dcc_file_size = size;
local p = count_to_port(dest_port, tcp);
expect_connection(to_addr("0.0.0.0"), address, p, ANALYZER_IRC_DATA, 5 min);
Analyzer::schedule_analyzer(0.0.0.0, address, p, Analyzer::ANALYZER_IRC_DATA, 5 min);
dcc_expected_transfers[address, p] = c$irc;
}
event expected_connection_seen(c: connection, a: count) &priority=10
event expected_connection_seen(c: connection, a: Analyzer::Tag) &priority=10
{
local id = c$id;
if ( [id$resp_h, id$resp_p] in dcc_expected_transfers )

View file

@ -45,14 +45,13 @@ redef capture_filters += { ["irc-6668"] = "port 6668" };
redef capture_filters += { ["irc-6669"] = "port 6669" };
# DPD configuration.
const irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
redef dpd_config += { [ANALYZER_IRC] = [$ports = irc_ports] };
redef likely_server_ports += { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
const ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
redef likely_server_ports += { ports };
event bro_init() &priority=5
{
Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log]);
Analyzer::register_for_ports(Analyzer::ANALYZER_IRC, ports);
}
function new_session(c: connection): Info

View file

@ -31,12 +31,14 @@ redef record connection += {
# Configure DPD and the packet filter.
redef capture_filters += { ["modbus"] = "tcp port 502" };
redef dpd_config += { [ANALYZER_MODBUS] = [$ports = set(502/tcp)] };
redef likely_server_ports += { 502/tcp };
const ports = { 502/tcp };
redef likely_server_ports += { ports };
event bro_init() &priority=5
{
Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus]);
Analyzer::register_for_ports(Analyzer::ANALYZER_MODBUS, ports);
}
event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool) &priority=5

View file

@ -74,9 +74,6 @@ export {
const mail_path_capture = ALL_HOSTS &redef;
global log_smtp: event(rec: Info);
## Configure the default ports for SMTP analysis.
const ports = { 25/tcp, 587/tcp } &redef;
}
redef record connection += {
@ -86,13 +83,14 @@ redef record connection += {
# Configure DPD
redef capture_filters += { ["smtp"] = "tcp port 25 or tcp port 587" };
redef dpd_config += { [ANALYZER_SMTP] = [$ports = ports] };
redef likely_server_ports += { 25/tcp, 587/tcp };
const ports = { 25/tcp, 587/tcp };
redef likely_server_ports += { ports };
event bro_init() &priority=5
{
Log::create_stream(SMTP::LOG, [$columns=SMTP::Info, $ev=log_smtp]);
Analyzer::register_for_ports(Analyzer::ANALYZER_SMTP, ports);
}
function find_address_in_smtp_header(header: string): string

View file

@ -34,9 +34,13 @@ export {
global log_socks: event(rec: Info);
}
const ports = { 1080/tcp };
redef likely_server_ports += { ports };
event bro_init() &priority=5
{
Log::create_stream(SOCKS::LOG, [$columns=Info, $ev=log_socks]);
Analyzer::register_for_ports(Analyzer::ANALYZER_SOCKS, ports);
}
redef record connection += {
@ -45,7 +49,6 @@ redef record connection += {
# Configure DPD
redef capture_filters += { ["socks"] = "tcp port 1080" };
redef dpd_config += { [ANALYZER_SOCKS] = [$ports = set(1080/tcp)] };
redef likely_server_ports += { 1080/tcp };
function set_session(c: connection, version: count)

View file

@ -71,10 +71,11 @@ export {
}
# Configure DPD and the packet filter
redef capture_filters += { ["ssh"] = "tcp port 22" };
redef dpd_config += { [ANALYZER_SSH] = [$ports = set(22/tcp)] };
redef likely_server_ports += { 22/tcp };
const ports = { 22/tcp };
redef capture_filters += { ["ssh"] = "tcp port 22" };
redef likely_server_ports += { ports };
redef record connection += {
ssh: Info &optional;
@ -83,6 +84,7 @@ redef record connection += {
event bro_init() &priority=5
{
Log::create_stream(SSH::LOG, [$columns=Info, $ev=log_ssh]);
Analyzer::register_for_ports(Analyzer::ANALYZER_SSH, ports);
}
function set_session(c: connection)
@ -116,7 +118,7 @@ function check_ssh_connection(c: connection, done: bool)
# Responder must have sent fewer than 40 packets.
c$resp$num_pkts < 40 &&
# If there was a content gap we can't reliably do this heuristic.
c?$conn && c$conn$missed_bytes == 0)# &&
c?$conn && c$conn$missed_bytes == 0 )# &&
# Only "normal" connections can count.
#c$conn?$conn_state && c$conn$conn_state in valid_states )
{
@ -176,6 +178,7 @@ event ssh_watcher(c: connection)
if ( ! connection_exists(id) )
return;
lookup_connection(c$id);
check_ssh_connection(c, F);
if ( ! c$ssh$done )
schedule +15secs { ssh_watcher(c) };

View file

@ -94,11 +94,6 @@ redef record Info += {
delay_tokens: set[string] &optional;
};
event bro_init() &priority=5
{
Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
}
redef capture_filters += {
["ssl"] = "tcp port 443",
["nntps"] = "tcp port 563",
@ -117,23 +112,15 @@ redef capture_filters += {
const ports = {
443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
};
} &redef;
redef dpd_config += {
[[ANALYZER_SSL]] = [$ports = ports]
};
redef likely_server_ports += { ports };
redef likely_server_ports += {
443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
};
# A queue that buffers log records.
global log_delay_queue: table[count] of Info;
# The top queue index where records are added.
global log_delay_queue_head = 0;
# The bottom queue index that points to the next record to be flushed.
global log_delay_queue_tail = 0;
event bro_init() &priority=5
{
Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, ports);
}
function set_session(c: connection)
{
@ -144,26 +131,17 @@ function set_session(c: connection)
function delay_log(info: Info, token: string)
{
info$delay_tokens = set();
if ( ! info?$delay_tokens )
info$delay_tokens = set();
add info$delay_tokens[token];
log_delay_queue[log_delay_queue_head] = info;
++log_delay_queue_head;
}
function undelay_log(info: Info, token: string)
{
if ( token in info$delay_tokens )
if ( info?$delay_tokens && token in info$delay_tokens )
delete info$delay_tokens[token];
}
global log_record: function(info: Info);
event delay_logging(info: Info)
{
log_record(info);
}
function log_record(info: Info)
{
if ( ! info?$delay_tokens || |info$delay_tokens| == 0 )
@ -172,26 +150,14 @@ function log_record(info: Info)
}
else
{
for ( unused_index in log_delay_queue )
when ( |info$delay_tokens| == 0 )
{
if ( log_delay_queue_head == log_delay_queue_tail )
return;
if ( |log_delay_queue[log_delay_queue_tail]$delay_tokens| > 0 )
{
if ( info$ts + max_log_delay > network_time() )
{
schedule 1sec { delay_logging(info) };
return;
}
else
{
Reporter::info(fmt("SSL delay tokens not released in time (%s)",
info$delay_tokens));
}
}
Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
delete log_delay_queue[log_delay_queue_tail];
++log_delay_queue_tail;
log_record(info);
}
timeout max_log_delay
{
Reporter::info(fmt("SSL delay tokens not released in time (%s tokens remaining)",
|info$delay_tokens|));
}
}
}
@ -288,28 +254,16 @@ event ssl_established(c: connection) &priority=-5
finish(c);
}
event protocol_confirmation(c: connection, atype: count, aid: count) &priority=5
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=5
{
# Check by checking for existence of c$ssl record.
if ( c?$ssl && analyzer_name(atype) == "SSL" )
if ( c?$ssl && atype == Analyzer::ANALYZER_SSL )
c$ssl$analyzer_id = aid;
}
event protocol_violation(c: connection, atype: count, aid: count,
event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
reason: string) &priority=5
{
if ( c?$ssl )
finish(c);
}
event bro_done()
{
if ( |log_delay_queue| == 0 )
return;
for ( unused_index in log_delay_queue )
{
Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
delete log_delay_queue[log_delay_queue_tail];
++log_delay_queue_tail;
}
}

View file

@ -27,10 +27,9 @@ export {
}
redef capture_filters += { ["syslog"] = "port 514" };
const ports = { 514/udp } &redef;
redef dpd_config += { [ANALYZER_SYSLOG_BINPAC] = [$ports = ports] };
redef likely_server_ports += { 514/udp };
const ports = { 514/udp };
redef likely_server_ports += { ports };
redef record connection += {
syslog: Info &optional;
@ -39,6 +38,7 @@ redef record connection += {
event bro_init() &priority=5
{
Log::create_stream(Syslog::LOG, [$columns=Info]);
Analyzer::register_for_ports(Analyzer::ANALYZER_SYSLOG, ports);
}
event syslog_message(c: connection, facility: count, severity: count, msg: string) &priority=5