Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-17 14:08:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch 'master' into topic/jsiwek/faf-cleanup

Browse source 
Conflicts:
	scripts/base/protocols/ftp/file-analysis.bro
	scripts/base/protocols/http/file-analysis.bro
	scripts/base/protocols/irc/file-analysis.bro
	scripts/base/protocols/smtp/file-analysis.bro
	src/file_analysis/File.cc
	src/file_analysis/File.h
	src/file_analysis/Manager.cc
	src/file_analysis/Manager.h
	testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-0.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-1.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-2.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-3.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-BTsa70Ua9x7-1.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-BTsa70Ua9x7.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-Rqjkzoroau4-0.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-Rqjkzoroau4.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-VLQvJybrm38-2.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-VLQvJybrm38.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-zrfwSs9K1yk-3.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-zrfwSs9K1yk.dat
	testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log
	testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item-BFymS6bFgT3-0.dat
	testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item-BFymS6bFgT3.dat
	testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item.dat
	testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
	testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item-wqKMAamJVSb-0.dat
	testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item-wqKMAamJVSb.dat
	testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item.dat
	testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
	testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-0.dat
	testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-1.dat
	testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-Ltd7QO7jEv3-1.dat
	testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-Ltd7QO7jEv3.dat
	testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-cwR7l6Zctxb-0.dat
	testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-cwR7l6Zctxb.dat
	testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
	testing/btest/scripts/base/protocols/ftp/ftp-extract.bro
	testing/btest/scripts/base/protocols/http/http-extract-files.bro
	testing/btest/scripts/base/protocols/irc/dcc-extract.test
	testing/btest/scripts/base/protocols/smtp/mime-extract.test
This commit is contained in:
Jon Siwek 2013-06-07 15:44:36 -05:00
commit f2574636b6
517 changed files with 15571 additions and 12637 deletions
Download patch file Download diff file Expand all files Collapse all files

6
scripts/base/protocols/ftp/file-analysis.bro
View file

@ -11,7 +11,7 @@ export {
function get_handle_string(c: connection): string
{
return cat(ANALYZER_FTP_DATA, " ", c$start_time, " ", id_string(c$id));
return cat(Analyzer::ANALYZER_FTP_DATA, " ", c$start_time, " ", id_string(c$id));
}
function get_file_handle(c: connection, is_orig: bool): string
@ -40,9 +40,9 @@ function get_file_handle(c: connection, is_orig: bool): string
module GLOBAL;
event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool)
&priority=5
{
if ( tag != ANALYZER_FTP_DATA ) return;
if ( tag != Analyzer::ANALYZER_FTP_DATA ) return;
set_file_handle(FTP::get_file_handle(c, is_orig));
}

80
scripts/base/protocols/ftp/main.bro
View file

@ -1,6 +1,6 @@
##! The logging this script does is primarily focused on logging FTP commands
##! along with metadata. For example, if files are transferred, the argument
##! will take on the full path that the client is at along with the requested
##! will take on the full path that the client is at along with the requested
##! file name.
@load ./utils-commands
@ -13,16 +13,16 @@ module FTP;
export {
## The FTP protocol logging stream identifier.
redef enum Log::ID += { LOG };
## List of commands that should have their command/response pairs logged.
const logged_commands = {
"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT", "PORT", "PASV", "EPRT",
"EPSV"
} &redef;
## This setting changes if passwords used in FTP sessions are captured or not.
const default_capture_password = F &redef;
## User IDs that can be considered "anonymous".
const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef;
@ -37,7 +37,7 @@ export {
## The port at which the acceptor is listening for the data connection.
resp_p: port &log;
};
type Info: record {
## Time when the command was sent.
ts: time &log;
@ -53,12 +53,12 @@ export {
command: string &log &optional;
## Argument for the command if one is given.
arg: string &log &optional;
## Libmagic "sniffed" file type if the command indicates a file transfer.
mime_type: string &log &optional;
## Size of the file if the command indicates a file transfer.
file_size: count &log &optional;
## Reply code from the server in response to the command.
reply_code: count &log &optional;
## Reply message from the server in response to the command.
@ -74,31 +74,31 @@ export {
## more concrete is discovered that the existing but unknown
## directory is ok to use.
cwd: string &default=".";
## Command that is currently waiting for a response.
cmdarg: CmdArg &optional;
## Queue for commands that have been sent but not yet responded to
## Queue for commands that have been sent but not yet responded to
## are tracked here.
pending_commands: PendingCmds;
## Indicates if the session is in active or passive mode.
passive: bool &default=F;
## Determines if the password will be captured for this request.
capture_password: bool &default=default_capture_password;
};
## This record is to hold a parsed FTP reply code. For example, for the
## This record is to hold a parsed FTP reply code. For example, for the
## 201 status code, the digits would be parsed as: x->2, y->0, z=>1.
type ReplyCode: record {
x: count;
y: count;
z: count;
};
## Parse FTP reply codes into the three constituent single digit values.
global parse_ftp_reply_code: function(code: count): ReplyCode;
## Event that can be handled to access the :bro:type:`FTP::Info`
## record as it is sent on to the logging framework.
global log_ftp: event(rec: Info);
@ -111,11 +111,10 @@ redef record connection += {
};
# Configure DPD
const ports = { 21/tcp, 2811/tcp } &redef; # 2811/tcp is GridFTP.
redef capture_filters += { ["ftp"] = "port 21 and port 2811" };
redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
redef likely_server_ports += { 21/tcp, 2811/tcp };
const ports = { 21/tcp, 2811/tcp };
redef likely_server_ports += { ports };
# Establish the variable for tracking expected connections.
global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
@ -123,6 +122,7 @@ global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
event bro_init() &priority=5
{
Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]);
Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, ports);
}
## A set of commands where the argument can be expected to refer
@ -166,7 +166,7 @@ function set_ftp_session(c: connection)
s$uid=c$uid;
s$id=c$id;
c$ftp=s;
# Add a shim command so the server can respond with some init response.
add_pending_cmd(c$ftp$pending_commands, "<init>", "");
}
@ -178,13 +178,13 @@ function ftp_message(s: Info)
# or it's a deliberately logged command.
if ( |s$tags| > 0 || (s?$cmdarg && s$cmdarg$cmd in logged_commands) )
{
if ( s?$password &&
! s$capture_password &&
if ( s?$password &&
! s$capture_password &&
to_lower(s$user) !in guest_ids )
{
s$password = "<hidden>";
}
local arg = s$cmdarg$arg;
if ( s$cmdarg$cmd in file_cmds )
{
@ -194,7 +194,7 @@ function ftp_message(s: Info)
arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path);
}
s$ts=s$cmdarg$ts;
s$command=s$cmdarg$cmd;
if ( arg == "" )
@ -204,9 +204,9 @@ function ftp_message(s: Info)
Log::write(FTP::LOG, s);
}
# The MIME and file_size fields are specific to file transfer commands
# and may not be used in all commands so they need reset to "blank"
# The MIME and file_size fields are specific to file transfer commands
# and may not be used in all commands so they need reset to "blank"
# values after logging.
delete s$mime_type;
delete s$file_size;
@ -221,8 +221,8 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel)
s$passive = chan$passive;
s$data_channel = chan;
ftp_data_expected[chan$resp_h, chan$resp_p] = s;
expect_connection(chan$orig_h, chan$resp_h, chan$resp_p, ANALYZER_FTP_DATA,
5mins);
Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, Analyzer::ANALYZER_FTP_DATA,
5mins);
}
event ftp_request(c: connection, command: string, arg: string) &priority=5
@ -237,19 +237,19 @@ event ftp_request(c: connection, command: string, arg: string) &priority=5
remove_pending_cmd(c$ftp$pending_commands, c$ftp$cmdarg);
ftp_message(c$ftp);
}
local id = c$id;
set_ftp_session(c);
# Queue up the new command and argument
add_pending_cmd(c$ftp$pending_commands, command, arg);
if ( command == "USER" )
c$ftp$user = arg;
else if ( command == "PASS" )
c$ftp$password = arg;
else if ( command == "PORT" || command == "EPRT" )
{
local data = (command == "PORT") ?
@ -277,7 +277,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
# TODO: figure out what to do with continued FTP response (not used much)
if ( cont_resp ) return;
# TODO: do some sort of generic clear text login processing here.
local response_xyz = parse_ftp_reply_code(code);
#if ( response_xyz$x == 2 && # successful
@ -293,17 +293,17 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
# if that's given as well which would be more correct.
c$ftp$file_size = extract_count(msg);
}
# PASV and EPSV processing
else if ( (code == 227 || code == 229) &&
(c$ftp$cmdarg$cmd == "PASV" || c$ftp$cmdarg$cmd == "EPSV") )
{
local data = (code == 227) ? parse_ftp_pasv(msg) : parse_ftp_epsv(msg);
if ( data$valid )
{
c$ftp$passive=T;
if ( code == 229 && data$h == [::] )
data$h = c$id$resp_h;
@ -327,9 +327,9 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
else if ( c$ftp$cmdarg$cmd == "PWD" || c$ftp$cmdarg$cmd == "XPWD" )
c$ftp$cwd = extract_path(msg);
}
# In case there are multiple commands queued, go ahead and remove the
# command here and log because we can't do the normal processing pipeline
# command here and log because we can't do the normal processing pipeline
# to wait for a new command before logging the command/response pair.
if ( |c$ftp$pending_commands| > 1 )
{
@ -338,7 +338,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
}
}
event expected_connection_seen(c: connection, a: count) &priority=10
event scheduled_analyzer_applied(c: connection, a: Analyzer::Tag) &priority=10
{
local id = c$id;
if ( [id$resp_h, id$resp_p] in ftp_data_expected )
@ -361,7 +361,7 @@ event connection_reused(c: connection) &priority=5
if ( "ftp-data" in c$service )
c$ftp_data_reuse = T;
}
event connection_state_remove(c: connection) &priority=-5
{
if ( c$ftp_data_reuse ) return;