Updates to make scripts work with logging-internals code.

This commit is contained in:
Seth Hall 2011-03-17 14:29:07 -04:00
parent f4b27016cf
commit f5668e41a2
8 changed files with 203 additions and 131 deletions

View file

@ -36,12 +36,14 @@ export {
# This is where users can get access to the active Log record for a
# connection so they can extend and enhance the logged data.
global active_conns: table[conn_id] of Log;
global log_conn: event(rec: Log);
}
event bro_init()
{
Log::create_stream("CONN", "Conn::Log");
Log::add_default_filter("CONN");
Log::create_stream(CONN, [$columns=Conn::Log, $ev=log_conn]);
Log::add_default_filter(CONN);
}
function conn_state(c: connection, trans: transport_proto): string
@ -143,7 +145,7 @@ event connection_established(c: connection) &priority = 10
event connection_state_remove(c: connection) &priority = -10
{
local conn_log = get_conn_log(c);
Log::write("CONN", conn_log);
Log::write(CONN, conn_log);
if ( c$id in active_conns )
delete active_conns[c$id];

View file

@ -23,8 +23,15 @@ redef enum Notice::Type += {
FTP_Site_Exec_Success,
};
redef enum Log::ID += { FTP };
# Configure DPD
const ports = { 21/tcp } &redef;
redef capture_filters += { ["ftp"] = "port 21" };
redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
export {
redef enum Log::ID += { FTP };
type LogTags: enum {
UNKNOWN
};
@ -47,7 +54,7 @@ export {
type SessionInfo: record {
log: Log;
## By setting the CWD to '/.', we can indicate that unless something
## more concrete is discovered that the exiting but unknown
## more concrete is discovered that the existing but unknown
## directory is ok to use.
cwd: string &default="/.";
command: CmdArg &optional;
@ -82,19 +89,15 @@ export {
## This tracks all of the currently established FTP control sessions.
global active_conns: table[conn_id] of SessionInfo &read_expire=5mins;
global log_ftp: event(rec: Log);
}
global ftp_data_expected: table[addr, port] of ExpectedConn &create_expire=5mins;
# Configure DPD
const ports = { 21/tcp } &redef;
redef capture_filters += { ["ftp"] = "port 21" };
redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
event bro_init()
{
Log::create_stream("FTP", "FTP::Log");
Log::add_default_filter("FTP");
Log::create_stream(FTP, [$columns=FTP::Log, $ev=log_ftp]);
Log::add_default_filter(FTP);
}
# A set of commands where the argument can be expected to refer
@ -128,7 +131,7 @@ function parse_ftp_reply_code(code: count): ReplyCode
return a;
}
function new_ftp_session(c: connection)
function get_ftp_session(c: connection): SessionInfo
{
local id = c$id;
@ -141,13 +144,14 @@ function new_ftp_session(c: connection)
add_pending_cmd(info$pending_commands, "<init>", "");
active_conns[id] = info;
return info;
}
function ftp_message(s: SessionInfo)
{
# If it either has a tag associated with it (something detected)
# or it's a deliberately logged command.
if ( |s$log$tags| > 0 || s$command$cmd in logged_commands )
if ( |s$log$tags| > 0 || (s?$command && s$command$cmd in logged_commands) )
{
local pass = "\\N";
if ( to_lower(s$log$user) in guest_ids && s$log?$password )
@ -162,7 +166,7 @@ function ftp_message(s: SessionInfo)
s$log$arg=arg;
# TODO: does the framework do this atomicly or do I need the copy?
Log::write("FTP", copy(s$log));
Log::write(FTP, copy(s$log));
}
# The MIME and file_size fields are specific to file transfer commands
@ -184,9 +188,7 @@ event ftp_request(c: connection, command: string, arg: string) &priority=1
#if ( is_string_binary(command) ) return;
local id = c$id;
if ( id !in active_conns )
new_ftp_session(c);
local session = active_conns[id];
local session = get_ftp_session(c);
# Log the previous command when a new command is seen.
# The downside here is that commands definitely aren't logged until the
@ -233,9 +235,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
if ( cont_resp ) return;
local id = c$id;
if ( id !in active_conns )
new_ftp_session(c);
local session = active_conns[id];
local session = get_ftp_session(c);
session$command = get_pending_cmd(session$pending_commands, code, msg);
@ -247,6 +247,8 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
#if ( response_xyz$x == 2 && # successful
# session$command$cmd == "PASS" )
# do_ftp_login(c, session);
if ( session$command$cmd == "PASS" )
print fmt("Woo: %s %s", session$log$user, session$log$password);
if ( code == 150 && session$command$cmd == "RETR" )
{
@ -333,27 +335,27 @@ event connection_state_remove(c: connection) &priority=1
delete active_conns[id];
}
event expected_connection_seen(c: connection, a: count) &priority=1
{
local id = c$id;
if ( [id$resp_h, id$resp_p] in ftp_data_expected )
add c$service["ftp-data"];
}
#event expected_connection_seen(c: connection, a: count) &priority=1
# {
# local id = c$id;
# if ( [id$resp_h, id$resp_p] in ftp_data_expected )
# add c$service["ftp-data"];
# }
event file_transferred(c: connection, prefix: string, descr: string,
mime_type: string) &priority=1
{
local id = c$id;
if ( [id$resp_h, id$resp_p] in ftp_data_expected )
{
local expected = ftp_data_expected[id$resp_h, id$resp_p];
local s = expected$session;
s$log$mime_type = mime_type;
s$log$mime_desc = descr;
# TODO: not sure if it's ok to delete this here, but it should
# always be called since the file analyzer is always attached
# to ftp-data sessions.
delete ftp_data_expected[id$resp_h, id$resp_p];
}
}
#event file_transferred(c: connection, prefix: string, descr: string,
# mime_type: string) &priority=1
# {
# local id = c$id;
# if ( [id$resp_h, id$resp_p] in ftp_data_expected )
# {
# local expected = ftp_data_expected[id$resp_h, id$resp_p];
# local s = expected$session;
# s$log$mime_type = mime_type;
# s$log$mime_desc = descr;
#
# # TODO: not sure if it's ok to delete this here, but it should
# # always be called since the file analyzer is always attached
# # to ftp-data sessions.
# delete ftp_data_expected[id$resp_h, id$resp_p];
# }
# }

View file

@ -8,13 +8,14 @@
module HTTP;
redef enum Log::ID += { HTTP };
redef enum Software::Type += {
WEB_SERVER,
WEB_BROWSER,
WEB_BROWSER_PLUGIN,
};
redef enum Log::ID += { HTTP };
export {
type LogTags: enum {
## Indicator of a URI based SQL injection attack.
@ -113,12 +114,13 @@ export {
## List of all active HTTP session indexed by conn_id.
global active_conns: table[conn_id] of SessionInfo &read_expire=5mins;
global log_http: event(rec: Log);
}
event bro_init()
{
Log::create_stream("HTTP", "HTTP::Log");
Log::add_default_filter("HTTP");
Log::create_stream(HTTP, [$columns=HTTP::Log, $ev=log_http]);
Log::add_default_filter(HTTP);
}
# DPD configuration.
@ -137,11 +139,7 @@ function new_http_log(id: conn_id): Log
{
local tags: set[LogTags] = set();
local proxied: set[string] = set();
local log: Log = [$ts=network_time(), $id=id, $tags=tags, $proxied=proxied,
# TODO: some bug with record default initialization
$user_agent="", $request_body_size=0, $response_body_size=0, $status_code=0, $status_msg="", $username="", $password="", $referrer="", $host=""
];
return log;
return [$ts=network_time(), $id=id, $tags=tags, $proxied=proxied];
}
function get_http_session(id: conn_id): SessionInfo
@ -250,7 +248,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
{
if ( name == "SERVER" )
{
local si = Software::default_parse(value, c$id$resp_h, WEB_SERVER);
local si = Software::parse(value, c$id$resp_h, WEB_SERVER);
Software::found(c, si);
}
else if ( name == "CONTENT-LENGTH" )
@ -264,10 +262,10 @@ event http_begin_entity(c: connection, is_orig: bool) &priority=1
if ( is_orig )
if ( sess$log_point == AFTER_REQUEST )
Log::write("HTTP", sess$log);
Log::write(HTTP, sess$log);
else
if ( sess$log_point == AFTER_REQUEST )
Log::write("HTTP", sess$log);
Log::write(HTTP, sess$log);
}
event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=1
@ -276,10 +274,10 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
if ( is_orig )
if ( sess$log_point == AFTER_REQUEST_BODY )
Log::write("HTTP", sess$log);
Log::write(HTTP, sess$log);
else
if ( sess$log_point == AFTER_REPLY_BODY )
Log::write("HTTP", sess$log);
Log::write(HTTP, sess$log);
}

View file

@ -2,9 +2,9 @@
module KnownHosts;
redef enum Log::ID += { KNOWN_HOSTS };
export {
redef enum Log::ID += { KNOWN_HOSTS };
type Log: record {
ts: time;
address: addr;
@ -20,13 +20,15 @@ export {
# Maintain the list of known hosts for 24 hours so that the existence
# of each individual address is logged each day.
global known_hosts: set[addr] &create_expire=1day;
global known_hosts: set[addr] &create_expire=1day &syncronized;
global log_known_hosts: event(rec: Log);
}
event bro_init()
{
Log::create_stream("KNOWN_HOSTS", "KnownHosts::Log");
Log::add_default_filter("KNOWN_HOSTS");
Log::create_stream(KNOWN_HOSTS, [$columns=KnownHosts::Log, $ev=log_known_hosts]);
Log::add_default_filter(KNOWN_HOSTS);
}
event connection_established(c: connection)
@ -36,11 +38,11 @@ event connection_established(c: connection)
if ( id$orig_h !in known_hosts && addr_matches_hosts(id$orig_h, logging) )
{
add known_hosts[id$orig_h];
Log::write("KNOWN_HOSTS", [$ts=network_time(), $address=id$orig_h]);
Log::write(KNOWN_HOSTS, [$ts=network_time(), $address=id$orig_h]);
}
if ( id$resp_h !in known_hosts && addr_matches_hosts(id$resp_h, logging) )
{
add known_hosts[id$resp_h];
Log::write("KNOWN_HOSTS", [$ts=network_time(), $address=id$resp_h]);
Log::write(KNOWN_HOSTS, [$ts=network_time(), $address=id$resp_h]);
}
}

View file

@ -1,33 +1,43 @@
#@load global-ext
@load functions
module KnownServices;
redef enum Log::ID += { KNOWN_SERVICES };
export {
redef enum Log::ID += { KNOWN_SERVICES };
type LogPoints: enum {
AFTER_PROTOCOL_DETECTION,
AT_CONNECTION_END,
};
type Log: record {
ts: time;
host: addr;
port_num: port;
# port_num: count; # split 'em?
# port_proto: string;
service: string &default="";
port_proto: transport_proto;
service: set[string];
log_point: LogPoints;
};
# The hosts whose services should be logged.
const logged_hosts = LocalHosts &redef;
const default_log_point = AFTER_PROTOCOL_DETECTION &redef;
global known_services: set[addr, port] &create_expire=1day &synchronized;
global log_known_services: event(rec: Log);
}
# The temporary holding place for new, unknown services.
global established_conns: set[addr, port] &create_expire=1day &redef;
global established_conns: table[addr, port] of Log &create_expire=1day &redef;
event bro_init()
{
Log::create_stream("KNOWN_SERVICES", "KnownServices::Log");
Log::add_default_filter("KNOWN_SERVICES");
Log::create_stream(KNOWN_SERVICES, [$columns=KnownServices::Log,
$ev=log_known_services]);
Log::add_default_filter(KNOWN_SERVICES);
}
event connection_established(c: connection)
@ -38,7 +48,7 @@ event connection_established(c: connection)
add established_conns[id$resp_h, id$resp_p];
}
event known_services_done(c: connection)
function known_services_done(c: connection)
{
local id = c$id;
if ( [id$resp_h, id$resp_p] !in known_services &&
@ -46,20 +56,30 @@ event known_services_done(c: connection)
"ftp-data" !in c$service ) # don't include ftp data sessions
{
add known_services[id$resp_h, id$resp_p];
Log::write( "KNOWN_SERVICES", [ $ts=c$start_time, $host=id$resp_h,
$port_num=id$resp_p, $service=c$service] );
Log::write(KNOWN_SERVICES, [$ts=c$start_time,
$host=id$resp_h,
$port_num=id$resp_p,
$port_proto=get_port_transport_proto(id$resp_p),
$service=c$service] );
}
}
event connection_state_remove(c: connection)
event connection_established(c: connection)
{
event known_services_done(c);
}
# Log the event after protocol detection if
event protocol_confirmation(c: connection, atype: count, aid: count) &priority=-10
{
event known_services_done(c);
local l = established_conns[c$id$resp, c$id$resp_p];
if ( l$log_point == AFTER_PROTOCOL_DETECTION )
known_services_done(c);
}
# Handle the connection ending in case no protocol was ever detected.
event connection_state_remove(c: connection)
{
known_services_done(c);
}

View file

@ -2,6 +2,10 @@
module Notice;
## This couldn't be named NOTICE because that name is already used by the
## global function NOTICE().
redef enum Log::ID += { NOTICE_LOG };
export {
type Type: enum {
NoticeNone, # placeholder
@ -96,6 +100,8 @@ export {
# These are implemented below
global email_notice_to: function(n: Notice::Info, dest: string) &redef;
global notice: function(n: Notice::Info);
global log_notice: event(rec: Info);
}
# Each notice has a unique ID associated with it.
@ -105,8 +111,8 @@ redef new_notice_tag = function(): string
event bro_init()
{
Log::create_stream("NOTICE", "Notice::Info");
Log::add_default_filter("NOTICE");
Log::create_stream(NOTICE_LOG, [$columns=Notice::Info, $ev=log_notice]);
Log::add_default_filter(NOTICE_LOG);
}
function add_notice_tag(c: connection): string
@ -255,7 +261,7 @@ function NOTICE(n: Notice::Info)
{
# Build the info here after we had a chance to set the
# $dropped field.
Log::write("NOTICE", n);
Log::write(NOTICE_LOG, n);
if ( action != NOTICE_FILE && n$do_alarm )
{

View file

@ -17,12 +17,14 @@ redef enum Notice::Type += {
Software_Version_Change,
};
redef enum Log::ID += { SOFTWARE };
export {
type Version: record {
major: count &default=0; ##< Major version number
minor: count &default=0; ##< Minor version number
minor2: count &default=0; ##< Minor subversion number
addl: string &default=""; ##< Additional version string (e.g. "beta42")
major: count &optional; ##< Major version number
minor: count &optional; ##< Minor version number
minor2: count &optional; ##< Minor subversion number
addl: string &optional; ##< Additional version string (e.g. "beta42")
};
type Type: enum {
@ -38,21 +40,20 @@ export {
PRINTER,
};
redef enum Log::ID += { SOFTWARE };
type Info: record {
## The time at which the software was first detected.
ts: time;
## The IP address detected running the software.
host: addr &default=0.0.0.0;
host: addr;
## The type of software detected (e.g. WEB_SERVER)
software_type: Type &default=UNKNOWN;
## Name of the software (e.g. Apache)
name: string &default="";
name: string;
## Version of the software
version: Version;
## The full unparsed version string found because the version parsing
## doesn't work 100% reliably and this acts as a fall back in the logs.
unparsed_version: string &default="";
unparsed_version: string &optional;
};
## The hosts whose software should be logged.
@ -79,7 +80,7 @@ export {
## This function can take many software version strings and parse them into
## a sensible Software::Version record. There are still many cases where
## scripts may have to have their own specific version parsing though.
global default_parse: function(unparsed_version: string,
global parse: function(unparsed_version: string,
host: addr,
software_type: Type): Info;
@ -93,19 +94,21 @@ export {
type SoftwareSet: table[string] of Info;
# The set of software associated with an address.
global tracked_software: table[addr] of SoftwareSet &create_expire=1day &synchronized;
global log_software: event(rec: Info);
}
event bro_init()
{
Log::create_stream("SOFTWARE", "Software::Info");
Log::add_default_filter("SOFTWARE");
Log::create_stream(SOFTWARE, [$columns=Software::Info, $ev=log_software]);
Log::add_default_filter(SOFTWARE);
}
# Don't even try to understand this now, just make sure the tests are
# working.
function default_parse(unparsed_version: string,
host: addr,
software_type: Type): Info
function parse(unparsed_version: string,
host: addr,
software_type: Type): Info
{
local software_name = "<parse error>";
local v: Version;
@ -113,30 +116,29 @@ function default_parse(unparsed_version: string,
# The regular expression should match the complete version number
# and software name.
local version_parts = split_n(unparsed_version, /[0-9\/\-\._ ]{2,}/, T, 1);
if ( 1 in version_parts )
software_name = version_parts[1];
if ( |version_parts| >= 2 )
{
software_name = version_parts[1];
# Remove the name/version separator because it's left at the begining
# of the version number from the previous split_all.
local sv = version_parts[2];
if ( /^[\/\-\._ ]/ in sv )
sv = sub(version_parts[2], /^[\/\-\._ ]/, "");
local version_numbers = split_n(sv, /[\-\._,\[\(\{ ]/, F, 4);
local addl = "";
if ( 4 in version_numbers && version_numbers[4] != "" )
addl = version_numbers[4];
v$addl = version_numbers[4];
else if ( 3 in version_parts && version_parts[3] != "" )
{
# TODO: there's a bug with do_split!
local vp = split_n(version_parts[3], /[\-\._,\[\]\(\)\{\} ]/, F, 2);
if ( |vp| >= 1 && vp[1] != "" )
addl = vp[1];
else if ( |vp| >= 2 )
addl = vp[2];
v$addl = vp[1];
else if ( |vp| >= 2 && vp[2] != "" )
v$addl = vp[2];
else
addl = version_parts[3];
v$addl = version_parts[3];
}
v$addl = addl;
if ( |version_numbers| >= 3 )
v$minor2 = to_count(version_numbers[3]);
@ -153,22 +155,60 @@ function default_parse(unparsed_version: string,
function cmp_versions(v1: Version, v2: Version): int
{
if ( v1$major < v2$major )
return -1;
if ( v1$major > v2$major )
return 1;
if ( v1$minor < v2$minor )
return -1;
if ( v1$minor > v2$minor )
return 1;
if ( v1$minor2 < v2$minor2 )
return -1;
if ( v1$minor2 > v2$minor2 )
return 1;
return strcmp(v1$addl, v2$addl);
if ( v1?$major && v2?$major )
{
if ( v1$major < v2$major )
return -1;
if ( v1$major > v2$major )
return 1;
}
else
{
if ( !v1?$major && !v2?$major )
return 0;
else
return -1;
}
if ( v1?$minor && v2?$minor )
{
if ( v1$minor < v2$minor )
return -1;
if ( v1$minor > v2$minor )
return 1;
}
else
{
if ( !v1?$minor && !v2?$minor )
return 0;
else
return -1;
}
if ( v1?$minor2 && v2?$minor2 )
{
if ( v1$minor2 < v2$minor2 )
return -1;
if ( v1$minor2 > v2$minor2 )
return 1;
}
else
{
if ( !v1?$minor2 && !v2?$minor2 )
{ return 0; }
else
{print "super WTF!"; return -1; }
}
if ( v1?$addl && v2?$addl )
return strcmp(v1$addl, v2$addl);
else
{
if ( !v1?$minor2 && !v2?$minor2 )
return 0;
else
return -1;
}
}
function software_endpoint_name(c: connection, host: addr): string
@ -225,7 +265,7 @@ event software_register(c: connection, info: Info)
}
}
Log::write("SOFTWARE", info);
Log::write(SOFTWARE, info);
ts[info$name] = info;
}

View file

@ -12,6 +12,8 @@ redef enum Notice::Type += {
SSH_Bytecount_Inconsistency,
};
redef enum Log::ID += { SSH };
redef enum Software::Type += {
SSH_SERVER,
SSH_CLIENT,
@ -22,8 +24,6 @@ redef capture_filters += { ["ssh"] = "tcp port 22" };
redef dpd_config += { [ANALYZER_SSH] = [$ports = set(22/tcp)] };
export {
# Create a new ID for our log stream
redef enum Log::ID += { SSH };
type Log: record {
ts: time;
id: conn_id;
@ -86,6 +86,8 @@ export {
# The list of active SSH connections and the associated session info.
global active_conns: table[conn_id] of Log &read_expire=2mins;
global log_ssh: event(rec: Log);
}
function local_filter(rec: record { id: conn_id; } ): bool
@ -98,9 +100,9 @@ event bro_init()
# Create the stream.
# First argument is the ID for the stream.
# Second argument is the log record type.
Log::create_stream("SSH", "SSH::Log");
Log::create_stream(SSH, [$columns=SSH::Log, $ev=log_ssh]);
# Add a default filter that simply logs everything to "ssh.log" using the default writer.
Log::add_default_filter("SSH");
Log::add_default_filter(SSH);
}
event check_ssh_connection(c: connection, done: bool)
@ -204,7 +206,7 @@ event check_ssh_connection(c: connection, done: bool)
ssh_log$direction = direction;
ssh_log$resp_size = c$resp$size;
Log::write("SSH", ssh_log);
Log::write(SSH, ssh_log);
delete active_conns[c$id];
# Stop watching this connection, we don't care about it anymore.
@ -247,7 +249,7 @@ event ssh_client_version(c: connection, version: string)
# Get rid of the protocol information when passing to the software framework.
local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, "");
local si = Software::default_parse(cleaned_version, c$id$orig_h, SSH_CLIENT);
local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT);
Software::found(c, si);
}
@ -258,7 +260,7 @@ event ssh_server_version(c: connection, version: string)
# Get rid of the protocol information when passing to the software framework.
local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, "");
local si = Software::default_parse(cleaned_version, c$id$resp_h, SSH_SERVER);
local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER);
Software::found(c, si);
}