Logging framework update and mass Log::ID renaming.

- Log path's are generated in the scripting land
  now.  The default Log stream ID to path string
  mapping works like this:
    - Notice::LOG -> "notice"
    - Notice::POLICY_LOG -> "notice_policy"
    - TestModule::LOG -> "test_module"

- Logging streams updated across all of the shipped
  scripts to be more user friendly.  Instead of
  the logging stream ID HTTP::HTTP, we now have
  HTTP::LOG, etc.

- The priorities on some bro_init handlers have
  been adjusted to make the process of applying
  filters or disabling streams easier for users.
This commit is contained in:
Seth Hall 2011-09-03 01:10:17 -04:00
parent fe53091cd1
commit 11c437faa3
77 changed files with 391 additions and 619 deletions

View file

@ -3,7 +3,7 @@
module Conn;
export {
redef enum Log::ID += { CONN };
redef enum Log::ID += { LOG };
type Info: record {
## This is the time of the first packet.
@ -95,7 +95,7 @@ redef record connection += {
event bro_init() &priority=5
{
Log::create_stream(CONN, [$columns=Info, $ev=log_conn]);
Log::create_stream(Conn::LOG, [$columns=Info, $ev=log_conn]);
}
function conn_state(c: connection, trans: transport_proto): string
@ -216,6 +216,6 @@ event connection_state_remove(c: connection) &priority=5
event connection_state_remove(c: connection) &priority=-5
{
Log::write(CONN, c$conn);
Log::write(Conn::LOG, c$conn);
}

View file

@ -3,7 +3,7 @@
module DNS;
export {
redef enum Log::ID += { DNS };
redef enum Log::ID += { LOG };
type Info: record {
ts: time &log;
@ -77,7 +77,7 @@ redef dpd_config += { [ANALYZER_DNS_TCP_BINPAC] = [$ports = dns_tcp_ports] };
event bro_init() &priority=5
{
Log::create_stream(DNS, [$columns=Info, $ev=log_dns]);
Log::create_stream(DNS::LOG, [$columns=Info, $ev=log_dns]);
}
function new_session(c: connection, trans_id: count): Info
@ -163,7 +163,7 @@ event do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &pri
{
if ( c$dns$ready )
{
Log::write(DNS, c$dns);
Log::write(DNS::LOG, c$dns);
add c$dns_state$finished_answers[c$dns$trans_id];
# This record is logged and no longer pending.
delete c$dns_state$pending[c$dns$trans_id];
@ -275,6 +275,6 @@ event connection_state_remove(c: connection) &priority=-5
# If Bro is expiring state, we should go ahead and log all unlogged
# request/response pairs now.
for ( trans_id in c$dns_state$pending )
Log::write(DNS, c$dns_state$pending[trans_id]);
Log::write(DNS::LOG, c$dns_state$pending[trans_id]);
}

View file

@ -14,7 +14,7 @@
module FTP;
export {
redef enum Log::ID += { FTP };
redef enum Log::ID += { LOG };
## This setting changes if passwords used in FTP sessions are captured or not.
const default_capture_password = F &redef;
@ -95,7 +95,7 @@ global ftp_data_expected: table[addr, port] of ExpectedConn &create_expire=5mins
event bro_init() &priority=5
{
Log::create_stream(FTP, [$columns=Info, $ev=log_ftp]);
Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]);
}
## A set of commands where the argument can be expected to refer
@ -165,7 +165,7 @@ function ftp_message(s: Info)
else
s$arg=arg;
Log::write(FTP, s);
Log::write(FTP::LOG, s);
}
# The MIME and file_size fields are specific to file transfer commands

View file

@ -4,7 +4,7 @@
module HTTP;
export {
redef enum Log::ID += { HTTP };
redef enum Log::ID += { LOG };
## Indicate a type of attack or compromise in the record to be logged.
type Tags: enum {
@ -86,7 +86,7 @@ redef record connection += {
# Initialize the HTTP logging stream.
event bro_init() &priority=5
{
Log::create_stream(HTTP, [$columns=Info, $ev=log_http]);
Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http]);
}
# DPD configuration.
@ -230,7 +230,7 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
# The reply body is done so we're ready to log.
if ( ! is_orig )
{
Log::write(HTTP, c$http);
Log::write(HTTP::LOG, c$http);
delete c$http_state$pending[c$http_state$current_response];
}
}
@ -242,7 +242,7 @@ event connection_state_remove(c: connection)
{
for ( r in c$http_state$pending )
{
Log::write(HTTP, c$http_state$pending[r]);
Log::write(HTTP::LOG, c$http_state$pending[r]);
}
}
}

View file

@ -73,7 +73,7 @@ event file_transferred(c: connection, prefix: string, descr: string,
local tmp = irc$command;
irc$command = "DCC";
Log::write(IRC, irc);
Log::write(IRC::LOG, irc);
irc$command = tmp;
if ( irc$extract_file && irc?$extraction_file )

View file

@ -7,7 +7,7 @@
module IRC;
export {
redef enum Log::ID += { IRC };
redef enum Log::ID += { LOG };
type Tag: enum {
EMPTY
@ -44,7 +44,7 @@ redef dpd_config += { [ANALYZER_IRC] = [$ports = irc_ports] };
event bro_init() &priority=5
{
Log::create_stream(IRC, [$columns=Info, $ev=irc_log]);
Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log]);
}
function new_session(c: connection): Info
@ -78,7 +78,7 @@ event irc_nick_message(c: connection, is_orig: bool, who: string, newnick: strin
{
if ( is_orig )
{
Log::write(IRC, c$irc);
Log::write(IRC::LOG, c$irc);
c$irc$nick = newnick;
}
}
@ -98,7 +98,7 @@ event irc_user_message(c: connection, is_orig: bool, user: string, host: string,
{
if ( is_orig )
{
Log::write(IRC, c$irc);
Log::write(IRC::LOG, c$irc);
c$irc$user = user;
}
}
@ -118,7 +118,7 @@ event irc_join_message(c: connection, is_orig: bool, info_list: irc_join_list) &
{
c$irc$value = l$channel;
c$irc$addl = (l$password != "" ? fmt(" with channel key: '%s'", l$password) : "");
Log::write(IRC, c$irc);
Log::write(IRC::LOG, c$irc);
}
}
}

View file

@ -1,4 +0,0 @@
@load ./main
@load ./file-ident
@load ./file-extract
@load ./file-hash

View file

@ -1,62 +0,0 @@
@load ./file-ident
@load base/utils/files
module MIME;
export {
## Pattern of file mime types to extract from MIME bodies.
const extract_file_types = /NO_DEFAULT/ &redef;
## The on-disk prefix for files to be extracted from MIME entity bodies.
const extraction_prefix = "mime-item" &redef;
redef record Info += {
## Optionally write the file to disk. Must be set prior to first
## data chunk being seen in an event.
extract_file: bool &default=F;
## Store the file handle here for the file currently being extracted.
extraction_file: file &log &optional;
};
redef record State += {
## Store a count of the number of files that have been transferred in
## this conversation to create unique file names on disk.
num_extracted_files: count &default=0;
};
}
event mime_segment_data(c: connection, length: count, data: string) &priority=5
{
if ( extract_file_types in c$mime$mime_type )
c$mime$extract_file = T;
}
event mime_segment_data(c: connection, length: count, data: string) &priority=3
{
if ( c$mime$extract_file && c$mime$content_len == 0 )
{
local suffix = fmt("%d.dat", ++c$mime_state$num_extracted_files);
local fname = generate_extraction_filename(extraction_prefix, c, suffix);
c$mime$extraction_file = open(fname);
enable_raw_output(c$mime$extraction_file);
}
}
event mime_segment_data(c: connection, length: count, data: string) &priority=-5
{
if ( c$mime$extract_file && c$mime?$extraction_file )
print c$mime$extraction_file, data;
}
event mime_end_entity(c: connection) &priority=-3
{
# TODO: this check is only due to a bug in mime_end_entity that
# causes the event to be generated twice for the same real event.
if ( ! c?$mime )
return;
if ( c$mime?$extraction_file )
close(c$mime$extraction_file);
}

View file

@ -1,79 +0,0 @@
@load ./file-ident
@load base/frameworks/notice
module MIME;
export {
redef enum Notice::Type += {
## Indicates that an MD5 sum was calculated for a MIME message.
MD5,
};
redef record Info += {
## The calculated MD5 sum for the MIME entity.
md5: string &log &optional;
## Optionally calculate the file's MD5 sum. Must be set prior to the
## first data chunk being see in an event.
calc_md5: bool &default=F;
## This boolean value indicates if an MD5 sum is being calculated
## for the current file transfer.
calculating_md5: bool &default=F;
};
## Generate MD5 sums for these filetypes.
const generate_md5 = /application\/x-dosexec/ # Windows and DOS executables
| /application\/x-executable/ # *NIX executable binary
&redef;
}
event mime_segment_data(c: connection, length: count, data: string) &priority=-5
{
if ( ! c?$mime ) return;
if ( c$mime$content_len == 0 )
{
if ( generate_md5 in c$mime$mime_type )
c$mime$calc_md5 = T;
if ( c$mime$calc_md5 )
{
c$mime$calculating_md5 = T;
md5_hash_init(c$id);
}
}
if ( c$mime$calculating_md5 )
md5_hash_update(c$id, data);
}
## In the event of a content gap during the MIME transfer, detect the state for
## the MD5 sum calculation and stop calculating the MD5 since it would be
## incorrect anyway.
event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5
{
if ( is_orig || ! c?$mime ) return;
if ( c$mime$calculating_md5 )
{
c$mime$calculating_md5 = F;
md5_hash_finish(c$id);
}
}
event mime_end_entity(c: connection) &priority=-3
{
# TODO: this check is only due to a bug in mime_end_entity that
# causes the event to be generated twice for the same real event.
if ( ! c?$mime )
return;
if ( c$mime$calculating_md5 )
{
c$mime$md5 = md5_hash_finish(c$id);
NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h),
$sub=c$mime$md5, $conn=c]);
}
}

View file

@ -1,16 +0,0 @@
@load ./main
module MIME;
export {
redef record Info += {
## Sniffed MIME type for the transfer.
mime_type: string &log &optional;
};
}
event mime_segment_data(c: connection, length: count, data: string) &priority=7
{
if ( c$mime$content_len == 0 )
c$mime$mime_type = split1(identify_data(data, T), /;/)[1];
}

View file

@ -1,101 +0,0 @@
##! The mime script does analysis of MIME encoded messages seen in certain
##! protocols (only SMTP and POP3 at the moment).
@load base/utils/strings
module MIME;
export {
redef enum Log::ID += { MIME };
# Let's assume for now that nothing transferring files using
# MIME attachments is multiplexing for simplicity's sake.
# We can make the assumption that one connection == one file (at a time)
type Info: record {
## This is the timestamp of when the MIME content transfer began.
ts: time &log;
uid: string &log;
id: conn_id &log;
## The application layer protocol over which the transfer was seen.
app_protocol: string &log &optional;
## The filename seen in the Content-Disposition header.
filename: string &log &optional;
## Track how many byte of the MIME encoded file have been seen.
content_len: count &log &default=0;
};
type State: record {
## Track the number of MIME encoded files transferred during this session.
level: count &default=0;
};
global log_mime: event(rec: Info);
}
redef record connection += {
mime: Info &optional;
mime_state: State &optional;
};
event bro_init()
{
Log::create_stream(MIME, [$columns=Info, $ev=log_mime]);
}
function new_mime_session(c: connection): Info
{
local info: Info;
info$ts=network_time();
info$uid=c$uid;
info$id=c$id;
return info;
}
function set_session(c: connection, new_entity: bool)
{
if ( ! c?$mime_state )
c$mime_state = [];
if ( ! c?$mime || new_entity )
c$mime = new_mime_session(c);
}
event mime_begin_entity(c: connection) &priority=10
{
set_session(c, T);
++c$mime_state$level;
if ( |c$service| > 0 )
c$mime$app_protocol = join_string_set(c$service, ",");
}
# This has priority -10 because other handlers need to know the current
# content_len before it's updated by this handler.
event mime_segment_data(c: connection, length: count, data: string) &priority=-10
{
c$mime$content_len = c$mime$content_len + length;
}
event mime_one_header(c: connection, h: mime_header_rec)
{
if ( h$name == "CONTENT-DISPOSITION" &&
/[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value )
c$mime$filename = sub(h$value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
}
event mime_end_entity(c: connection) &priority=-5
{
# This check and the delete below are just to cope with a bug where
# mime_end_entity can be generated multiple times for the same event.
if ( ! c?$mime )
return;
# Don't log anything if there wasn't any content.
if ( c$mime$content_len > 0 )
Log::write(MIME, c$mime);
delete c$mime;
}

View file

@ -12,7 +12,7 @@ export {
MD5,
};
redef enum Log::ID += { SMTP_ENTITIES };
redef enum Log::ID += { ENTITIES_LOG };
type EntityInfo: record {
## This is the timestamp of when the MIME content transfer began.
@ -74,7 +74,7 @@ export {
event bro_init() &priority=5
{
Log::create_stream(SMTP_ENTITIES, [$columns=EntityInfo, $ev=log_mime]);
Log::create_stream(SMTP::ENTITIES_LOG, [$columns=EntityInfo, $ev=log_mime]);
}
function set_session(c: connection, new_entity: bool)
@ -185,7 +185,7 @@ event mime_end_entity(c: connection) &priority=-5
# Only log is there was some content.
if ( c$smtp$current_entity$content_len > 0 )
Log::write(SMTP_ENTITIES, c$smtp$current_entity);
Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity);
delete c$smtp$current_entity;
}

View file

@ -5,7 +5,7 @@
module SMTP;
export {
redef enum Log::ID += { SMTP };
redef enum Log::ID += { LOG };
type Info: record {
ts: time &log;
@ -73,7 +73,7 @@ redef dpd_config += { [ANALYZER_SMTP] = [$ports = ports] };
event bro_init() &priority=5
{
Log::create_stream(SMTP, [$columns=SMTP::Info, $ev=log_smtp]);
Log::create_stream(SMTP::LOG, [$columns=SMTP::Info, $ev=log_smtp]);
}
function find_address_in_smtp_header(header: string): string
@ -119,7 +119,7 @@ function set_smtp_session(c: connection)
function smtp_message(c: connection)
{
if ( c$smtp$has_client_activity )
Log::write(SMTP, c$smtp);
Log::write(SMTP::LOG, c$smtp);
}
event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5

View file

@ -14,7 +14,7 @@
module SSH;
export {
redef enum Log::ID += { SSH };
redef enum Log::ID += { LOG };
redef enum Notice::Type += {
## This indicates that a heuristically detected "successful" SSH
@ -79,7 +79,7 @@ redef record connection += {
event bro_init() &priority=5
{
Log::create_stream(SSH, [$columns=Info, $ev=log_ssh]);
Log::create_stream(SSH::LOG, [$columns=Info, $ev=log_ssh]);
}
function set_session(c: connection)
@ -149,11 +149,11 @@ event SSH::heuristic_successful_login(c: connection) &priority=-5
$msg="Heuristically detected successful SSH login.",
$conn=c]);
Log::write(SSH, c$ssh);
Log::write(SSH::LOG, c$ssh);
}
event SSH::heuristic_failed_login(c: connection) &priority=-5
{
Log::write(SSH, c$ssh);
Log::write(SSH::LOG, c$ssh);
}
event connection_state_remove(c: connection) &priority=-5

View file

@ -4,7 +4,7 @@
module SSL;
export {
redef enum Log::ID += { SSL };
redef enum Log::ID += { LOG };
redef enum Notice::Type += {
Self_Signed_Cert
@ -43,7 +43,7 @@ redef record connection += {
event bro_init() &priority=5
{
Log::create_stream(SSL, [$columns=Info, $ev=log_ssl]);
Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
}
redef capture_filters += {
@ -117,6 +117,6 @@ event ssl_established(c: connection) &priority=-5
{
set_session(c);
Log::write(SSL, c$ssl);
Log::write(SSL::LOG, c$ssl);
}

View file

@ -5,7 +5,7 @@
module Syslog;
export {
redef enum Log::ID += { SYSLOG };
redef enum Log::ID += { LOG };
type Info: record {
ts: time &log;
@ -29,7 +29,7 @@ redef record connection += {
event bro_init() &priority=5
{
Log::create_stream(SYSLOG, [$columns=Info]);
Log::create_stream(Syslog::LOG, [$columns=Info]);
}
event syslog_message(c: connection, facility: count, severity: count, msg: string) &priority=5
@ -48,5 +48,5 @@ event syslog_message(c: connection, facility: count, severity: count, msg: strin
event syslog_message(c: connection, facility: count, severity: count, msg: string) &priority=-5
{
Log::write(SYSLOG, c$syslog);
Log::write(Syslog::LOG, c$syslog);
}