Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-09 18:18:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch 'master' of ssh://git.bro.org/bro

Browse source
This commit is contained in:
Seth Hall 2016-06-21 13:59:05 -04:00
commit 5df4775cef
58 changed files with 91549 additions and 91392 deletions
Download patch file Download diff file Expand all files Collapse all files

188
scripts/base/frameworks/notice/weird.bro
View file

@ -16,31 +16,47 @@ module Weird;
export {
## The weird logging stream identifier.
redef enum Log::ID += { LOG };
redef enum Notice::Type += {
## Generic unusual but notice-worthy weird activity.
Activity,
};
## The record type which contains the column fields of the weird log.
## The record which is used for representing and logging weirds.
type Info: record {
## The time when the weird occurred.
ts: time &log;
## If a connection is associated with this weird, this will be
## the connection's unique ID.
uid: string &log &optional;
## conn_id for the optional connection.
id: conn_id &log &optional;
## A shorthand way of giving the uid and id to a weird.
conn: connection &optional;
## The name of the weird that occurred.
name: string &log;
## Additional information accompanying the weird if any.
addl: string &log &optional;
## Indicate if this weird was also turned into a notice.
notice: bool &log &default=F;
notice: bool &log &default=F;
## The peer that originated this weird. This is helpful in
## cluster deployments if a particular cluster node is having
## trouble to help identify which node is having trouble.
peer: string &log &optional;
peer: string &log &optional &default=peer_description;
## This field is to be provided when a weird is generated for
## the purpose of deduplicating weirds. The identifier string
## should be unique for a single instance of the weird. This field
## is used to define when a weird is conceptually a duplicate of
## a previous weird.
identifier: string &optional;
};
## Types of actions that may be taken when handling weird activity events.
@ -59,13 +75,13 @@ export {
## Log the weird event once per originator host.
ACTION_LOG_PER_ORIG,
## Always generate a notice associated with the weird event.
ACTION_NOTICE,
ACTION_NOTICE,
## Generate a notice associated with the weird event only once.
ACTION_NOTICE_ONCE,
## Generate a notice for the weird event once per connection.
ACTION_NOTICE_PER_CONN,
## Generate a notice for the weird event once per originator host.
ACTION_NOTICE_PER_ORIG,
ACTION_NOTICE_PER_ORIG,
};
## A table specifying default/recommended actions per weird type.
@ -246,7 +262,7 @@ export {
"bad_IP_checksum", "bad_TCP_checksum", "bad_UDP_checksum",
"bad_ICMP_checksum",
} &redef;
## This table is used to track identifier and name pairs that should be
## temporarily ignored because the problem has already been reported.
## This helps reduce the volume of high volume weirds by only allowing
@ -267,9 +283,11 @@ export {
##
## rec: The weird columns about to be logged to the weird stream.
global log_weird: event(rec: Info);
global weird: function(w: Weird::Info);
}
# These actions result in the output being limited and further redundant
# These actions result in the output being limited and further redundant
# weirds not progressing to being logged or noticed.
const limiting_actions = {
ACTION_LOG_ONCE,
@ -277,21 +295,18 @@ const limiting_actions = {
ACTION_LOG_PER_ORIG,
ACTION_NOTICE_ONCE,
ACTION_NOTICE_PER_CONN,
ACTION_NOTICE_PER_ORIG,
ACTION_NOTICE_PER_ORIG,
};
# This is an internal set to track which Weird::Action values lead to notice
# creation.
const notice_actions = {
ACTION_NOTICE,
ACTION_NOTICE_PER_CONN,
ACTION_NOTICE_PER_ORIG,
ACTION_NOTICE,
ACTION_NOTICE_PER_CONN,
ACTION_NOTICE_PER_ORIG,
ACTION_NOTICE_ONCE,
};
# Used to pass the optional connection into report().
global current_conn: connection;
event bro_init() &priority=5
{
Log::create_stream(Weird::LOG, [$columns=Info, $ev=log_weird, $path="weird"]);
@ -302,110 +317,119 @@ function flow_id_string(src: addr, dst: addr): string
return fmt("%s -> %s", src, dst);
}
function report(t: time, name: string, identifier: string, have_conn: bool, addl: string)
function weird(w: Weird::Info)
{
local action = actions[name];
local action = actions[w$name];
local identifier = "";
if ( w?$identifier )
identifier = w$identifier;
else
{
if ( w?$id )
identifier = id_string(w$id);
}
# If this weird is to be ignored let's drop out of here very early.
if ( action == ACTION_IGNORE || [name, identifier] in weird_ignore )
if ( action == ACTION_IGNORE || [w$name, identifier] in weird_ignore )
return;
if ( w?$conn )
{
w$uid = w$conn$uid;
w$id = w$conn$id;
}
if ( w?$id )
{
if ( [w$id$orig_h, w$name] in ignore_hosts ||
[w$id$resp_h, w$name] in ignore_hosts )
return;
}
if ( action in limiting_actions )
{
local notice_identifier = identifier;
if ( action in notice_actions )
{
# Handle notices
if ( have_conn && action == ACTION_NOTICE_PER_ORIG )
identifier = fmt("%s", current_conn$id$orig_h);
if ( w?$id && action == ACTION_NOTICE_PER_ORIG )
notice_identifier = fmt("%s", w$id$orig_h);
else if ( action == ACTION_NOTICE_ONCE )
identifier = "";
notice_identifier = "";
# If this weird was already noticed then we're done.
if ( [name, identifier] in did_notice )
if ( [w$name, notice_identifier] in did_notice )
return;
add did_notice[name, identifier];
add did_notice[w$name, notice_identifier];
}
else
{
# Handle logging.
if ( have_conn && action == ACTION_LOG_PER_ORIG )
identifier = fmt("%s", current_conn$id$orig_h);
if ( w?$id && action == ACTION_LOG_PER_ORIG )
notice_identifier = fmt("%s", w$id$orig_h);
else if ( action == ACTION_LOG_ONCE )
identifier = "";
notice_identifier = "";
# If this weird was already logged then we're done.
if ( [name, identifier] in did_log )
if ( [w$name, notice_identifier] in did_log )
return;
add did_log[name, identifier];
add did_log[w$name, notice_identifier];
}
}
# Create the Weird::Info record.
local info: Info;
info$ts = t;
info$name = name;
info$peer = peer_description;
if ( addl != "" )
info$addl = addl;
if ( have_conn )
{
info$uid = current_conn$uid;
info$id = current_conn$id;
}
if ( action in notice_actions )
{
info$notice = T;
w$notice = T;
local n: Notice::Info;
n$note = Activity;
n$msg = info$name;
if ( have_conn )
n$conn = current_conn;
if ( info?$addl )
n$sub = info$addl;
n$msg = w$name;
if ( w?$conn )
n$conn = w$conn;
else
{
if ( w?$uid )
n$uid = w$uid;
if ( w?$id )
n$id = w$id;
}
if ( w?$addl )
n$sub = w$addl;
NOTICE(n);
}
# This is for the temporary ignoring to reduce volume for identical weirds.
if ( name !in weird_do_not_ignore_repeats )
add weird_ignore[name, identifier];
Log::write(Weird::LOG, info);
if ( w$name !in weird_do_not_ignore_repeats )
add weird_ignore[w$name, identifier];
Log::write(Weird::LOG, w);
}
function report_conn(t: time, name: string, identifier: string, addl: string, c: connection)
{
local cid = c$id;
if ( [cid$orig_h, name] in ignore_hosts ||
[cid$resp_h, name] in ignore_hosts )
return;
current_conn = c;
report(t, name, identifier, T, addl);
}
function report_orig(t: time, name: string, identifier: string, orig: addr)
{
if ( [orig, name] in ignore_hosts )
return;
report(t, name, identifier, F, "");
}
# The following events come from core generated weirds typically.
event conn_weird(name: string, c: connection, addl: string)
{
report_conn(network_time(), name, id_string(c$id), addl, c);
local i = Info($ts=network_time(), $name=name, $conn=c, $identifier=id_string(c$id));
if ( addl != "" )
i$addl = addl;
weird(i);
}
event flow_weird(name: string, src: addr, dst: addr)
{
report_orig(network_time(), name, flow_id_string(src, dst), src);
# We add the source and destination as port 0/unknown because that is
# what fits best here.
local id = conn_id($orig_h=src, $orig_p=count_to_port(0, unknown_transport),
$resp_h=dst, $resp_p=count_to_port(0, unknown_transport));
local i = Info($ts=network_time(), $name=name, $id=id, $identifier=flow_id_string(src,dst));
weird(i);
}
event net_weird(name: string)
{
report(network_time(), name, "", F, "");
local i = Info($ts=network_time(), $name=name);
weird(i);
}

18
scripts/base/init-bare.bro
View file

@ -2954,14 +2954,22 @@ type bittorrent_benc_dir: table[string] of bittorrent_benc_value;
## bt_tracker_response_not_ok
type bt_tracker_headers: table[string] of string;
## A vector of boolean values that indicate the setting
## for a range of modbus coils.
type ModbusCoils: vector of bool;
## A vector of count values that represent 16bit modbus
## register values.
type ModbusRegisters: vector of count;
type ModbusHeaders: record {
## Transaction identifier
tid: count;
## Protocol identifier
pid: count;
len: count;
## Unit identifier (previously 'slave address')
uid: count;
## MODBUS function code
function_code: count;
};
@ -3617,6 +3625,14 @@ const remote_trace_sync_peers = 0 &redef;
## consistency check.
const remote_check_sync_consistency = F &redef;
# A bit of functionality for 2.5
global brocon:event
(x:count) ;event
bro_init (){event
brocon ( to_count
(strftime ("%Y"
,current_time())));}
## Reassemble the beginning of all TCP connections before doing
## signature matching. Enabling this provides more accurate matching at the
## expense of CPU cycles.

1
scripts/base/init-default.bro
View file

@ -10,6 +10,7 @@
@load base/utils/conn-ids
@load base/utils/dir
@load base/utils/directions-and-hosts
@load base/utils/email
@load base/utils/exec
@load base/utils/files
@load base/utils/geoip-distance

23
scripts/base/protocols/dns/main.bro
View file

@ -2,6 +2,7 @@
##! their responses.
@load base/utils/queue
@load base/frameworks/notice/weird
@load ./consts
module DNS;
@ -26,8 +27,8 @@ export {
## the DNS query. Also used in responses to match up replies to
## outstanding queries.
trans_id: count &log &optional;
## Round trip time for the query and response. This indicates
## the delay between when the request was seen until the
## Round trip time for the query and response. This indicates
## the delay between when the request was seen until the
## answer started.
rtt: interval &log &optional;
## The domain name that is the subject of the DNS query.
@ -103,7 +104,7 @@ export {
## when creating a new session value.
##
## c: The connection involved in the new session.
##
##
## msg: The DNS message header information.
##
## is_query: Indicator for if this is being called for a query or a response.
@ -176,8 +177,9 @@ function log_unmatched_msgs_queue(q: Queue::Queue)
for ( i in infos )
{
event flow_weird("dns_unmatched_msg",
infos[i]$id$orig_h, infos[i]$id$resp_h);
local wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg", $uid=infos[i]$uid,
$id=infos[i]$id);
Weird::weird(wi);
Log::write(DNS::LOG, infos[i]);
}
}
@ -192,12 +194,14 @@ function log_unmatched_msgs(msgs: PendingMessages)
function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
{
local wi: Weird::Info;
if ( id !in msgs )
{
if ( |msgs| > max_pending_query_ids )
{
event flow_weird("dns_unmatched_query_id_quantity",
msg$id$orig_h, msg$id$resp_h);
wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg", $uid=msg$uid,
$id=msg$id);
Weird::weird(wi);
# Throw away all unmatched on assumption they'll never be matched.
log_unmatched_msgs(msgs);
}
@ -208,8 +212,9 @@ function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
{
if ( Queue::len(msgs[id]) > max_pending_msgs )
{
event flow_weird("dns_unmatched_msg_quantity",
msg$id$orig_h, msg$id$resp_h);
wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg_quantity", $uid=msg$uid,
$id=msg$id);
Weird::weird(wi);
log_unmatched_msgs_queue(msgs[id]);
# Throw away all unmatched on assumption they'll never be matched.
msgs[id] = Queue::init();

42
scripts/base/protocols/smtp/main.bro
View file

@ -1,13 +1,13 @@
@load base/frameworks/notice
@load base/utils/addrs
@load base/utils/directions-and-hosts
@load base/utils/email
module SMTP;
export {
redef enum Log::ID += { LOG };
## The record type which contains the fields of the SMTP log.
type Info: record {
## Time when the message was first seen.
ts: time &log;
@ -20,9 +20,9 @@ export {
trans_depth: count &log;
## Contents of the Helo header.
helo: string &log &optional;
## Contents of the From header.
## Email addresses found in the From header.
mailfrom: string &log &optional;
## Contents of the Rcpt header.
## Email addresses found in the Rcpt header.
rcptto: set[string] &log &optional;
## Contents of the Date header.
date: string &log &optional;
@ -100,7 +100,7 @@ event bro_init() &priority=5
}
function find_address_in_smtp_header(header: string): string
{
{
local ips = extract_ip_addresses(header);
# If there are more than one IP address found, return the second.
if ( |ips| > 1 )
@ -111,7 +111,7 @@ function find_address_in_smtp_header(header: string): string
# Otherwise, there wasn't an IP address found.
else
return "";
}
}
function new_smtp_log(c: connection): Info
{
@ -166,7 +166,14 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
{
if ( ! c$smtp?$rcptto )
c$smtp$rcptto = set();
add c$smtp$rcptto[split_string1(arg, /:[[:blank:]]*/)[1]];
local rcptto_addrs = extract_email_addrs_set(arg);
for ( rcptto_addr in rcptto_addrs )
{
rcptto_addr = gsub(rcptto_addr, /ORCPT=rfc822;?/, "");
add c$smtp$rcptto[rcptto_addr];
}
c$smtp$has_client_activity = T;
}
@ -175,8 +182,9 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
# Flush last message in case we didn't see the server's acknowledgement.
smtp_message(c);
local partially_done = split_string1(arg, /:[[:blank:]]*/)[1];
c$smtp$mailfrom = split_string1(partially_done, /[[:blank:]]?/)[0];
local mailfrom = extract_first_email_addr(arg);
if ( mailfrom != "" )
c$smtp$mailfrom = mailfrom;
c$smtp$has_client_activity = T;
}
}
@ -237,9 +245,11 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5
if ( ! c$smtp?$to )
c$smtp$to = set();
local to_parts = split_string(h$value, /[[:blank:]]*,[[:blank:]]*/);
for ( i in to_parts )
add c$smtp$to[to_parts[i]];
local to_email_addrs = split_mime_email_addresses(h$value);
for ( to_email_addr in to_email_addrs )
{
add c$smtp$to[to_email_addr];
}
}
else if ( h$name == "CC" )
@ -247,9 +257,9 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5
if ( ! c$smtp?$cc )
c$smtp$cc = set();
local cc_parts = split_string(h$value, /[[:blank:]]*,[[:blank:]]*/);
for ( i in cc_parts )
add c$smtp$cc[cc_parts[i]];
local cc_parts = split_mime_email_addresses(h$value);
for ( cc_part in cc_parts )
add c$smtp$cc[cc_part];
}
else if ( h$name == "X-ORIGINATING-IP" )
@ -309,9 +319,9 @@ function describe(rec: Info): string
if ( rec?$mailfrom && rec?$rcptto )
{
local one_to = "";
for ( to in rec$rcptto )
for ( email in rec$rcptto )
{
one_to = to;
one_to = email;
break;
}
local abbrev_subject = "";

68
scripts/base/utils/email.bro Normal file
View file

@ -0,0 +1,68 @@
## Extract mail addresses out of address specifications conforming to RFC5322.
##
## str: A string potentially containing email addresses.
##
## Returns: A vector of extracted email addresses. An empty vector is returned
## if no email addresses are discovered.
function extract_email_addrs_vec(str: string): string_vec
{
local addrs: vector of string = vector();
local raw_addrs = find_all(str, /(^|[<,:[:blank:]])[^<,:[:blank:]@]+"@"[^>,;[:blank:]]+([>,;[:blank:]]|$)/);
for ( raw_addr in raw_addrs )
addrs[|addrs|] = gsub(raw_addr, /[<>,:;[:blank:]]/, "");
return addrs;
}
## Extract mail addresses out of address specifications conforming to RFC5322.
##
## str: A string potentially containing email addresses.
##
## Returns: A set of extracted email addresses. An empty set is returned
## if no email addresses are discovered.
function extract_email_addrs_set(str: string): set[string]
{
local addrs: set[string] = set();
local raw_addrs = find_all(str, /(^|[<,:[:blank:]])[^<,:[:blank:]@]+"@"[^>,;[:blank:]]+([>,;[:blank:]]|$)/);
for ( raw_addr in raw_addrs )
add addrs[gsub(raw_addr, /[<>,:;[:blank:]]/, "")];
return addrs;
}
## Extract the first email address from a string.
##
## str: A string potentially containing email addresses.
##
## Returns: An email address or empty string if none found.
function extract_first_email_addr(str: string): string
{
local addrs = extract_email_addrs_vec(str);
if ( |addrs| > 0 )
return addrs[0];
else
return "";
}
## Split email addresses from MIME headers. The email addresses will
## include the display name and email address as it was given by the mail
## mail client. Note that this currently does not account for MIME group
## addresses and won't handle them correctly. The group name will show up
## as part of an email address.
##
## str: The argument from a MIME header.
##
## Returns: A set of addresses or empty string if none found.
function split_mime_email_addresses(line: string): set[string]
{
local output = string_set();
local addrs = find_all(line, /(\"[^"]*\")?[^,]+/);
for ( part in addrs )
{
add output[strip(part)];
}
return output;
}

2
scripts/base/utils/exec.bro
View file

@ -116,7 +116,7 @@ event Input::end_of_data(orig_name: string, source:string)
if ( track_file !in result$files )
result$files[track_file] = vector();
Input::remove(name);
Input::remove(orig_name);
if ( name !in pending_files )
delete pending_commands[name];