Merge remote-tracking branch 'origin/master' into topic/vladg/radius

This commit is contained in:
Vlad Grigorescu 2014-02-11 16:17:40 -05:00
commit 80ec40bc93
169 changed files with 3244 additions and 517 deletions

View file

@ -7,10 +7,10 @@ module Unified2;
export {
redef enum Log::ID += { LOG };
## Directory to watch for Unified2 files.
## File to watch for Unified2 files.
const watch_file = "" &redef;
## File to watch for Unified2 records.
## Directory to watch for Unified2 records.
const watch_dir = "" &redef;
## The sid-msg.map file you would like to use for your alerts.

View file

@ -23,7 +23,8 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
@if ( Cluster::local_node_type() != Cluster::MANAGER )
event Notice::begin_suppression(n: Notice::Info)
{
suppressing[n$note, n$identifier] = n;
local suppress_until = n$ts + n$suppress_for;
suppressing[n$note, n$identifier] = suppress_until;
}
@endif

View file

@ -242,12 +242,6 @@ export {
## being suppressed.
global suppressed: event(n: Notice::Info);
## This event is generated when a notice stops being suppressed.
##
## n: The record containing notice data regarding the notice type
## that was being suppressed.
global end_suppression: event(n: Notice::Info);
## Call this function to send a notice in an email. It is already used
## by default with the built in :bro:enum:`Notice::ACTION_EMAIL` and
## :bro:enum:`Notice::ACTION_PAGE` actions.
@ -285,27 +279,22 @@ export {
}
# This is used as a hack to implement per-item expiration intervals.
function per_notice_suppression_interval(t: table[Notice::Type, string] of Notice::Info, idx: any): interval
function per_notice_suppression_interval(t: table[Notice::Type, string] of time, idx: any): interval
{
local n: Notice::Type;
local s: string;
[n,s] = idx;
local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts);
local suppress_time = t[n,s] - network_time();
if ( suppress_time < 0secs )
suppress_time = 0secs;
# If there is no more suppression time left, the notice needs to be sent
# to the end_suppression event.
if ( suppress_time == 0secs )
event Notice::end_suppression(t[n,s]);
return suppress_time;
}
# This is the internally maintained notice suppression table. It's
# indexed on the Notice::Type and the $identifier field from the notice.
global suppressing: table[Type, string] of Notice::Info = {}
global suppressing: table[Type, string] of time = {}
&create_expire=0secs
&expire_func=per_notice_suppression_interval;
@ -400,11 +389,22 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)
# First off, finish the headers and include the human readable messages
# then leave a blank line after the message.
email_text = string_cat(email_text, "\nMessage: ", n$msg);
if ( n?$sub )
email_text = string_cat(email_text, "\nSub-message: ", n$sub);
email_text = string_cat(email_text, "\nMessage: ", n$msg, "\n");
email_text = string_cat(email_text, "\n\n");
if ( n?$sub )
email_text = string_cat(email_text, "Sub-message: ", n$sub, "\n");
email_text = string_cat(email_text, "\n");
# Add information about the file if it exists.
if ( n?$file_desc )
email_text = string_cat(email_text, "File Description: ", n$file_desc, "\n");
if ( n?$file_mime_type )
email_text = string_cat(email_text, "File MIME Type: ", n$file_mime_type, "\n");
if ( n?$file_desc || n?$file_mime_type )
email_text = string_cat(email_text, "\n");
# Next, add information about the connection if it exists.
if ( n?$id )
@ -467,7 +467,8 @@ hook Notice::notice(n: Notice::Info) &priority=-5
[n$note, n$identifier] !in suppressing &&
n$suppress_for != 0secs )
{
suppressing[n$note, n$identifier] = n;
local suppress_until = n$ts + n$suppress_for;
suppressing[n$note, n$identifier] = suppress_until;
event Notice::begin_suppression(n);
}
}

View file

@ -2432,6 +2432,17 @@ type X509: record {
not_valid_after: time; ##< Timestamp after when certificate is not valid.
};
## An X509 extension.
##
## .. bro:see:: x509_extension
type X509_extension_info: record {
name: string; ##< Long name of extension; oid if name not known.
short_name: string &optional; ##< Short name of extension if known.
oid: string; ##< Oid of extension.
critical: bool; ##< True if extension is critical.
value: string; ##< Extension content parsed to string for known extensions. Raw data otherwise.
};
## HTTP session statistics.
##
## .. bro:see:: http_stats
@ -2866,6 +2877,12 @@ global load_sample_freq = 20 &redef;
## .. bro:see:: gap_report
const gap_report_freq = 1.0 sec &redef;
## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace
## and not report missing segments for such connections.
## If this is enabled, then missing data at the end of connections may not
## be reported via :bro:see:`content_gap`.
const detect_filtered_trace = F &redef;
## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial
## connections. A connection is partial if it is missing a full handshake. Note
## that gap reports for partial connections might not be reliable.
@ -3074,6 +3091,9 @@ export {
## Toggle whether to do GTPv1 decapsulation.
const enable_gtpv1 = T &redef;
## Toggle whether to do GRE decapsulation.
const enable_gre = T &redef;
## With this option set, the Teredo analysis will first check to see if
## other protocol analyzers have confirmed that they think they're
## parsing the right protocol and only continue with Teredo tunnel
@ -3099,7 +3119,8 @@ export {
## may work better.
const delay_gtp_confirmation = F &redef;
## How often to cleanup internal state for inactive IP tunnels.
## How often to cleanup internal state for inactive IP tunnels
## (includes GRE tunnels).
const ip_tunnel_timeout = 24hrs &redef;
} # end export
module GLOBAL;

View file

@ -61,3 +61,4 @@
@load base/misc/find-checksum-offloading
@load base/misc/find-filtered-trace

View file

@ -0,0 +1,49 @@
##! Discovers trace files that contain TCP traffic consisting only of
##! control packets (e.g. it's been filtered to contain only SYN/FIN/RST
##! packets and no content). On finding such a trace, a warning is
##! emitted that suggests toggling the :bro:see:`detect_filtered_trace`
##! option may be desired if the user does not want Bro to report
##! missing TCP segments.
module FilteredTraceDetection;
export {
## Flag to enable filtered trace file detection and warning message.
global enable: bool = T &redef;
}
global saw_tcp_conn_with_data: bool = F;
global saw_a_tcp_conn: bool = F;
event connection_state_remove(c: connection)
{
if ( ! reading_traces() )
return;
if ( ! enable )
return;
if ( saw_tcp_conn_with_data )
return;
if ( ! is_tcp_port(c$id$orig_p) )
return;
saw_a_tcp_conn = T;
if ( /[Dd]/ in c$history )
saw_tcp_conn_with_data = T;
}
event bro_done()
{
if ( ! enable )
return;
if ( ! saw_a_tcp_conn )
return;
if ( ! saw_tcp_conn_with_data )
Reporter::warning("The analyzed trace file was determined to contain only TCP control packets, which may indicate it's been pre-filtered. By default, Bro reports the missing segments for this type of trace, but the 'detect_filtered_trace' option may be toggled if that's not desired.");
}

View file

@ -63,15 +63,17 @@ export {
## The DNS query was rejected by the server.
rejected: bool &log &default=F;
## This value indicates if this request/response pair is ready
## to be logged.
ready: bool &default=F;
## The total number of resource records in a reply message's
## answer section.
total_answers: count &optional;
## The total number of resource records in a reply message's
## answer, authority, and additional sections.
total_replies: count &optional;
## Whether the full DNS query has been seen.
saw_query: bool &default=F;
## Whether the full DNS reply has been seen.
saw_reply: bool &default=F;
};
## An event that can be handled to access the :bro:type:`DNS::Info`
@ -90,7 +92,7 @@ export {
## ans: The general information of a RR response.
##
## reply: The specific response information according to RR type/class.
global do_reply: event(c: connection, msg: dns_msg, ans: dns_answer, reply: string);
global do_reply: hook(c: connection, msg: dns_msg, ans: dns_answer, reply: string);
## A hook that is called whenever a session is being set.
## This can be used if additional initialization logic needs to happen
@ -103,17 +105,42 @@ export {
## is_query: Indicator for if this is being called for a query or a response.
global set_session: hook(c: connection, msg: dns_msg, is_query: bool);
## Yields a queue of :bro:see:`DNS::Info` objects for a given
## DNS message query/transaction ID.
type PendingMessages: table[count] of Queue::Queue;
## Called when a pending DNS query has not been matched with a reply (or
## vice versa) in a sufficent amount of time.
##
## pending: table of pending messages, indexed by transaction ID.
##
## id: the index of he element being expired.
##
## Returns: amount of time to delay expiration of the element.
global expire_pending_msg: function(pending: PendingMessages, id: count): interval;
## The amount of time that DNS queries or replies for a given
## query/transaction ID are allowed to be queued while waiting for
## a matching reply or query.
const pending_msg_expiry_interval = 2min &redef;
## Give up trying to match pending DNS queries or replies for a given
## query/transaction ID once this number of unmatched queries or replies
## is reached (this shouldn't happen unless either the DNS server/resolver
## is broken, Bro is not seeing all the DNS traffic, or an AXFR query
## response is ongoing).
const max_pending_msgs = 50 &redef;
## A record type which tracks the status of DNS queries for a given
## :bro:type:`connection`.
type State: record {
## Indexed by query id, returns Info record corresponding to
## query/response which haven't completed yet.
pending: table[count] of Queue::Queue;
## queries that haven't been matched with a response yet.
pending_queries: PendingMessages &read_expire=pending_msg_expiry_interval &expire_func=expire_pending_msg;
## This is the list of DNS responses that have completed based
## on the number of responses declared and the number received.
## The contents of the set are transaction IDs.
finished_answers: set[count];
## Indexed by query id, returns Info record corresponding to
## replies that haven't been matched with a query yet.
pending_replies: PendingMessages &read_expire=pending_msg_expiry_interval &expire_func=expire_pending_msg;
};
}
@ -143,6 +170,51 @@ function new_session(c: connection, trans_id: count): Info
return info;
}
function log_unmatched_msgs_queue(q: Queue::Queue)
{
local infos: vector of Info;
Queue::get_vector(q, infos);
for ( i in infos )
Log::write(DNS::LOG, infos[i]);
}
function log_unmatched_msgs(msgs: PendingMessages)
{
for ( trans_id in msgs )
{
log_unmatched_msgs_queue(msgs[trans_id]);
delete msgs[trans_id];
}
}
function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
{
if ( id !in msgs )
msgs[id] = Queue::init();
else if ( Queue::len(msgs[id]) > max_pending_msgs )
{
local info: Info = Queue::peek(msgs[id]);
event flow_weird("dns_unmatched_msg_quantity", info$id$orig_h,
info$id$resp_h);
log_unmatched_msgs_queue(msgs[id]);
# Throw away all unmatched on assumption they'll never be matched.
msgs[id] = Queue::init();
}
Queue::put(msgs[id], msg);
}
function pop_msg(msgs: PendingMessages, id: count): Info
{
local rval: Info = Queue::get(msgs[id]);
if ( Queue::len(msgs[id]) == 0 )
delete msgs[id];
return rval;
}
hook set_session(c: connection, msg: dns_msg, is_query: bool) &priority=5
{
if ( ! c?$dns_state )
@ -151,29 +223,39 @@ hook set_session(c: connection, msg: dns_msg, is_query: bool) &priority=5
c$dns_state = state;
}
if ( msg$id !in c$dns_state$pending )
c$dns_state$pending[msg$id] = Queue::init();
local info: Info;
# If this is either a query or this is the reply but
# no Info records are in the queue (we missed the query?)
# we need to create an Info record and put it in the queue.
if ( is_query ||
Queue::len(c$dns_state$pending[msg$id]) == 0 )
{
info = new_session(c, msg$id);
Queue::put(c$dns_state$pending[msg$id], info);
}
if ( is_query )
# If this is a query, assign the newly created info variable
# so that the world looks correct to anything else handling
# this query.
c$dns = info;
{
if ( msg$id in c$dns_state$pending_replies &&
Queue::len(c$dns_state$pending_replies[msg$id]) > 0 )
{
# Match this DNS query w/ what's at head of pending reply queue.
c$dns = pop_msg(c$dns_state$pending_replies, msg$id);
}
else
{
# Create a new DNS session and put it in the query queue so
# we can wait for a matching reply.
c$dns = new_session(c, msg$id);
enqueue_new_msg(c$dns_state$pending_queries, msg$id, c$dns);
}
}
else
# Peek at the next item in the queue for this trans_id and
# assign it to c$dns since this is a response.
c$dns = Queue::peek(c$dns_state$pending[msg$id]);
{
if ( msg$id in c$dns_state$pending_queries &&
Queue::len(c$dns_state$pending_queries[msg$id]) > 0 )
{
# Match this DNS reply w/ what's at head of pending query queue.
c$dns = pop_msg(c$dns_state$pending_queries, msg$id);
}
else
{
# Create a new DNS session and put it in the reply queue so
# we can wait for a matching query.
c$dns = new_session(c, msg$id);
event conn_weird("dns_unmatched_reply", c, "");
enqueue_new_msg(c$dns_state$pending_replies, msg$id, c$dns);
}
}
if ( ! is_query )
{
@ -183,36 +265,36 @@ hook set_session(c: connection, msg: dns_msg, is_query: bool) &priority=5
if ( ! c$dns?$total_answers )
c$dns$total_answers = msg$num_answers;
if ( c$dns?$total_replies &&
c$dns$total_replies != msg$num_answers + msg$num_addl + msg$num_auth )
{
event conn_weird("dns_changed_number_of_responses", c,
fmt("The declared number of responses changed from %d to %d",
c$dns$total_replies,
msg$num_answers + msg$num_addl + msg$num_auth));
}
else
{
# Store the total number of responses expected from the first reply.
if ( ! c$dns?$total_replies )
c$dns$total_replies = msg$num_answers + msg$num_addl + msg$num_auth;
}
if ( msg$rcode != 0 && msg$num_queries == 0 )
c$dns$rejected = T;
}
}
event dns_message(c: connection, is_orig: bool, msg: dns_msg, len: count) &priority=5
{
hook set_session(c, msg, is_orig);
if ( msg$opcode != 0 )
# Currently only standard queries are tracked.
return;
hook set_session(c, msg, ! msg$QR);
}
event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
hook DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
{
if ( msg$opcode != 0 )
# Currently only standard queries are tracked.
return;
if ( ! msg$QR )
# This is weird: the inquirer must also be providing answers in
# the request, which is not what we want to track.
return;
if ( ans$answer_type == DNS_ANS )
{
if ( ! c?$dns )
{
event conn_weird("dns_unmatched_reply", c, "");
hook set_session(c, msg, F);
}
c$dns$AA = msg$AA;
c$dns$RA = msg$RA;
@ -226,29 +308,35 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
c$dns$TTLs = vector();
c$dns$TTLs[|c$dns$TTLs|] = ans$TTL;
}
if ( c$dns?$answers && c$dns?$total_answers &&
|c$dns$answers| == c$dns$total_answers )
{
# Indicate this request/reply pair is ready to be logged.
c$dns$ready = T;
}
}
}
event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=-5
event dns_end(c: connection, msg: dns_msg) &priority=5
{
if ( c$dns$ready )
if ( ! c?$dns )
return;
if ( msg$QR )
c$dns$saw_reply = T;
else
c$dns$saw_query = T;
}
event dns_end(c: connection, msg: dns_msg) &priority=-5
{
if ( c?$dns && c$dns$saw_reply && c$dns$saw_query )
{
Log::write(DNS::LOG, c$dns);
# This record is logged and no longer pending.
Queue::get(c$dns_state$pending[c$dns$trans_id]);
delete c$dns;
}
}
event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
{
if ( msg$opcode != 0 )
# Currently only standard queries are tracked.
return;
c$dns$RD = msg$RD;
c$dns$TC = msg$TC;
c$dns$qclass = qclass;
@ -265,60 +353,66 @@ event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qcla
c$dns$query = query;
}
event dns_unknown_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
{
hook DNS::do_reply(c, msg, ans, fmt("<unknown type=%s>", ans$qtype));
}
event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
{
event DNS::do_reply(c, msg, ans, fmt("%s", a));
hook DNS::do_reply(c, msg, ans, fmt("%s", a));
}
event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, str: string) &priority=5
{
event DNS::do_reply(c, msg, ans, str);
hook DNS::do_reply(c, msg, ans, str);
}
event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
{
event DNS::do_reply(c, msg, ans, fmt("%s", a));
hook DNS::do_reply(c, msg, ans, fmt("%s", a));
}
event dns_A6_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
{
event DNS::do_reply(c, msg, ans, fmt("%s", a));
hook DNS::do_reply(c, msg, ans, fmt("%s", a));
}
event dns_NS_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
{
event DNS::do_reply(c, msg, ans, name);
hook DNS::do_reply(c, msg, ans, name);
}
event dns_CNAME_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
{
event DNS::do_reply(c, msg, ans, name);
hook DNS::do_reply(c, msg, ans, name);
}
event dns_MX_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string,
preference: count) &priority=5
{
event DNS::do_reply(c, msg, ans, name);
hook DNS::do_reply(c, msg, ans, name);
}
event dns_PTR_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
{
event DNS::do_reply(c, msg, ans, name);
hook DNS::do_reply(c, msg, ans, name);
}
event dns_SOA_reply(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa) &priority=5
{
event DNS::do_reply(c, msg, ans, soa$mname);
hook DNS::do_reply(c, msg, ans, soa$mname);
}
event dns_WKS_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
{
event DNS::do_reply(c, msg, ans, "");
hook DNS::do_reply(c, msg, ans, "");
}
event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
{
event DNS::do_reply(c, msg, ans, "");
hook DNS::do_reply(c, msg, ans, "");
}
# TODO: figure out how to handle these
@ -339,7 +433,8 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
event dns_rejected(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
{
c$dns$rejected = T;
if ( c?$dns )
c$dns$rejected = T;
}
event connection_state_remove(c: connection) &priority=-5
@ -347,16 +442,23 @@ event connection_state_remove(c: connection) &priority=-5
if ( ! c?$dns_state )
return;
# If Bro is expiring state, we should go ahead and log all unlogged
# request/response pairs now.
for ( trans_id in c$dns_state$pending )
{
local infos: vector of Info;
Queue::get_vector(c$dns_state$pending[trans_id], infos);
for ( i in infos )
{
Log::write(DNS::LOG, infos[i]);
}
}
# If Bro is expiring state, we should go ahead and log all unmatched
# queries and replies now.
log_unmatched_msgs(c$dns_state$pending_queries);
log_unmatched_msgs(c$dns_state$pending_replies);
}
function expire_pending_msg(pending: PendingMessages, id: count): interval
{
local infos: vector of Info;
Queue::get_vector(pending[id], infos);
for ( i in infos )
{
Log::write(DNS::LOG, infos[i]);
event flow_weird("dns_unmatched_msg", infos[i]$id$orig_h,
infos[i]$id$resp_h);
}
return 0sec;
}

View file

@ -1,6 +1,8 @@
# List of HTTP headers pulled from:
# http://annevankesteren.nl/2007/10/http-methods
signature dpd_http_client {
ip-proto == tcp
payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
payload /^[[:space:]]*(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH|BCOPY|BDELETE|BMOVE|BPROPFIND|BPROPPATCH|NOTIFY|POLL|SUBSCRIBE|UNSUBSCRIBE|X-MS-ENUMATTS|RPC_OUT_DATA|RPC_IN_DATA)[[:space:]]*/
tcp-state originator
}
@ -11,3 +13,5 @@ signature dpd_http_server {
requires-reverse-signature dpd_http_client
enable "http"
}