Merge remote-tracking branch 'origin/master' into topic/vladg/kerberos

Conflicts:
	testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
	testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log
This commit is contained in:
Vlad Grigorescu 2014-10-27 13:56:07 -04:00
commit e6d6ba6ec6
511 changed files with 108792 additions and 86637 deletions

View file

@ -9,3 +9,8 @@ signature file-tar {
file-magic /([[:print:]\x00]){100}(([[:digit:]\x00\x20]){8}){3}/
file-mime "application/x-tar", 150
}
signature file-swf {
file-magic /(F|C|Z)WS/
file-mime "application/x-shockwave-flash", 60
}

View file

@ -2769,19 +2769,6 @@ signature file-magic-auto408 {
file-magic /(.{512})(\xec\xa5\xc1)/
}
# >0 string,=FWS (len=3), ["Macromedia Flash data,"], swap_endian=0
# >>3 byte&,x, ["version %d"], swap_endian=0
signature file-magic-auto409 {
file-mime "application/x-shockwave-flash", 1
file-magic /(FWS)(.{1})/
}
# >0 string,=CWS (len=3), ["Macromedia Flash data (compressed),"], swap_endian=0
signature file-magic-auto410 {
file-mime "application/x-shockwave-flash", 60
file-magic /(CWS)/
}
# >0 regex/20,=^\.[A-Za-z0-9][A-Za-z0-9][ \t] (len=29), ["troff or preprocessor input text"], swap_endian=0
signature file-magic-auto411 {
file-mime "text/troff", 59

View file

@ -56,7 +56,7 @@ export {
## local file path which was read, or some other input source.
source: string &log &optional;
## A value to represent the depth of this file in relation
## A value to represent the depth of this file in relation
## to its source. In SMTP, it is the depth of the MIME
## attachment on the message. In HTTP, it is the depth of the
## request within the TCP connection.
@ -73,7 +73,7 @@ export {
mime_type: string &log &optional;
## A filename for the file if one is available from the source
## for the file. These will frequently come from
## for the file. These will frequently come from
## "Content-Disposition" headers in network protocols.
filename: string &log &optional;
@ -149,10 +149,19 @@ export {
## Returns: true if the analyzer will be added, or false if analysis
## for the file isn't currently active or the *args*
## were invalid for the analyzer type.
global add_analyzer: function(f: fa_file,
tag: Files::Tag,
global add_analyzer: function(f: fa_file,
tag: Files::Tag,
args: AnalyzerArgs &default=AnalyzerArgs()): bool;
## Adds all analyzers associated with a give MIME type to the analysis of
## a file. Note that analyzers added via MIME types cannot take further
## arguments.
##
## f: the file.
##
## mtype: the MIME type; it will be compared case-insensitive.
global add_analyzers_for_mime_type: function(f: fa_file, mtype: string);
## Removes an analyzer from the analysis of a given file.
##
## f: the file.
@ -196,7 +205,7 @@ export {
## A callback to generate a file handle on demand when
## one is needed by the core.
get_file_handle: function(c: connection, is_orig: bool): string;
## A callback to "describe" a file. In the case of an HTTP
## transfer the most obvious description would be the URL.
## It's like an extremely compressed version of the normal log.
@ -207,7 +216,7 @@ export {
## Register callbacks for protocols that work with the Files framework.
## The callbacks must uniquely identify a file and each protocol can
## only have a single callback registered for it.
##
##
## tag: Tag for the protocol analyzer having a callback being registered.
##
## reg: A :bro:see:`Files::ProtoRegistration` record.
@ -225,6 +234,42 @@ export {
## callback: Function to execute when the given file analyzer is being added.
global register_analyzer_add_callback: function(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs));
## Registers a set of MIME types for an analyzer. If a future connection on one of
## these types is seen, the analyzer will be automatically assigned to parsing it.
## The function *adds* to all MIME types already registered, it doesn't replace
## them.
##
## tag: The tag of the analyzer.
##
## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive).
##
## Returns: True if the MIME types were successfully registered.
global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool;
## Registers a MIME type for an analyzer. If a future file with this type is seen,
## the analyzer will be automatically assigned to parsing it. The function *adds*
## to all MIME types already registered, it doesn't replace them.
##
## tag: The tag of the analyzer.
##
## mt: The MIME type in the form "foo/bar" (case-insensitive).
##
## Returns: True if the MIME type was successfully registered.
global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool;
## Returns a set of all MIME types currently registered for a specific analyzer.
##
## tag: The tag of the analyzer.
##
## Returns: The set of MIME types.
global registered_mime_types: function(tag: Analyzer::Tag) : set[string];
## Returns a table of all MIME-type-to-analyzer mappings currently registered.
##
## Returns: A table mapping each analyzer to the set of MIME types
## registered for it.
global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string];
## Event that can be handled to access the Info record as it is sent on
## to the logging framework.
global log_files: event(rec: Info);
@ -237,6 +282,9 @@ redef record fa_file += {
# Store the callbacks for protocol analyzers that have files.
global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
# Store the MIME type to analyzer mappings.
global mime_types: table[Analyzer::Tag] of set[string];
global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
event bro_init() &priority=5
@ -259,13 +307,13 @@ function set_info(f: fa_file)
f$info$source = f$source;
f$info$duration = f$last_active - f$info$ts;
f$info$seen_bytes = f$seen_bytes;
if ( f?$total_bytes )
if ( f?$total_bytes )
f$info$total_bytes = f$total_bytes;
f$info$missing_bytes = f$missing_bytes;
f$info$overflow_bytes = f$overflow_bytes;
if ( f?$is_orig )
f$info$is_orig = f$is_orig;
if ( f?$mime_type )
if ( f?$mime_type )
f$info$mime_type = f$mime_type;
}
@ -289,6 +337,15 @@ function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
return T;
}
function add_analyzers_for_mime_type(f: fa_file, mtype: string)
{
local dummy_args: AnalyzerArgs;
local analyzers = __add_analyzers_for_mime_type(f$id, mtype, dummy_args);
for ( tag in analyzers )
add f$info$analyzers[Files::analyzer_name(tag)];
}
function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
{
analyzer_add_callbacks[tag] = callback;
@ -312,6 +369,9 @@ function analyzer_name(tag: Files::Tag): string
event file_new(f: fa_file) &priority=10
{
set_info(f);
if ( f?$mime_type )
add_analyzers_for_mime_type(f, f$mime_type);
}
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
@ -349,6 +409,41 @@ function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
return result;
}
function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool
{
local rc = T;
for ( mt in mime_types )
{
if ( ! register_for_mime_type(tag, mt) )
rc = F;
}
return rc;
}
function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
{
if ( ! __register_for_mime_type(tag, mt) )
return F;
if ( tag !in mime_types )
mime_types[tag] = set();
add mime_types[tag][mt];
return T;
}
function registered_mime_types(tag: Analyzer::Tag) : set[string]
{
return tag in mime_types ? mime_types[tag] : set();
}
function all_registered_mime_types(): table[Analyzer::Tag] of set[string]
{
return mime_types;
}
function describe(f: fa_file): string
{
local tag = Analyzer::get_tag(f$source);

View file

@ -4,6 +4,17 @@
module Input;
export {
type Event: enum {
EVENT_NEW = 0,
EVENT_CHANGED = 1,
EVENT_REMOVED = 2,
};
type Mode: enum {
MANUAL = 0,
REREAD = 1,
STREAM = 2
};
## The default input reader used. Defaults to `READER_ASCII`.
const default_reader = READER_ASCII &redef;

View file

@ -81,6 +81,9 @@ export {
## Where the data was discovered.
where: Where &log;
## The name of the node where the match was discovered.
node: string &optional &log;
## If the data was discovered within a connection, the
## connection record should go here to give context to the data.
conn: connection &optional;
@ -240,6 +243,11 @@ function Intel::seen(s: Seen)
s$indicator_type = Intel::ADDR;
}
if ( ! s?$node )
{
s$node = peer_description;
}
if ( have_full_data )
{
local items = get_items(s);

View file

@ -1,7 +1,5 @@
@load ./main
@load ./postprocessors
@load ./writers/ascii
@load ./writers/dataseries
@load ./writers/sqlite
@load ./writers/elasticsearch
@load ./writers/none

View file

@ -5,9 +5,15 @@
module Log;
# Log::ID and Log::Writer are defined in types.bif due to circular dependencies.
export {
## Type that defines an ID unique to each log stream. Scripts creating new log
## streams need to redef this enum to add their own specific log ID. The log ID
## implicitly determines the default name of the generated log file.
type Log::ID: enum {
## Dummy place-holder.
UNKNOWN
};
## If true, local logging is by default enabled for all filters.
const enable_local_logging = T &redef;
@ -27,13 +33,13 @@ export {
const set_separator = "," &redef;
## String to use for empty fields. This should be different from
## *unset_field* to make the output unambiguous.
## *unset_field* to make the output unambiguous.
## Can be overwritten by individual writers.
const empty_field = "(empty)" &redef;
## String to use for an unset &optional field.
## Can be overwritten by individual writers.
const unset_field = "-" &redef;
const unset_field = "-" &redef;
## Type defining the content of a logging stream.
type Stream: record {

View file

@ -1,60 +0,0 @@
##! Interface for the DataSeries log writer.
module LogDataSeries;
export {
## Compression to use with the DS output file. Options are:
##
## 'none' -- No compression.
## 'lzf' -- LZF compression (very quick, but leads to larger output files).
## 'lzo' -- LZO compression (very fast decompression times).
## 'zlib' -- GZIP compression (slower than LZF, but also produces smaller output).
## 'bz2' -- BZIP2 compression (slower than GZIP, but also produces smaller output).
const compression = "zlib" &redef;
## The extent buffer size.
## Larger values here lead to better compression and more efficient writes,
## but also increase the lag between the time events are received and
## the time they are actually written to disk.
const extent_size = 65536 &redef;
## Should we dump the XML schema we use for this DS file to disk?
## If yes, the XML schema shares the name of the logfile, but has
## an XML ending.
const dump_schema = F &redef;
## How many threads should DataSeries spawn to perform compression?
## Note that this dictates the number of threads per log stream. If
## you're using a lot of streams, you may want to keep this number
## relatively small.
##
## Default value is 1, which will spawn one thread / stream.
##
## Maximum is 128, minimum is 1.
const num_threads = 1 &redef;
## Should time be stored as an integer or a double?
## Storing time as a double leads to possible precision issues and
## can (significantly) increase the size of the resulting DS log.
## That said, timestamps stored in double form are consistent
## with the rest of Bro, including the standard ASCII log. Hence, we
## use them by default.
const use_integer_for_time = F &redef;
}
# Default function to postprocess a rotated DataSeries log file. It moves the
# rotated file to a new name that includes a timestamp with the opening time,
# and then runs the writer's default postprocessor command on it.
function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
{
# Move file to name including both opening and closing time.
local dst = fmt("%s.%s.ds", info$path,
strftime(Log::default_rotation_date_format, info$open));
system(fmt("/bin/mv %s %s", info$fname, dst));
# Run default postprocessor.
return Log::run_rotation_postprocessor_cmd(info, dst);
}
redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };

View file

@ -1,48 +0,0 @@
##! Log writer for sending logs to an ElasticSearch server.
##!
##! Note: This module is in testing and is not yet considered stable!
##!
##! There is one known memory issue. If your elasticsearch server is
##! running slowly and taking too long to return from bulk insert
##! requests, the message queue to the writer thread will continue
##! growing larger and larger giving the appearance of a memory leak.
module LogElasticSearch;
export {
## Name of the ES cluster.
const cluster_name = "elasticsearch" &redef;
## ES server.
const server_host = "127.0.0.1" &redef;
## ES port.
const server_port = 9200 &redef;
## Name of the ES index.
const index_prefix = "bro" &redef;
## The ES type prefix comes before the name of the related log.
## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
const type_prefix = "" &redef;
## The time before an ElasticSearch transfer will timeout. Note that
## the fractional part of the timeout will be ignored. In particular,
## time specifications less than a second result in a timeout value of
## 0, which means "no timeout."
const transfer_timeout = 2secs;
## The batch size is the number of messages that will be queued up before
## they are sent to be bulk indexed.
const max_batch_size = 1000 &redef;
## The maximum amount of wall-clock time that is allowed to pass without
## finishing a bulk log send. This represents the maximum delay you
## would like to have with your logs before they are sent to ElasticSearch.
const max_batch_interval = 1min &redef;
## The maximum byte size for a buffered JSON string to send to the bulk
## insert API.
const max_byte_size = 1024 * 1024 &redef;
}

View file

@ -75,6 +75,13 @@ type addr_vec: vector of addr;
## directly and then remove this alias.
type table_string_of_string: table[string] of string;
## A set of file analyzer tags.
##
## .. todo:: We need this type definition only for declaring builtin functions
## via ``bifcl``. We should extend ``bifcl`` to understand composite types
## directly and then remove this alias.
type files_tag_set: set[Files::Tag];
## A structure indicating a MIME type and strength of a match against
## file magic signatures.
##
@ -2478,8 +2485,7 @@ type http_message_stat: record {
header_length: count;
};
## Maximum number of HTTP entity data delivered to events. The amount of data
## can be limited for better performance, zero disables truncation.
## Maximum number of HTTP entity data delivered to events.
##
## .. bro:see:: http_entity_data skip_http_entity_data skip_http_data
global http_entity_data_delivery_size = 1500 &redef;
@ -2731,6 +2737,7 @@ type ModbusRegisters: vector of count;
type ModbusHeaders: record {
tid: count;
pid: count;
len: count;
uid: count;
function_code: count;
};
@ -3508,9 +3515,6 @@ const global_hash_seed: string = "" &redef;
## The maximum is currently 128 bits.
const bits_per_uid: count = 96 &redef;
# Load BiFs defined by plugins.
@load base/bif/plugins
# Load these frameworks here because they use fairly deep integration with
# BiFs and script-land defined types.
@load base/frameworks/logging
@ -3519,3 +3523,7 @@ const bits_per_uid: count = 96 &redef;
@load base/frameworks/files
@load base/bif
# Load BiFs defined by plugins.
@load base/bif/plugins

View file

@ -47,13 +47,13 @@ redef record connection += {
const ports = { 67/udp, 68/udp };
redef likely_server_ports += { 67/udp };
event bro_init()
event bro_init() &priority=5
{
Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp]);
Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, ports);
}
event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string)
event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=5
{
local info: Info;
info$ts = network_time();
@ -71,6 +71,9 @@ event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_lis
info$assigned_ip = c$id$orig_h;
c$dhcp = info;
}
event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=-5
{
Log::write(DHCP::LOG, c$dhcp);
}

View file

@ -30,6 +30,7 @@ export {
const HELLO_REQUEST = 0;
const CLIENT_HELLO = 1;
const SERVER_HELLO = 2;
const HELLO_VERIFY_REQUEST = 3; # RFC 6347
const SESSION_TICKET = 4; # RFC 5077
const CERTIFICATE = 11;
const SERVER_KEY_EXCHANGE = 12;
@ -40,6 +41,7 @@ export {
const FINISHED = 20;
const CERTIFICATE_URL = 21; # RFC 3546
const CERTIFICATE_STATUS = 22; # RFC 3546
const SUPPLEMENTAL_DATA = 23; # RFC 4680
## Mapping between numeric codes and human readable strings for alert
## levels.
@ -112,7 +114,8 @@ export {
[19] = "client_certificate_type",
[20] = "server_certificate_type",
[21] = "padding", # temporary till 2015-03-12
[22] = "encrypt_then_mac", # temporary till 2015-06-05
[22] = "encrypt_then_mac",
[23] = "extended_master_secret", # temporary till 2015-09-26
[35] = "SessionTicket TLS",
[40] = "extended_random",
[13172] = "next_protocol_negotiation",

View file

@ -12,7 +12,7 @@ export {
## Time when the SSL connection was first detected.
ts: time &log;
## Unique ID for the connection.
uid: string &log;
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## SSL/TLS version that the server offered.
@ -25,9 +25,25 @@ export {
## indicates the server name that the client was requesting.
server_name: string &log &optional;
## Session ID offered by the client for session resumption.
session_id: string &log &optional;
## Not used for logging.
session_id: string &optional;
## Flag to indicate if the session was resumed reusing
## the key material exchanged in an earlier connection.
resumed: bool &log &default=F;
## Flag to indicate if we saw a non-empty session ticket being
## sent by the client using an empty session ID. This value
## is used to determine if a session is being resumed. It's
## not logged.
client_ticket_empty_session_seen: bool &default=F;
## Flag to indicate if we saw a client key exchange message sent
## by the client. This value is used to determine if a session
## is being resumed. It's not logged.
client_key_exchange_seen: bool &default=F;
## Last alert that was seen during the connection.
last_alert: string &log &optional;
## Next protocol the server chose using the application layer
## next protocol extension, if present.
next_protocol: string &log &optional;
## The analyzer ID used for the analyzer instance attached
## to each connection. It is not used for logging since it's a
@ -36,11 +52,11 @@ export {
## Flag to indicate if this ssl session has been established
## succesfully, or if it was aborted during the handshake.
established: bool &log &default=F;
established: bool &log &default=F;
## Flag to indicate if this record already has been logged, to
## prevent duplicates.
logged: bool &default=F;
logged: bool &default=F;
};
## The default root CA bundle. By default, the mozilla-ca-list.bro
@ -149,8 +165,11 @@ event ssl_client_hello(c: connection, version: count, possible_ts: time, client_
set_session(c);
# Save the session_id if there is one set.
if ( session_id != /^\x00{32}$/ )
if ( |session_id| > 0 && session_id != /^\x00{32}$/ )
{
c$ssl$session_id = bytestring_to_hexstr(session_id);
c$ssl$client_ticket_empty_session_seen = F;
}
}
event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=5
@ -159,6 +178,9 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_
c$ssl$version = version_strings[version];
c$ssl$cipher = cipher_desc[cipher];
if ( c$ssl?$session_id && c$ssl$session_id == bytestring_to_hexstr(session_id) )
c$ssl$resumed = T;
}
event ssl_server_curve(c: connection, curve: count) &priority=5
@ -180,6 +202,45 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
}
}
event ssl_extension_application_layer_protocol_negotiation(c: connection, is_orig: bool, protocols: string_vec)
{
set_session(c);
if ( is_orig )
return;
if ( |protocols| > 0 )
c$ssl$next_protocol = protocols[0];
}
event ssl_handshake_message(c: connection, is_orig: bool, msg_type: count, length: count) &priority=5
{
set_session(c);
if ( is_orig && msg_type == SSL::CLIENT_KEY_EXCHANGE )
c$ssl$client_key_exchange_seen = T;
}
# Extension event is fired _before_ the respective client or server hello.
# Important for client_ticket_empty_session_seen.
event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
{
set_session(c);
if ( is_orig && SSL::extensions[code] == "SessionTicket TLS" && |val| > 0 )
# In this case, we might have an empty ID. Set back to F in client_hello event
# if it is not empty after all.
c$ssl$client_ticket_empty_session_seen = T;
}
event ssl_change_cipher_spec(c: connection, is_orig: bool) &priority=5
{
set_session(c);
if ( is_orig && c$ssl$client_ticket_empty_session_seen && ! c$ssl$client_key_exchange_seen )
c$ssl$resumed = T;
}
event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5
{
set_session(c);

View file

@ -65,12 +65,14 @@ function request2curl(r: Request, bodyfile: string, headersfile: string): string
cmd = fmt("%s -m %.0f", cmd, r$max_time);
if ( r?$client_data )
cmd = fmt("%s -d -", cmd);
cmd = fmt("%s -d @-", cmd);
if ( r?$addl_curl_args )
cmd = fmt("%s %s", cmd, r$addl_curl_args);
cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url));
# Make sure file will exist even if curl did not write one.
cmd = fmt("%s && touch %s", cmd, str_shell_escape(bodyfile));
return cmd;
}

View file

@ -106,6 +106,15 @@ event Input::end_of_data(name: string, source:string)
local track_file = parts[2];
# If the file is empty, still add it to the result$files table. This is needed
# because it is expected that the file was read even if it was empty.
local result = results[name];
if ( ! result?$files )
result$files = table();
if ( track_file !in result$files )
result$files[track_file] = vector();
Input::remove(name);
if ( name !in pending_files )