mirror of
https://github.com/zeek/zeek.git
synced 2025-10-03 07:08:19 +00:00
Merge remote-tracking branch 'origin/master' into topic/bernhard/input-threads
Conflicts: src/CMakeLists.txt testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
This commit is contained in:
Bernhard Amann
commit
3b82d69eb3
167 changed files with 3528 additions and 1066 deletions
|
|
@ -105,5 +105,8 @@ event protocol_violation(c: connection, atype: count, aid: count,
|
|||
reason: string) &priority=-5
|
||||
{
|
||||
if ( c?$dpd )
|
||||
{
|
||||
Log::write(DPD::LOG, c$dpd);
|
||||
delete c$dpd;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
@load ./main
|
||||
@load ./postprocessors
|
||||
@load ./writers/ascii
|
||||
@load ./writers/dataseries
|
||||
|
|
|
|||
|
|
@ -332,7 +332,7 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
|
|||
function default_path_func(id: ID, path: string, rec: any) : string
|
||||
{
|
||||
local id_str = fmt("%s", id);
|
||||
|
||||
|
||||
local parts = split1(id_str, /::/);
|
||||
if ( |parts| == 2 )
|
||||
{
|
||||
|
|
@ -340,7 +340,7 @@ function default_path_func(id: ID, path: string, rec: any) : string
|
|||
# or a filter path explicitly set by the user, so continue using it.
|
||||
if ( path != "" )
|
||||
return path;
|
||||
|
||||
|
||||
# Example: Notice::LOG -> "notice"
|
||||
if ( parts[2] == "LOG" )
|
||||
{
|
||||
|
|
@ -356,11 +356,11 @@ function default_path_func(id: ID, path: string, rec: any) : string
|
|||
output = cat(output, sub_bytes(module_parts[4],1,1), "_", sub_bytes(module_parts[4], 2, |module_parts[4]|));
|
||||
return to_lower(output);
|
||||
}
|
||||
|
||||
|
||||
# Example: Notice::POLICY_LOG -> "notice_policy"
|
||||
if ( /_LOG$/ in parts[2] )
|
||||
parts[2] = sub(parts[2], /_LOG$/, "");
|
||||
|
||||
|
||||
return cat(to_lower(parts[1]),"_",to_lower(parts[2]));
|
||||
}
|
||||
else
|
||||
|
|
@ -376,13 +376,16 @@ function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : boo
|
|||
if ( pp_cmd == "" )
|
||||
return T;
|
||||
|
||||
# Turn, e.g., Log::WRITER_ASCII into "ascii".
|
||||
local writer = subst_string(to_lower(fmt("%s", info$writer)), "log::writer_", "");
|
||||
|
||||
# The date format is hard-coded here to provide a standardized
|
||||
# script interface.
|
||||
system(fmt("%s %s %s %s %s %d",
|
||||
system(fmt("%s %s %s %s %s %d %s",
|
||||
pp_cmd, npath, info$path,
|
||||
strftime("%y-%m-%d_%H.%M.%S", info$open),
|
||||
strftime("%y-%m-%d_%H.%M.%S", info$close),
|
||||
info$terminating));
|
||||
info$terminating, writer));
|
||||
|
||||
return T;
|
||||
}
|
||||
|
|
@ -407,7 +410,7 @@ function add_filter(id: ID, filter: Filter) : bool
|
|||
# definition.
|
||||
if ( ! filter?$path_func )
|
||||
filter$path_func = default_path_func;
|
||||
|
||||
|
||||
filters[id, filter$name] = filter;
|
||||
return __add_filter(id, filter);
|
||||
}
|
||||
|
|
|
|||
60
scripts/base/frameworks/logging/writers/dataseries.bro
Normal file
60
scripts/base/frameworks/logging/writers/dataseries.bro
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
##! Interface for the DataSeries log writer.
|
||||
|
||||
module LogDataSeries;
|
||||
|
||||
export {
|
||||
## Compression to use with the DS output file. Options are:
|
||||
##
|
||||
## 'none' -- No compression.
|
||||
## 'lzf' -- LZF compression. Very quick, but leads to larger output files.
|
||||
## 'lzo' -- LZO compression. Very fast decompression times.
|
||||
## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output.
|
||||
## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output.
|
||||
const compression = "lzo" &redef;
|
||||
|
||||
## The extent buffer size.
|
||||
## Larger values here lead to better compression and more efficient writes, but
|
||||
## also increase the lag between the time events are received and the time they
|
||||
## are actually written to disk.
|
||||
const extent_size = 65536 &redef;
|
||||
|
||||
## Should we dump the XML schema we use for this DS file to disk?
|
||||
## If yes, the XML schema shares the name of the logfile, but has
|
||||
## an XML ending.
|
||||
const dump_schema = F &redef;
|
||||
|
||||
## How many threads should DataSeries spawn to perform compression?
|
||||
## Note that this dictates the number of threads per log stream. If
|
||||
## you're using a lot of streams, you may want to keep this number
|
||||
## relatively small.
|
||||
##
|
||||
## Default value is 1, which will spawn one thread / stream.
|
||||
##
|
||||
## Maximum is 128, minimum is 1.
|
||||
const num_threads = 1 &redef;
|
||||
|
||||
## Should time be stored as an integer or a double?
|
||||
## Storing time as a double leads to possible precision issues and
|
||||
## can (significantly) increase the size of the resulting DS log.
|
||||
## That said, timestamps stored in double form are consistent
|
||||
## with the rest of Bro, including the standard ASCII log. Hence, we
|
||||
## use them by default.
|
||||
const use_integer_for_time = F &redef;
|
||||
}
|
||||
|
||||
# Default function to postprocess a rotated DataSeries log file. It moves the
|
||||
# rotated file to a new name that includes a timestamp with the opening time, and
|
||||
# then runs the writer's default postprocessor command on it.
|
||||
function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
|
||||
{
|
||||
# Move file to name including both opening and closing time.
|
||||
local dst = fmt("%s.%s.ds", info$path,
|
||||
strftime(Log::default_rotation_date_format, info$open));
|
||||
|
||||
system(fmt("/bin/mv %s %s", info$fname, dst));
|
||||
|
||||
# Run default postprocessor.
|
||||
return Log::run_rotation_postprocessor_cmd(info, dst);
|
||||
}
|
||||
|
||||
redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };
|
||||
|
|
@ -23,7 +23,10 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
|
|||
@if ( Cluster::local_node_type() != Cluster::MANAGER )
|
||||
# The notice policy is completely handled by the manager and shouldn't be
|
||||
# done by workers or proxies to save time for packet processing.
|
||||
redef policy = {};
|
||||
event bro_init() &priority=-11
|
||||
{
|
||||
Notice::policy = table();
|
||||
}
|
||||
|
||||
event Notice::begin_suppression(n: Notice::Info)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ type icmp_conn: record {
|
|||
itype: count; ##< The ICMP type of the packet that triggered the instantiation of the record.
|
||||
icode: count; ##< The ICMP code of the packet that triggered the instantiation of the record.
|
||||
len: count; ##< The length of the ICMP payload of the packet that triggered the instantiation of the record.
|
||||
hlim: count; ##< The encapsulating IP header's Hop Limit value.
|
||||
v6: bool; ##< True if it's an ICMPv6 packet.
|
||||
};
|
||||
|
||||
|
|
@ -2336,6 +2337,11 @@ type bt_tracker_headers: table[string] of string;
|
|||
## BPF filter the user has set via the -f command line options. Empty if none.
|
||||
const cmd_line_bpf_filter = "" &redef;
|
||||
|
||||
## The maximum number of open files to keep cached at a given time.
|
||||
## If set to zero, this is automatically determined by inspecting
|
||||
## the current/maximum limit on open files for the process.
|
||||
const max_files_in_cache = 0 &redef;
|
||||
|
||||
## Deprecated.
|
||||
const log_rotate_interval = 0 sec &redef;
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
@load ./utils-commands
|
||||
@load base/utils/paths
|
||||
@load base/utils/numbers
|
||||
@load base/utils/addrs
|
||||
|
||||
module FTP;
|
||||
|
||||
|
|
@ -22,7 +23,7 @@ export {
|
|||
const default_capture_password = F &redef;
|
||||
|
||||
## User IDs that can be considered "anonymous".
|
||||
const guest_ids = { "anonymous", "ftp", "guest" } &redef;
|
||||
const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef;
|
||||
|
||||
type Info: record {
|
||||
## Time when the command was sent.
|
||||
|
|
@ -160,17 +161,16 @@ function ftp_message(s: Info)
|
|||
# or it's a deliberately logged command.
|
||||
if ( |s$tags| > 0 || (s?$cmdarg && s$cmdarg$cmd in logged_commands) )
|
||||
{
|
||||
if ( s?$password && to_lower(s$user) !in guest_ids )
|
||||
if ( s?$password &&
|
||||
! s$capture_password &&
|
||||
to_lower(s$user) !in guest_ids )
|
||||
{
|
||||
s$password = "<hidden>";
|
||||
}
|
||||
|
||||
local arg = s$cmdarg$arg;
|
||||
if ( s$cmdarg$cmd in file_cmds )
|
||||
{
|
||||
if ( is_v4_addr(s$id$resp_h) )
|
||||
arg = fmt("ftp://%s%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
|
||||
else
|
||||
arg = fmt("ftp://[%s]%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
|
||||
}
|
||||
arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), build_path_compressed(s$cwd, arg));
|
||||
|
||||
s$ts=s$cmdarg$ts;
|
||||
s$command=s$cmdarg$cmd;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
##! Utilities specific for HTTP processing.
|
||||
|
||||
@load ./main
|
||||
@load base/utils/addrs
|
||||
|
||||
module HTTP;
|
||||
|
||||
|
|
@ -51,7 +52,7 @@ function extract_keys(data: string, kv_splitter: pattern): string_vec
|
|||
function build_url(rec: Info): string
|
||||
{
|
||||
local uri = rec?$uri ? rec$uri : "/<missed_request>";
|
||||
local host = rec?$host ? rec$host : fmt("%s", rec$id$resp_h);
|
||||
local host = rec?$host ? rec$host : addr_to_uri(rec$id$resp_h);
|
||||
if ( rec$id$resp_p != 80/tcp )
|
||||
host = fmt("%s:%s", host, rec$id$resp_p);
|
||||
return fmt("%s%s", host, uri);
|
||||
|
|
|
|||
|
|
@ -77,8 +77,12 @@ export {
|
|||
[12] = "srp",
|
||||
[13] = "signature_algorithms",
|
||||
[14] = "use_srtp",
|
||||
[15] = "heartbeat",
|
||||
[35] = "SessionTicket TLS",
|
||||
[40] = "extended_random",
|
||||
[13172] = "next_protocol_negotiation",
|
||||
[13175] = "origin_bound_certificates",
|
||||
[13180] = "encrypted_client_certificates",
|
||||
[65281] = "renegotiation_info"
|
||||
} &default=function(i: count):string { return fmt("unknown-%d", i); };
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ export {
|
|||
session_id: string &log &optional;
|
||||
## Subject of the X.509 certificate offered by the server.
|
||||
subject: string &log &optional;
|
||||
## Subject of the signer of the X.509 certificate offered by the server.
|
||||
issuer_subject: string &log &optional;
|
||||
## NotValidBefore field value from the server certificate.
|
||||
not_valid_before: time &log &optional;
|
||||
## NotValidAfter field value from the serve certificate.
|
||||
|
|
@ -146,6 +148,7 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
|
|||
|
||||
# Also save other certificate information about the primary cert.
|
||||
c$ssl$subject = cert$subject;
|
||||
c$ssl$issuer_subject = cert$issuer;
|
||||
c$ssl$not_valid_before = cert$not_valid_before;
|
||||
c$ssl$not_valid_after = cert$not_valid_after;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,3 +98,18 @@ function find_ip_addresses(input: string): string_array
|
|||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
## Returns the string representation of an IP address suitable for inclusion
|
||||
## in a URI. For IPv4, this does no special formatting, but for IPv6, the
|
||||
## address is included in square brackets.
|
||||
##
|
||||
## a: the address to make suitable for URI inclusion.
|
||||
##
|
||||
## Returns: the string representation of *a* suitable for URI inclusion.
|
||||
function addr_to_uri(a: addr): string
|
||||
{
|
||||
if ( is_v4_addr(a) )
|
||||
return fmt("%s", a);
|
||||
else
|
||||
return fmt("[%s]", a);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
@load ./addrs
|
||||
|
||||
## This function can be used to generate a consistent filename for when
|
||||
## contents of a file, stream, or connection are being extracted to disk.
|
||||
function generate_extraction_filename(prefix: string, c: connection, suffix: string): string
|
||||
{
|
||||
local conn_info = fmt("%s:%d-%s:%d",
|
||||
c$id$orig_h, c$id$orig_p, c$id$resp_h, c$id$resp_p);
|
||||
local conn_info = fmt("%s:%d-%s:%d", addr_to_uri(c$id$orig_h), c$id$orig_p,
|
||||
addr_to_uri(c$id$resp_h), c$id$resp_p);
|
||||
|
||||
if ( prefix != "" )
|
||||
conn_info = fmt("%s_%s", prefix, conn_info);
|
||||
|
|
|
|||
|
|
@ -8,27 +8,31 @@ export {
|
|||
## Address space that is considered private and unrouted.
|
||||
## By default it has RFC defined non-routable IPv4 address space.
|
||||
const private_address_space: set[subnet] = {
|
||||
10.0.0.0/8,
|
||||
192.168.0.0/16,
|
||||
127.0.0.0/8,
|
||||
172.16.0.0/12
|
||||
10.0.0.0/8,
|
||||
192.168.0.0/16,
|
||||
172.16.0.0/12,
|
||||
100.64.0.0/10, # RFC6598 Carrier Grade NAT
|
||||
127.0.0.0/8,
|
||||
[fe80::]/10,
|
||||
[::1]/128,
|
||||
} &redef;
|
||||
|
||||
## Networks that are considered "local".
|
||||
const local_nets: set[subnet] &redef;
|
||||
|
||||
## This is used for retrieving the subnet when you multiple
|
||||
## :bro:id:`Site::local_nets`. A membership query can be done with an
|
||||
## :bro:type:`addr` and the table will yield the subnet it was found
|
||||
|
||||
## This is used for retrieving the subnet when using multiple entries in
|
||||
## :bro:id:`Site::local_nets`. It's populated automatically from there.
|
||||
## A membership query can be done with an
|
||||
## :bro:type:`addr` and the table will yield the subnet it was found
|
||||
## within.
|
||||
global local_nets_table: table[subnet] of subnet = {};
|
||||
|
||||
## Networks that are considered "neighbors".
|
||||
const neighbor_nets: set[subnet] &redef;
|
||||
|
||||
|
||||
## If local network administrators are known and they have responsibility
|
||||
## for defined address space, then a mapping can be defined here between
|
||||
## networks for which they have responsibility and a set of email
|
||||
## networks for which they have responsibility and a set of email
|
||||
## addresses.
|
||||
const local_admins: table[subnet] of set[string] = {} &redef;
|
||||
|
||||
|
|
@ -40,27 +44,33 @@ export {
|
|||
|
||||
## Function that returns true if an address corresponds to one of
|
||||
## the local networks, false if not.
|
||||
## The function inspects :bro:id:`Site::local_nets`.
|
||||
global is_local_addr: function(a: addr): bool;
|
||||
|
||||
|
||||
## Function that returns true if an address corresponds to one of
|
||||
## the neighbor networks, false if not.
|
||||
## The function inspects :bro:id:`Site::neighbor_nets`.
|
||||
global is_neighbor_addr: function(a: addr): bool;
|
||||
|
||||
|
||||
## Function that returns true if an address corresponds to one of
|
||||
## the private/unrouted networks, false if not.
|
||||
## The function inspects :bro:id:`Site::private_address_space`.
|
||||
global is_private_addr: function(a: addr): bool;
|
||||
|
||||
## Function that returns true if a host name is within a local
|
||||
## Function that returns true if a host name is within a local
|
||||
## DNS zone.
|
||||
## The function inspects :bro:id:`Site::local_zones`.
|
||||
global is_local_name: function(name: string): bool;
|
||||
|
||||
## Function that returns true if a host name is within a neighbor
|
||||
|
||||
## Function that returns true if a host name is within a neighbor
|
||||
## DNS zone.
|
||||
## The function inspects :bro:id:`Site::neighbor_zones`.
|
||||
global is_neighbor_name: function(name: string): bool;
|
||||
|
||||
|
||||
## Function that returns a common separated list of email addresses
|
||||
## that are considered administrators for the IP address provided as
|
||||
## an argument.
|
||||
## The function inspects :bro:id:`Site::local_admins`.
|
||||
global get_emails: function(a: addr): string;
|
||||
}
|
||||
|
||||
|
|
@ -73,22 +83,22 @@ function is_local_addr(a: addr): bool
|
|||
{
|
||||
return a in local_nets;
|
||||
}
|
||||
|
||||
|
||||
function is_neighbor_addr(a: addr): bool
|
||||
{
|
||||
return a in neighbor_nets;
|
||||
}
|
||||
|
||||
|
||||
function is_private_addr(a: addr): bool
|
||||
{
|
||||
return a in private_address_space;
|
||||
}
|
||||
|
||||
|
||||
function is_local_name(name: string): bool
|
||||
{
|
||||
return local_dns_suffix_regex in name;
|
||||
}
|
||||
|
||||
|
||||
function is_neighbor_name(name: string): bool
|
||||
{
|
||||
return local_dns_neighbor_suffix_regex in name;
|
||||
|
|
@ -96,7 +106,7 @@ function is_neighbor_name(name: string): bool
|
|||
|
||||
# This is a hack for doing a for loop.
|
||||
const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32};
|
||||
|
||||
|
||||
# TODO: make this work with IPv6
|
||||
function find_all_emails(ip: addr): set[string]
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue