Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-17 14:08:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/master' into topic/johanna/tls12-decryption

Browse source
This commit is contained in:
Johanna Amann 2022-01-05 10:27:55 +00:00
commit d1e7134156
640 changed files with 14727 additions and 14980 deletions
Download patch file Download diff file Expand all files Collapse all files

28
scripts/base/frameworks/analyzer/main.zeek
View file

@ -9,6 +9,13 @@
##! These tags are defined internally by
##! the analyzers themselves, and documented in their analyzer-specific
##! description along with the events that they generate.
##!
##! Analyzer tags are also inserted into a global :zeek:type:`AllAnalyzers::Tag` enum
##! type. This type contains duplicates of all of the :zeek:type:`Analyzer::Tag`,
##! :zeek:type:`PacketAnalyzer::Tag` and :zeek:type:`Files::Tag` enum values
##! and can be used for arguments to function/hook/event definitions where they
##! need to handle any analyzer type. See :zeek:id:`Analyzer::register_for_ports`
##! for an example.
@load base/frameworks/packet-filter/utils
@ -66,13 +73,13 @@ export {
## tag: The tag of the analyzer.
##
## Returns: The set of ports.
global registered_ports: function(tag: Analyzer::Tag) : set[port];
global registered_ports: function(tag: AllAnalyzers::Tag) : set[port];
## Returns a table of all ports-to-analyzer mappings currently registered.
##
## Returns: A table mapping each analyzer to the set of ports
## registered for it.
global all_registered_ports: function() : table[Analyzer::Tag] of set[port];
global all_registered_ports: function() : table[AllAnalyzers::Tag] of set[port];
## Translates an analyzer type to a string with the analyzer's name.
##
@ -126,12 +133,16 @@ export {
global disabled_analyzers: set[Analyzer::Tag] = {
ANALYZER_TCPSTATS,
} &redef;
## A table of ports mapped to analyzers that handle those ports. This is
## used by BPF filtering and DPD. Session analyzers can add to this using
## Analyzer::register_for_port(s) and packet analyzers can add to this
## using PacketAnalyzer::register_for_port(s).
global ports: table[AllAnalyzers::Tag] of set[port];
}
@load base/bif/analyzer.bif
global ports: table[Analyzer::Tag] of set[port];
event zeek_init() &priority=5
{
if ( disable_all )
@ -176,22 +187,22 @@ function register_for_port(tag: Analyzer::Tag, p: port) : bool
return T;
}
function registered_ports(tag: Analyzer::Tag) : set[port]
function registered_ports(tag: AllAnalyzers::Tag) : set[port]
{
return tag in ports ? ports[tag] : set();
}
function all_registered_ports(): table[Analyzer::Tag] of set[port]
function all_registered_ports(): table[AllAnalyzers::Tag] of set[port]
{
return ports;
}
function name(atype: Analyzer::Tag) : string
function name(atype: AllAnalyzers::Tag) : string
{
return __name(atype);
}
function get_tag(name: string): Analyzer::Tag
function get_tag(name: string): AllAnalyzers::Tag
{
return __tag(name);
}
@ -223,4 +234,3 @@ function get_bpf(): string
}
return output;
}

2
scripts/base/frameworks/cluster/nodes/manager.zeek
View file

@ -2,7 +2,7 @@
##!
##! The manager is passive (the workers connect to us), and once connected
##! the manager registers for the events on the workers that are needed
##! to get the desired data from the workers. This script will be
##! to get the desired data from the workers. This script will be
##! automatically loaded if necessary based on the type of node being started.
##! This is where the cluster manager sets it's specific settings for other

4
scripts/base/frameworks/cluster/pools.zeek
View file

@ -364,7 +364,7 @@ event zeek_init() &priority=-5
if ( manager_is_logger )
{
local mgr = nodes_with_type(Cluster::MANAGER);
if ( |mgr| > 0 )
{
local eln = pool_eligibility[Cluster::LOGGER]$eligible_nodes;
@ -438,7 +438,7 @@ event zeek_init() &priority=-5
pet = pool_eligibility[pool$spec$node_type];
local nodes_to_init = |pet$eligible_nodes|;
if ( pool$spec?$max_nodes &&
pool$spec$max_nodes < |pet$eligible_nodes| )
nodes_to_init = pool$spec$max_nodes;

12
scripts/base/frameworks/dpd/main.zeek
View file

@ -35,7 +35,7 @@ export {
## Number of protocol violations to tolerate before disabling an analyzer.
option max_violations: table[Analyzer::Tag] of count = table() &default = 5;
## Analyzers which you don't want to throw
## Analyzers which you don't want to throw
option ignore_violations: set[Analyzer::Tag] = set();
## Ignore violations which go this many bytes into the connection.
@ -53,7 +53,7 @@ event zeek_init() &priority=5
Log::create_stream(DPD::LOG, [$columns=Info, $path="dpd", $policy=log_policy]);
}
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10
event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) &priority=10
{
local analyzer = Analyzer::name(atype);
@ -63,7 +63,7 @@ event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &pr
add c$service[analyzer];
}
event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count,
reason: string) &priority=10
{
local analyzer = Analyzer::name(atype);
@ -85,7 +85,7 @@ event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
c$dpd = info;
}
event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason: string) &priority=5
event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count, reason: string) &priority=5
{
if ( atype in ignore_violations )
return;
@ -114,8 +114,8 @@ event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason
}
}
event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
reason: string) &priority=-5
event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count,
reason: string) &priority=-5
{
if ( c?$dpd )
{

3
scripts/base/frameworks/files/magic/general.sig
View file

@ -252,7 +252,7 @@ signature file-mpqgame {
file-magic /^MPQ\x1a/
}
# Blizzard CASC Format game file
# Blizzard CASC Format game file
signature file-blizgame {
file-mime "application/x-blizgame", 100
file-magic /^BLTE/
@ -302,4 +302,3 @@ signature file-iso9660 {
file-mime "application/x-iso9660-image", 99
file-magic /CD001/
}

5
scripts/base/frameworks/files/magic/office.sig
View file

@ -1,7 +1,6 @@
# This signature is non-specific and terrible but after
# searching for a long time there doesn't seem to be a
# better option.
# searching for a long time there doesn't seem to be a
# better option.
signature file-msword {
file-magic /^\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1/
file-mime "application/msword", 50

8
scripts/base/frameworks/files/main.zeek
View file

@ -104,7 +104,7 @@ export {
missing_bytes: count &log &default=0;
## The number of bytes in the file stream that were not delivered to
## stream file analyzers. This could be overlapping bytes or
## stream file analyzers. This could be overlapping bytes or
## bytes that couldn't be reassembled.
overflow_bytes: count &log &default=0;
@ -150,7 +150,7 @@ export {
## f: the file.
global enable_reassembly: function(f: fa_file);
## Disables the file reassembler on this file. If the file is not
## Disables the file reassembler on this file. If the file is not
## transferred out of order this will have no effect.
##
## f: the file.
@ -266,7 +266,7 @@ export {
};
## Register callbacks for protocols that work with the Files framework.
## The callbacks must uniquely identify a file and each protocol can
## The callbacks must uniquely identify a file and each protocol can
## only have a single callback registered for it.
##
## tag: Tag for the protocol analyzer having a callback being registered.
@ -280,7 +280,7 @@ export {
## manipulation when they are being added to a file before the core code
## takes over. This is unlikely to be interesting for users and should
## only be called by file analyzer authors but is *not required*.
##
##
## tag: Tag for the file analyzer.
##
## callback: Function to execute when the given file analyzer is being added.

10
scripts/base/frameworks/intel/main.zeek
View file

@ -49,7 +49,7 @@ export {
## A URL for more information about the data.
url: string &optional;
};
## Represents a piece of intelligence.
type Item: record {
## The intelligence indicator.
@ -57,12 +57,12 @@ export {
## The type of data that the indicator field represents.
indicator_type: Type;
## Metadata for the item. Typically represents more deeply
## descriptive data for a piece of intelligence.
meta: MetaData;
};
## Enum to represent where data came from when it was discovered.
## The convention is to prefix the name with ``IN_``.
type Where: enum {
@ -158,8 +158,8 @@ export {
global extend_match: hook(info: Info, s: Seen, items: set[Item]);
## The expiration timeout for intelligence items. Once an item expires, the
## :zeek:id:`Intel::item_expired` hook is called. Reinsertion of an item
## resets the timeout. A negative value disables expiration of intelligence
## :zeek:id:`Intel::item_expired` hook is called. Reinsertion of an item
## resets the timeout. A negative value disables expiration of intelligence
## items.
const item_expiration = -1 min &redef;

5
scripts/base/frameworks/logging/writers/ascii.zeek
View file

@ -66,6 +66,11 @@ export {
## This option is also available as a per-filter ``$config`` option.
const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef;
## Handling of optional fields when writing out JSON. By default the
## JSON formatter skips key and val when the field is absent. Setting
## the following field to T includes the key, with a null value.
const json_include_unset_fields = F &redef;
## If true, include lines with log meta information such as column names
## with types, the values of ASCII logging options that are in use, and
## the time when the file was opened and closed (the latter at the end).

2
scripts/base/frameworks/netcontrol/plugin.zeek
View file

@ -41,7 +41,7 @@ export {
name: function(state: PluginState) : string;
## If true, plugin can expire rules itself. If false, the NetControl
## framework will manage rule expiration.
## framework will manage rule expiration.
can_expire: bool;
## One-time initialization function called when plugin gets registered, and

6
scripts/base/frameworks/netcontrol/plugins/debug.zeek
View file

@ -46,7 +46,7 @@ function debug_add_rule(p: PluginState, r: Rule) : bool
local s = fmt("add_rule: %s", r);
debug_log(p, s);
if ( do_something(p) )
if ( do_something(p) )
{
event NetControl::rule_added(r, p);
return T;
@ -76,12 +76,10 @@ global debug_plugin = Plugin(
function create_debug(do_something: bool) : PluginState
{
local p: PluginState = [$plugin=debug_plugin];
# FIXME: Why's the default not working?
p$config = table();
p$config["all"] = (do_something ? "1" : "0");
return p;
}

3
scripts/base/frameworks/netcontrol/plugins/packetfilter.zeek
View file

@ -1,7 +1,7 @@
##! NetControl plugin for the process-level PacketFilter that comes with
##! Zeek. Since the PacketFilter in Zeek is quite limited in scope
##! and can only add/remove filters for addresses, this is quite
##! limited in scope at the moment.
##! limited in scope at the moment.
@load ../plugin
@ -110,4 +110,3 @@ function create_packetfilter() : PluginState
return p;
}

2
scripts/base/frameworks/netcontrol/types.zeek
View file

@ -1,7 +1,7 @@
##! This file defines the types that are used by the NetControl framework.
##!
##! The most important type defined in this file is :zeek:see:`NetControl::Rule`,
##! which is used to describe all rules that can be expressed by the NetControl framework.
##! which is used to describe all rules that can be expressed by the NetControl framework.
module NetControl;

10
scripts/base/frameworks/notice/actions/add-geodata.zeek
View file

@ -1,6 +1,6 @@
##! This script adds geographic location data to notices for the "remote"
##! host in a connection. It does make the assumption that one of the
##! addresses in a connection is "local" and one is "remote" which is
##! host in a connection. It does make the assumption that one of the
##! addresses in a connection is "local" and one is "remote" which is
##! probably a safe assumption to make in most cases. If both addresses
##! are remote, it will use the $src address.
@ -17,13 +17,13 @@ export {
## in order for this to work.
ACTION_ADD_GEODATA
};
redef record Info += {
## If GeoIP support is built in, notices can have geographic
## information attached to them.
remote_location: geo_location &log &optional;
};
## Notice types which should have the "remote" location looked up.
## If GeoIP support is not built in, this does nothing.
option lookup_location_types: set[Notice::Type] = {};
@ -35,7 +35,7 @@ hook policy(n: Notice::Info) &priority=10
add n$actions[ACTION_ADD_GEODATA];
}
# This is handled at a high priority in case other notice handlers
# This is handled at a high priority in case other notice handlers
# want to use the data.
hook notice(n: Notice::Info) &priority=10
{

5
scripts/base/frameworks/notice/actions/email_admin.zeek
View file

@ -10,9 +10,9 @@ module Notice;
export {
redef enum Action += {
## Indicate that the generated email should be addressed to the
## Indicate that the generated email should be addressed to the
## appropriate email addresses as found by the
## :zeek:id:`Site::get_emails` function based on the relevant
## :zeek:id:`Site::get_emails` function based on the relevant
## address or addresses indicated in the notice.
ACTION_EMAIL_ADMIN
};
@ -23,7 +23,6 @@ hook notice(n: Notice::Info)
if ( |Site::local_admins| > 0 &&
ACTION_EMAIL_ADMIN in n$actions )
{
local email = "";
if ( n?$src && |Site::get_emails(n$src)| > 0 )
add n$email_dest[Site::get_emails(n$src)];
if ( n?$dst && |Site::get_emails(n$dst)| > 0 )

4
scripts/base/frameworks/openflow/cluster.zeek
View file

@ -112,12 +112,12 @@ function lookup_controller(name: string): vector of Controller
if ( Cluster::local_node_type() != Cluster::MANAGER )
return vector();
# I am not quite sure if we can actually get away with this - in the
# I am not quite sure if we can actually get away with this - in the
# current state, this means that the individual nodes cannot lookup
# a controller by name.
#
# This means that there can be no reactions to things on the actual
# worker nodes - because they cannot look up a name. On the other hand -
# worker nodes - because they cannot look up a name. On the other hand -
# currently we also do not even send the events to the worker nodes (at least
# not if we are using broker). Because of that I am not really feeling that
# badly about it...

25
scripts/base/frameworks/signatures/main.zeek
View file

@ -60,7 +60,7 @@ export {
SIG_ALARM_PER_ORIG,
## Alarm once and then never again.
SIG_ALARM_ONCE,
## Count signatures per responder host and alarm with the
## Count signatures per responder host and alarm with the
## :zeek:enum:`Signatures::Count_Signature` notice if a threshold
## defined by :zeek:id:`Signatures::count_thresholds` is reached.
SIG_COUNT_PER_RESP,
@ -100,15 +100,15 @@ export {
## Number of hosts, from a summary count.
host_count: count &log &optional;
};
## Actions for a signature.
## Actions for a signature.
const actions: table[string] of Action = {
["unspecified"] = SIG_IGNORE, # place-holder
} &redef &default = SIG_ALARM;
## Signature IDs that should always be ignored.
option ignored_ids = /NO_DEFAULT_MATCHES/;
## Generate a notice if, for a pair [orig, signature], the number of
## different responders has reached one of the thresholds.
const horiz_scan_thresholds = { 5, 10, 50, 100, 500, 1000 } &redef;
@ -120,7 +120,7 @@ export {
## Generate a notice if a :zeek:enum:`Signatures::SIG_COUNT_PER_RESP`
## signature is triggered as often as given by one of these thresholds.
const count_thresholds = { 5, 10, 50, 100, 500, 1000, 10000, 1000000, } &redef;
## The interval between when :zeek:enum:`Signatures::Signature_Summary`
## notices are generated.
option summary_interval = 1 day;
@ -147,7 +147,7 @@ event zeek_init() &priority=5
{
Log::create_stream(Signatures::LOG, [$columns=Info, $ev=log_signature, $path="signatures", $policy=log_policy]);
}
# Returns true if the given signature has already been triggered for the given
# [orig, resp] pair.
function has_signature_matched(id: string, orig: addr, resp: addr): bool
@ -173,7 +173,7 @@ event signature_match(state: signature_state, msg: string, data: string)
# Trim the matched data down to something reasonable
if ( |data| > 140 )
data = fmt("%s...", sub_bytes(data, 0, 140));
local src_addr: addr;
local src_port: port;
local dst_addr: addr;
@ -212,7 +212,7 @@ event signature_match(state: signature_state, msg: string, data: string)
local notice = F;
if ( action == SIG_ALARM )
notice = T;
if ( action == SIG_COUNT_PER_RESP )
{
local dst = state$conn$id$resp_h;
@ -252,7 +252,7 @@ event signature_match(state: signature_state, msg: string, data: string)
$conn=state$conn, $src=src_addr,
$dst=dst_addr, $msg=fmt("%s: %s", src_addr, msg),
$sub=data]);
if ( action == SIG_FILE_BUT_NO_SCAN || action == SIG_SUMMARY )
return;
@ -279,7 +279,7 @@ event signature_match(state: signature_state, msg: string, data: string)
fmt("%s has triggered signature %s on %d hosts",
orig, sig_id, hcount);
Log::write(Signatures::LOG,
Log::write(Signatures::LOG,
[$ts=network_time(), $note=Multiple_Sig_Responders,
$src_addr=orig, $sig_id=sig_id, $event_msg=msg,
$host_count=hcount, $sub_msg=horz_scan_msg]);
@ -296,9 +296,9 @@ event signature_match(state: signature_state, msg: string, data: string)
fmt("%s has triggered %d different signatures on host %s",
orig, vcount, resp);
Log::write(Signatures::LOG,
Log::write(Signatures::LOG,
[$ts=network_time(),
$note=Multiple_Signatures,
$note=Multiple_Signatures,
$src_addr=orig,
$dst_addr=resp, $sig_id=sig_id, $sig_count=vcount,
$event_msg=fmt("%s different signatures triggered", vcount),
@ -311,4 +311,3 @@ event signature_match(state: signature_state, msg: string, data: string)
last_vthresh[orig] = vcount;
}
}

68
scripts/base/frameworks/software/main.zeek
View file

@ -13,18 +13,18 @@ module Software;
export {
## The software logging stream identifier.
redef enum Log::ID += { LOG };
## A default logging policy hook for the stream.
global log_policy: Log::PolicyHook;
## Scripts detecting new types of software need to redef this enum to add
## their own specific software types which would then be used when they
## their own specific software types which would then be used when they
## create :zeek:type:`Software::Info` records.
type Type: enum {
## A placeholder type for when the type of software is not known.
UNKNOWN,
};
## A structure to represent the numeric version of software.
type Version: record {
## Major version number.
@ -38,7 +38,7 @@ export {
## Additional version string (e.g. "beta42").
addl: string &optional;
} &log;
## The record type that is used for representing and logging software.
type Info: record {
## The time at which the software was detected.
@ -58,9 +58,9 @@ export {
## parsing doesn't always work reliably in all cases and this
## acts as a fallback in the logs.
unparsed_version: string &log &optional;
## This can indicate that this software being detected should
## definitely be sent onward to the logging framework. By
## definitely be sent onward to the logging framework. By
## default, only software that is "interesting" due to a change
## in version or it being currently unknown is sent to the
## logging framework. This can be set to T to force the record
@ -68,7 +68,7 @@ export {
## tracking needs to happen in a specific way to the software.
force_log: bool &default=F;
};
## Hosts whose software should be detected and tracked.
## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS.
option asset_tracking = LOCAL_HOSTS;
@ -78,21 +78,21 @@ export {
## id: The connection id where the software was discovered.
##
## info: A record representing the software discovered.
##
##
## Returns: T if the software was logged, F otherwise.
global found: function(id: conn_id, info: Info): bool;
## Compare two version records.
##
##
## Returns: -1 for v1 < v2, 0 for v1 == v2, 1 for v1 > v2.
## If the numerical version numbers match, the *addl* string
## is compared lexicographically.
global cmp_versions: function(v1: Version, v2: Version): int;
## Sometimes software will expose itself on the network with
## slight naming variations. This table provides a mechanism
## for a piece of software to be renamed to a single name
## even if it exposes itself with an alternate name. The
## Sometimes software will expose itself on the network with
## slight naming variations. This table provides a mechanism
## for a piece of software to be renamed to a single name
## even if it exposes itself with an alternate name. The
## yielded string is the name that will be logged and generally
## used for everything.
global alternate_names: table[string] of string {
@ -100,17 +100,17 @@ export {
} &default=function(a: string): string { return a; };
## Type to represent a collection of :zeek:type:`Software::Info` records.
## It's indexed with the name of a piece of software such as "Firefox"
## It's indexed with the name of a piece of software such as "Firefox"
## and it yields a :zeek:type:`Software::Info` record with more
## information about the software.
type SoftwareSet: table[string] of Info;
## The set of software associated with an address. Data expires from
## this table after one day by default so that a detected piece of
## this table after one day by default so that a detected piece of
## software will be logged once each day. In a cluster, this table is
## uniformly distributed among proxy nodes.
global tracked: table[addr] of SoftwareSet &create_expire=1day;
## This event can be handled to access the :zeek:type:`Software::Info`
## record as it is sent on to the logging framework.
global log_software: event(rec: Info);
@ -128,7 +128,7 @@ event zeek_init() &priority=5
{
Log::create_stream(Software::LOG, [$columns=Info, $ev=log_software, $path="software", $policy=log_policy]);
}
type Description: record {
name: string;
version: Version;
@ -138,13 +138,13 @@ type Description: record {
# Defining this here because of a circular dependency between two functions.
global parse_mozilla: function(unparsed_version: string): Description;
# Don't even try to understand this now, just make sure the tests are
# Don't even try to understand this now, just make sure the tests are
# working.
function parse(unparsed_version: string): Description
{
local software_name = "<parse error>";
local v: Version;
# Parse browser-alike versions separately
if ( /^(Mozilla|Opera)\/[0-9]+\./ in unparsed_version )
{
@ -220,10 +220,10 @@ function parse(unparsed_version: string): Description
{
v$addl = strip(version_parts[2]);
}
}
}
if ( 3 in version_numbers && version_numbers[3] != "" )
v$minor3 = extract_count(version_numbers[3]);
if ( 2 in version_numbers && version_numbers[2] != "" )
@ -234,7 +234,7 @@ function parse(unparsed_version: string): Description
v$major = extract_count(version_numbers[0]);
}
}
return [$version=v, $unparsed_version=unparsed_version, $name=alternate_names[software_name]];
}
@ -245,7 +245,7 @@ function parse_with_cache(unparsed_version: string): Description
{
if (unparsed_version in parse_cache)
return parse_cache[unparsed_version];
local res = parse(unparsed_version);
parse_cache[unparsed_version] = res;
return res;
@ -256,7 +256,7 @@ function parse_mozilla(unparsed_version: string): Description
local software_name = "<unknown browser>";
local v: Version;
local parts: string_vec;
if ( /Opera [0-9\.]*$/ in unparsed_version )
{
software_name = "Opera";
@ -349,7 +349,7 @@ function parse_mozilla(unparsed_version: string): Description
if ( 2 in parts )
v = parse(parts[2])$version;
}
else if ( /AdobeAIR\/[0-9\.]*/ in unparsed_version )
{
software_name = "AdobeAIR";
@ -392,7 +392,7 @@ function cmp_versions(v1: Version, v2: Version): int
else
return v1?$major ? 1 : -1;
}
if ( v1?$minor && v2?$minor )
{
if ( v1$minor < v2$minor )
@ -407,7 +407,7 @@ function cmp_versions(v1: Version, v2: Version): int
else
return v1?$minor ? 1 : -1;
}
if ( v1?$minor2 && v2?$minor2 )
{
if ( v1$minor2 < v2$minor2 )
@ -462,7 +462,7 @@ function software_endpoint_name(id: conn_id, host: addr): string
# Convert a version into a string "a.b.c-x".
function software_fmt_version(v: Version): string
{
return fmt("%s%s%s%s%s",
return fmt("%s%s%s%s%s",
v?$major ? fmt("%d", v$major) : "0",
v?$minor ? fmt(".%d", v$minor) : "",
v?$minor2 ? fmt(".%d", v$minor2) : "",
@ -510,10 +510,10 @@ event Software::register(info: Info)
local changed = cmp_versions(old$version, info$version) != 0;
if ( changed )
event Software::version_change(old, info);
event Software::version_change(old, info);
else if ( ! info$force_log )
# If the version hasn't changed, then we're just redetecting the
# same thing, then we don't care.
# same thing, then we don't care.
return;
}
@ -526,7 +526,7 @@ function found(id: conn_id, info: Info): bool
if ( ! info$force_log && ! addr_matches_host(info$host, asset_tracking) )
return F;
if ( ! info?$ts )
if ( ! info?$ts )
info$ts = network_time();
if ( info?$version )

2
scripts/base/frameworks/sumstats/cluster.zeek
View file

@ -220,7 +220,7 @@ event zeek_init() &priority=100
# This variable is maintained by manager nodes as they collect and aggregate
# results.
# Index on a uid.
global stats_keys: table[string] of set[Key] &read_expire=1min
global stats_keys: table[string] of set[Key] &read_expire=1min
&expire_func=function(s: table[string] of set[Key], idx: string): interval
{
Reporter::warning(fmt("SumStat key request for the %s SumStat uid took longer than 1 minute and was automatically cancelled.", idx));

3
scripts/base/frameworks/sumstats/main.zeek
View file

@ -510,7 +510,7 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
return F;
# Add in the extra ResultVals to make threshold_vals easier to write.
# This length comparison should work because we just need to make
# This length comparison should work because we just need to make
# sure that we have the same number of reducers and results.
if ( |ss$reducers| != |result| )
{
@ -568,4 +568,3 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result)
ss$threshold_crossed(key, result);
}

2
scripts/base/frameworks/sumstats/plugins/sample.zeek
View file

@ -95,7 +95,7 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
{
local other_vector: vector of Observation;
local othercount: count;
if ( rv1$sample_elements > rv2$sample_elements )
{
result$samples = copy(rv1$samples);

2
scripts/base/frameworks/sumstats/plugins/unique.zeek
View file

@ -46,7 +46,7 @@ hook register_observe_plugins()
if ( ! r?$unique_max || |rv$unique_vals| <= r$unique_max )
add rv$unique_vals[obs];
rv$unique = |rv$unique_vals|;
});
}

11
scripts/base/frameworks/tunnels/main.zeek
View file

@ -90,20 +90,9 @@ export {
global finalize_tunnel: Conn::RemovalHook;
}
const ayiya_ports = { 5072/udp };
const teredo_ports = { 3544/udp };
const gtpv1_ports = { 2152/udp, 2123/udp };
redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports, vxlan_ports, geneve_ports };
event zeek_init() &priority=5
{
Log::create_stream(Tunnel::LOG, [$columns=Info, $path="tunnel", $policy=log_policy]);
Analyzer::register_for_ports(Analyzer::ANALYZER_AYIYA, ayiya_ports);
Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, teredo_ports);
Analyzer::register_for_ports(Analyzer::ANALYZER_GTPV1, gtpv1_ports);
Analyzer::register_for_ports(Analyzer::ANALYZER_VXLAN, vxlan_ports);
Analyzer::register_for_ports(Analyzer::ANALYZER_GENEVE, geneve_ports);
}
function register_all(ecv: EncapsulatingConnVector)

36
scripts/base/init-bare.zeek
View file

@ -1016,9 +1016,16 @@ const TCP_RESET = 6; ##< Endpoint has sent RST.
const UDP_INACTIVE = 0; ##< Endpoint is still inactive.
const UDP_ACTIVE = 1; ##< Endpoint has sent something.
## If true, don't verify checksums. Useful for running on altered trace
## files, and for saving a few cycles, but at the risk of analyzing invalid
## data. Note that the ``-C`` command-line option overrides the setting of this
## If true, don't verify checksums, and accept packets that give a length of
## zero in the IPv4 header. This is useful when running against traces of local
## traffic and the NIC checksum offloading feature is enabled. It can also
## be useful for running on altered trace files, and for saving a few cycles
## at the risk of analyzing invalid data.
## With this option, packets that have a value of zero in the total-length field
## of the IPv4 header are also accepted, and the capture-length is used instead.
## The total-length field is commonly set to zero when the NIC sequence offloading
## feature is enabled.
## Note that the ``-C`` command-line option overrides the setting of this
## variable.
const ignore_checksums = F &redef;
@ -3884,6 +3891,14 @@ type dns_loc_rr: record {
is_query: count; ##< The RR is a query/Response.
};
## DNS SVCB and HTTPS RRs
##
## .. zeek:see:: dns_SVCB dns_HTTPS
type dns_svcb_rr: record {
svc_priority: count; ##< Service priority for the current record, 0 indicates that this record is in AliasMode and cannot carry svc_params; otherwise this is in ServiceMode, and may include svc_params
target_name: string; ##< Target name, the hostname of the service endpoint.
};
# DNS answer types.
#
# .. zeek:see:: dns_answerr
@ -5021,14 +5036,14 @@ export {
## With this set, the Teredo analyzer waits until it sees both sides
## of a connection using a valid Teredo encapsulation before issuing
## a :zeek:see:`protocol_confirmation`. If it's false, the first
## a :zeek:see:`analyzer_confirmation`. If it's false, the first
## occurrence of a packet with valid Teredo encapsulation causes a
## confirmation.
const delay_teredo_confirmation = T &redef;
## With this set, the GTP analyzer waits until the most-recent upflow
## and downflow packets are a valid GTPv1 encapsulation before
## issuing :zeek:see:`protocol_confirmation`. If it's false, the
## issuing :zeek:see:`analyzer_confirmation`. If it's false, the
## first occurrence of a packet with valid GTPv1 encapsulation causes
## confirmation. Since the same inner connection can be carried
## differing outer upflow/downflow connections, setting to false
@ -5045,17 +5060,6 @@ export {
## may choose whether to perform the validation.
const validate_vxlan_checksums = T &redef;
## The set of UDP ports used for VXLAN traffic. Traffic using this
## UDP destination port will attempt to be decapsulated. Note that if
## if you customize this, you may still want to manually ensure that
## :zeek:see:`likely_server_ports` also gets populated accordingly.
const vxlan_ports: set[port] = { 4789/udp } &redef;
## The set of UDP ports used for Geneve traffic. Traffic using this
## UDP destination port will attempt to be decapsulated. Note that if
## if you customize this, you may still want to manually ensure that
## :zeek:see:`likely_server_ports` also gets populated accordingly.
const geneve_ports: set[port] = { 6081/udp } &redef;
} # end export
module Reporter;

12
scripts/base/packet-protocols/__load__.zeek
View file

@ -1,3 +1,5 @@
@load ./main.zeek
@load base/packet-protocols/root
@load base/packet-protocols/ip
@load base/packet-protocols/skip
@ -12,9 +14,15 @@
@load base/packet-protocols/pppoe
@load base/packet-protocols/vlan
@load base/packet-protocols/mpls
@load base/packet-protocols/gre
@load base/packet-protocols/iptunnel
@load base/packet-protocols/vntag
@load base/packet-protocols/udp
@load base/packet-protocols/tcp
@load base/packet-protocols/icmp
@load base/packet-protocols/gre
@load base/packet-protocols/iptunnel
@load base/packet-protocols/ayiya
@load base/packet-protocols/geneve
@load base/packet-protocols/vxlan
@load base/packet-protocols/teredo
@load base/packet-protocols/gtpv1

1
scripts/base/packet-protocols/ayiya/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

19
scripts/base/packet-protocols/ayiya/main.zeek Normal file
View file

@ -0,0 +1,19 @@
module PacketAnalyzer::AYIYA;
# Needed for port registration for BPF
@load base/frameworks/analyzer/main
const IPPROTO_IPV4 : count = 4;
const IPPROTO_IPV6 : count = 41;
const ayiya_ports = { 5072/udp };
redef likely_server_ports += { ayiya_ports };
event zeek_init() &priority=20
{
PacketAnalyzer::register_protocol_detection(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_AYIYA);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_AYIYA, IPPROTO_IPV4, PacketAnalyzer::ANALYZER_IP);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_AYIYA, IPPROTO_IPV6, PacketAnalyzer::ANALYZER_IP);
PacketAnalyzer::register_for_ports(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_AYIYA, ayiya_ports);
}

1
scripts/base/packet-protocols/geneve/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

27
scripts/base/packet-protocols/geneve/main.zeek Normal file
View file

@ -0,0 +1,27 @@
module PacketAnalyzer::Geneve;
export {
## The set of UDP ports used for Geneve traffic. Traffic using this
## UDP destination port will attempt to be decapsulated. Note that if
## if you customize this, you may still want to manually ensure that
## :zeek:see:`likely_server_ports` also gets populated accordingly.
const geneve_ports: set[port] = { 6081/udp } &redef;
}
redef likely_server_ports += { geneve_ports };
event zeek_init() &priority=20
{
PacketAnalyzer::register_for_ports(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_GENEVE, geneve_ports);
# This is defined by IANA as being "Trans Ether Bridging" but the Geneve RFC
# says to use it for Ethernet. See
# https://datatracker.ietf.org/doc/html/draft-gross-geneve-00#section-3.4
# for details.
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_GENEVE, 0x6558, PacketAnalyzer::ANALYZER_ETHERNET);
# Some additional mappings for protocols that we already handle natively.
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_GENEVE, 0x0800, PacketAnalyzer::ANALYZER_IP);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_GENEVE, 0x08DD, PacketAnalyzer::ANALYZER_IP);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_GENEVE, 0x0806, PacketAnalyzer::ANALYZER_ARP);
}

1
scripts/base/packet-protocols/gtpv1/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

28
scripts/base/packet-protocols/gtpv1/main.zeek Normal file
View file

@ -0,0 +1,28 @@
module PacketAnalyzer::GTPV1;
# This needs to be loaded here so the function is available. Function BIFs normally aren't
# loaded until after the packet analysis init scripts are run, and then zeek complains it
# can't find the function.
@load base/bif/plugins/Zeek_GTPv1.functions.bif
# Needed for port registration for BPF
@load base/frameworks/analyzer/main
export {
## Default analyzer
const default_analyzer: PacketAnalyzer::Tag = PacketAnalyzer::ANALYZER_IP &redef;
}
const gtpv1_ports = { 2152/udp, 2123/udp };
redef likely_server_ports += { gtpv1_ports };
event zeek_init() &priority=20
{
PacketAnalyzer::register_protocol_detection(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_GTPV1);
PacketAnalyzer::register_for_ports(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_GTPV1, gtpv1_ports);
}
event connection_state_remove(c: connection)
{
remove_gtpv1_connection(c$id);
}

61
scripts/base/packet-protocols/main.zeek Normal file
View file

@ -0,0 +1,61 @@
module PacketAnalyzer;
@load base/frameworks/analyzer/main.zeek
export {
## Registers a set of well-known ports for an analyzer. If a future
## connection on one of these ports is seen, the analyzer will be
## automatically assigned to parsing it. The function *adds* to all ports
## already registered, it doesn't replace them.
##
## tag: The tag of the analyzer.
##
## ports: The set of well-known ports to associate with the analyzer.
##
## Returns: True if the ports were successfully registered.
global register_for_ports: function(parent: PacketAnalyzer::Tag,
child: PacketAnalyzer::Tag,
ports: set[port]) : bool;
## Registers an individual well-known port for an analyzer. If a future
## connection on this port is seen, the analyzer will be automatically
## assigned to parsing it. The function *adds* to all ports already
## registered, it doesn't replace them.
##
## tag: The tag of the analyzer.
##
## p: The well-known port to associate with the analyzer.
##
## Returns: True if the port was successfully registered.
global register_for_port: function(parent: PacketAnalyzer::Tag,
child: PacketAnalyzer::Tag,
p: port) : bool;
}
function register_for_ports(parent: PacketAnalyzer::Tag,
child: PacketAnalyzer::Tag,
ports: set[port]) : bool
{
local rc = T;
for ( p in ports )
{
if ( ! register_for_port(parent, child, p) )
rc = F;
}
return rc;
}
function register_for_port(parent: PacketAnalyzer::Tag,
child: PacketAnalyzer::Tag,
p: port) : bool
{
register_packet_analyzer(parent, port_to_count(p), child);
if ( child !in Analyzer::ports )
Analyzer::ports[child] = set();
add Analyzer::ports[child][p];
return T;
}

1
scripts/base/packet-protocols/teredo/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

28
scripts/base/packet-protocols/teredo/main.zeek Normal file
View file

@ -0,0 +1,28 @@
module PacketAnalyzer::TEREDO;
# This needs to be loaded here so the functions are available. Function BIFs normally aren't
# loaded until after the packet analysis init scripts are run, and then zeek complains it
# can't find the function.
@load base/bif/plugins/Zeek_Teredo.functions.bif
# Needed for port registration for BPF
@load base/frameworks/analyzer/main
export {
## Default analyzer
const default_analyzer: PacketAnalyzer::Tag = PacketAnalyzer::ANALYZER_IP &redef;
}
const teredo_ports = { 3544/udp };
redef likely_server_ports += { teredo_ports };
event zeek_init() &priority=20
{
PacketAnalyzer::register_protocol_detection(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_TEREDO);
PacketAnalyzer::register_for_ports(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_TEREDO, teredo_ports);
}
event connection_state_remove(c: connection)
{
remove_teredo_connection(c$id);
}

1
scripts/base/packet-protocols/vxlan/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

20
scripts/base/packet-protocols/vxlan/main.zeek Normal file
View file

@ -0,0 +1,20 @@
module PacketAnalyzer::VXLAN;
export {
# There's no indicator in the VXLAN packet header format about what the next protocol
# in the chain is. All of the documentation just lists Ethernet, so default to that.
const default_analyzer: PacketAnalyzer::Tag = PacketAnalyzer::ANALYZER_ETHERNET &redef;
## The set of UDP ports used for VXLAN traffic. Traffic using this
## UDP destination port will attempt to be decapsulated. Note that if
## if you customize this, you may still want to manually ensure that
## :zeek:see:`likely_server_ports` also gets populated accordingly.
const vxlan_ports: set[port] = { 4789/udp } &redef;
}
redef likely_server_ports += { vxlan_ports };
event zeek_init() &priority=20
{
PacketAnalyzer::register_for_ports(PacketAnalyzer::ANALYZER_UDP, PacketAnalyzer::ANALYZER_VXLAN, vxlan_ports);
}

8
scripts/base/protocols/conn/contents.zeek
View file

@ -1,5 +1,5 @@
##! This script can be used to extract either the originator's data or the
##! responders data or both. By default nothing is extracted, and in order
##! This script can be used to extract either the originator's data or the
##! responders data or both. By default nothing is extracted, and in order
##! to actually extract data the ``c$extract_orig`` and/or the
##! ``c$extract_resp`` variable must be set to ``T``. One way to achieve this
##! would be to handle the :zeek:id:`connection_established` event elsewhere
@ -19,7 +19,7 @@ export {
## The prefix given to files containing extracted connections as they
## are opened on disk.
option extraction_prefix = "contents";
## If this variable is set to ``T``, then all contents of all
## connections will be extracted.
option default_extract = F;
@ -38,7 +38,7 @@ event connection_established(c: connection) &priority=-5
local orig_f = open(orig_file);
set_contents_file(c$id, CONTENTS_ORIG, orig_f);
}
if ( c$extract_resp )
{
local resp_file = generate_extraction_filename(extraction_prefix, c, "resp.dat");

10
scripts/base/protocols/conn/inactivity.zeek
View file

@ -6,19 +6,19 @@ module Conn;
export {
## Define inactivity timeouts by the service detected being used over
## the connection.
option analyzer_inactivity_timeouts: table[Analyzer::Tag] of interval = {
option analyzer_inactivity_timeouts: table[AllAnalyzers::Tag] of interval = {
# For interactive services, allow longer periods of inactivity.
[[Analyzer::ANALYZER_SSH, Analyzer::ANALYZER_FTP]] = 1 hrs,
};
## Define inactivity timeouts based on common protocol ports.
option port_inactivity_timeouts: table[port] of interval = {
[[21/tcp, 22/tcp, 23/tcp, 513/tcp]] = 1 hrs,
};
}
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count)
event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count)
{
if ( atype in analyzer_inactivity_timeouts )
set_inactivity_timeout(c$id, analyzer_inactivity_timeouts[atype]);

2
scripts/base/protocols/dce-rpc/main.zeek
View file

@ -17,7 +17,7 @@ export {
## The connection's 4-tuple of endpoint addresses/ports.
id : conn_id &log;
## Round trip time from the request to the response.
## If either the request or response wasn't seen,
## If either the request or response wasn't seen,
## this will be null.
rtt : interval &log &optional;

2
scripts/base/protocols/dhcp/main.zeek
View file

@ -78,7 +78,7 @@ export {
## The DHCP message types seen by this DHCP transaction
msg_types: vector of string &log &default=string_vec();
## Duration of the DHCP "session" representing the
## Duration of the DHCP "session" representing the
## time from the first message to the last.
duration: interval &log &default=0secs;

11
scripts/base/protocols/dns/consts.zeek
View file

@ -172,4 +172,15 @@ export {
[4] = "SHA384",
} &default = function(n: count): string { return fmt("digest-%d", n); };
## SVCB/HTTPS SvcParam keys, as defined in
## https://www.ietf.org/archive/id/draft-ietf-dnsop-svcb-https-07.txt, sec 14.3.2
const svcparam_keys = {
[0] = "mandatory",
[1] = "alpn",
[2] = "no-default-alpn",
[3] = "port",
[4] = "ipv4hint",
[5] = "ech",
[6] = "ipv6hint",
} &default = function(n: count): string { return fmt("key-%d", n); };
}

8
scripts/base/protocols/dns/main.zeek
View file

@ -375,7 +375,7 @@ hook DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
if ( ! c$dns?$rtt )
{
c$dns$rtt = network_time() - c$dns$ts;
# This could mean that only a reply was seen since
# This could mean that only a reply was seen since
# we assume there must be some passage of time between
# request and response.
if ( c$dns$rtt == 0secs )
@ -547,9 +547,9 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer, target: string
#
# }
# event dns_EDNS_ecs(c: connection, msg: dns_msg, opt: dns_edns_ecs)
# {
#
# }
# {
#
# }
#
#event dns_TSIG_addl(c: connection, msg: dns_msg, ans: dns_tsig_additional)
# {

6
scripts/base/protocols/ftp/files.zeek
View file

@ -18,14 +18,14 @@ export {
## Describe the file being transferred.
global describe_file: function(f: fa_file): string;
redef record fa_file += {
redef record fa_file += {
ftp: FTP::Info &optional;
};
}
function get_file_handle(c: connection, is_orig: bool): string
{
if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected )
if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected )
return "";
return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig);
@ -54,7 +54,7 @@ event zeek_init() &priority=5
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
{
if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected )
if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected )
return;
local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p];

20
scripts/base/protocols/ftp/utils-commands.zeek
View file

@ -11,12 +11,12 @@ export {
## Counter to track how many commands have been executed.
seq: count &default=0;
};
## Structure for tracking pending commands in the event that the client
## sends a large number of commands before the server has a chance to
## sends a large number of commands before the server has a chance to
## reply.
type PendingCmds: table[count] of CmdArg;
## Possible response codes for a wide variety of FTP commands.
option cmd_reply_code: set[string, count] = {
# According to RFC 959
@ -65,7 +65,7 @@ export {
["MDTM", [213, 500, 501, 550]], # RFC3659
["MLST", [150, 226, 250, 500, 501, 550]], # RFC3659
["MLSD", [150, 226, 250, 500, 501, 550]], # RFC3659
["CLNT", [200, 500]], # No RFC (indicate client software)
["MACB", [200, 500, 550]], # No RFC (test for MacBinary support)
@ -79,11 +79,11 @@ function add_pending_cmd(pc: PendingCmds, cmd: string, arg: string): CmdArg
{
local ca = [$cmd = cmd, $arg = arg, $seq=|pc|+1, $ts=network_time()];
pc[ca$seq] = ca;
return ca;
}
# Determine which is the best command to match with based on the
# Determine which is the best command to match with based on the
# response code and message.
function get_pending_cmd(pc: PendingCmds, reply_code: count, reply_msg: string): CmdArg
{
@ -94,18 +94,18 @@ function get_pending_cmd(pc: PendingCmds, reply_code: count, reply_msg: string):
for ( cmd_seq, cmd in pc )
{
local score: int = 0;
# if the command is compatible with the reply code
# code 500 (syntax error) is compatible with all commands
if ( reply_code == 500 || [cmd$cmd, reply_code] in cmd_reply_code )
score = score + 100;
# if the command or the command arg appears in the reply message
if ( strstr(reply_msg, cmd$cmd) > 0 )
score = score + 20;
if ( strstr(reply_msg, cmd$arg) > 0 )
score = score + 10;
if ( score > best_score ||
( score == best_score && best_seq > cmd_seq ) ) # break tie with sequence number
{
@ -132,7 +132,7 @@ function remove_pending_cmd(pc: PendingCmds, ca: CmdArg): bool
else
return F;
}
function pop_pending_cmd(pc: PendingCmds, reply_code: count, reply_msg: string): CmdArg
{
local ca = get_pending_cmd(pc, reply_code, reply_msg);

4
scripts/base/protocols/http/entities.zeek
View file

@ -97,7 +97,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
{
if ( f$source == "HTTP" && c?$http )
if ( f$source == "HTTP" && c?$http )
{
f$http = c$http;
@ -199,6 +199,6 @@ event file_sniff(f: fa_file, meta: fa_metadata) &priority=5
event http_end_entity(c: connection, is_orig: bool) &priority=5
{
if ( c?$http && c$http?$current_entity )
if ( c?$http && c$http?$current_entity )
delete c$http$current_entity;
}

8
scripts/base/protocols/http/utils.zeek
View file

@ -16,7 +16,7 @@ export {
##
## Returns: A vector of strings containing the keys.
global extract_keys: function(data: string, kv_splitter: pattern): string_vec;
## Creates a URL from an :zeek:type:`HTTP::Info` record. This should
## handle edge cases such as proxied requests appropriately.
##
@ -24,7 +24,7 @@ export {
##
## Returns: A URL, not prefixed by ``"http://"``.
global build_url: function(rec: Info): string;
## Creates a URL from an :zeek:type:`HTTP::Info` record. This should
## handle edge cases such as proxied requests appropriately.
##
@ -41,7 +41,7 @@ export {
function extract_keys(data: string, kv_splitter: pattern): string_vec
{
local key_vec: vector of string = vector();
local parts = split_string(data, kv_splitter);
for ( part_index in parts )
{
@ -64,7 +64,7 @@ function build_url(rec: Info): string
host = fmt("%s:%d", host, resp_p);
return fmt("%s%s", host, uri);
}
function build_url_http(rec: Info): string
{
return fmt("http://%s", build_url(rec));

2
scripts/base/protocols/irc/files.zeek
View file

@ -31,7 +31,7 @@ event zeek_init() &priority=5
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
{
if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers )
if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers )
return;
local irc = dcc_expected_transfers[c$id$resp_h, c$id$resp_p];

16
scripts/base/protocols/irc/main.zeek
View file

@ -1,11 +1,11 @@
##! Implements the core IRC analysis support. The logging model is to log
##! IRC commands along with the associated response and some additional
##! IRC commands along with the associated response and some additional
##! metadata about the connection if it's available.
module IRC;
export {
redef enum Log::ID += { LOG };
global log_policy: Log::PolicyHook;
@ -21,7 +21,7 @@ export {
nick: string &log &optional;
## Username given for the connection.
user: string &log &optional;
## Command given by the client.
command: string &log &optional;
## Value for the command given by the client.
@ -29,8 +29,8 @@ export {
## Any additional data for the command.
addl: string &log &optional;
};
## Event that can be handled to access the IRC record as it is sent on
## Event that can be handled to access the IRC record as it is sent on
## to the logging framework.
global irc_log: event(rec: Info);
}
@ -48,7 +48,7 @@ event zeek_init() &priority=5
Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log, $path="irc", $policy=log_policy]);
Analyzer::register_for_ports(Analyzer::ANALYZER_IRC, ports);
}
function new_session(c: connection): Info
{
local info: Info;
@ -57,12 +57,12 @@ function new_session(c: connection): Info
info$id = c$id;
return info;
}
function set_session(c: connection)
{
if ( ! c?$irc )
c$irc = new_session(c);
c$irc$ts=network_time();
}

10
scripts/base/protocols/krb/main.zeek
View file

@ -95,7 +95,7 @@ function set_session(c: connection): bool
$id = c$id);
Conn::register_removal_hook(c, finalize_krb);
}
return c$krb$logged;
}
@ -115,7 +115,7 @@ event krb_error(c: connection, msg: Error_Msg) &priority=5
if ( msg?$error_text && msg$error_text in ignored_errors )
{
if ( c?$krb )
if ( c?$krb )
delete c$krb;
return;
@ -174,7 +174,7 @@ event krb_as_response(c: connection, msg: KDC_Response) &priority=5
if ( ! c$krb?$client && ( msg?$client_name || msg?$client_realm ) )
{
c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "",
c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "",
msg?$client_realm ? msg$client_realm : "");
}
@ -202,7 +202,7 @@ event krb_tgs_request(c: connection, msg: KDC_Request) &priority=5
c$krb$request_type = "TGS";
if ( msg?$service_name )
c$krb$service = msg$service_name;
if ( msg?$from )
if ( msg?$from )
c$krb$from = msg$from;
if ( msg?$till )
c$krb$till = msg$till;
@ -221,7 +221,7 @@ event krb_tgs_response(c: connection, msg: KDC_Response) &priority=5
if ( ! c$krb?$client && ( msg?$client_name || msg?$client_realm ) )
{
c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "",
c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "",
msg?$client_realm ? msg$client_realm : "");
}

2
scripts/base/protocols/ntlm/main.zeek
View file

@ -33,7 +33,7 @@ export {
## Indicate whether or not the authentication was successful.
success : bool &log &optional;
## Internally used field to indicate if the login attempt
## Internally used field to indicate if the login attempt
## has already been logged.
done: bool &default=F;
};

4
scripts/base/protocols/radius/main.zeek
View file

@ -24,7 +24,7 @@ export {
mac : string &log &optional;
## The address given to the network access server, if
## present. This is only a hint from the RADIUS server
## and the network access server is not required to honor
## and the network access server is not required to honor
## the address.
framed_addr : addr &log &optional;
## Address (IPv4, IPv6, or FQDN) of the initiator end of the tunnel,
@ -33,7 +33,7 @@ export {
tunnel_client: string &log &optional;
## Connect info, if present.
connect_info : string &log &optional;
## Reply message from the server challenge. This is
## Reply message from the server challenge. This is
## frequently shown to the user authenticating.
reply_msg : string &log &optional;
## Successful or failed authentication.

26
scripts/base/protocols/rdp/main.zeek
View file

@ -41,15 +41,15 @@ export {
desktop_width: count &log &optional;
## Desktop height of the client machine.
desktop_height: count &log &optional;
## The color depth requested by the client in
## The color depth requested by the client in
## the high_color_depth field.
requested_color_depth: string &log &optional;
## If the connection is being encrypted with native
## RDP encryption, this is the type of cert
## RDP encryption, this is the type of cert
## being used.
cert_type: string &log &optional;
## The number of certs seen. X.509 can transfer an
## The number of certs seen. X.509 can transfer an
## entire certificate chain.
cert_count: count &log &default=0;
## Indicates if the provided certificate or certificate
@ -57,7 +57,7 @@ export {
cert_permanent: bool &log &optional;
## Encryption level of the connection.
encryption_level: string &log &optional;
## Encryption method of the connection.
## Encryption method of the connection.
encryption_method: string &log &optional;
};
@ -65,7 +65,7 @@ export {
## continuing to process encrypted traffic.
option disable_analyzer_after_detection = F;
## The amount of time to monitor an RDP session from when it is first
## The amount of time to monitor an RDP session from when it is first
## identified. When this interval is reached, the session is logged.
option rdp_check_interval = 10secs;
@ -113,7 +113,7 @@ function write_log(c: connection)
info$done = T;
# Verify that the RDP session contains
# RDP data before writing it to the log.
# RDP data before writing it to the log.
if ( info?$cookie || info?$keyboard_layout || info?$result )
Log::write(RDP::LOG, info);
}
@ -124,16 +124,16 @@ event check_record(c: connection)
if ( c$rdp$done )
return;
# If the value rdp_check_interval has passed since the
# RDP session was started, then log the record.
# If the value rdp_check_interval has passed since the
# RDP session was started, then log the record.
local diff = network_time() - c$rdp$ts;
if ( diff > rdp_check_interval )
{
write_log(c);
# Remove the analyzer if it is still attached.
if ( disable_analyzer_after_detection &&
connection_exists(c$id) &&
if ( disable_analyzer_after_detection &&
connection_exists(c$id) &&
c$rdp?$analyzer_id )
{
disable_analyzer(c$id, c$rdp$analyzer_id);
@ -240,7 +240,7 @@ event rdp_server_certificate(c: connection, cert_type: count, permanently_issued
# now so we manually count this one.
if ( c$rdp$cert_type == "RSA" )
++c$rdp$cert_count;
c$rdp$cert_permanent = permanently_issued;
}
@ -265,7 +265,7 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
}
}
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=5
event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) &priority=5
{
if ( atype == Analyzer::ANALYZER_RDP )
{
@ -274,7 +274,7 @@ event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &pr
}
}
event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason: string) &priority=5
event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count, reason: string) &priority=5
{
# If a protocol violation occurs, then log the record immediately.
if ( c?$rdp )

62
scripts/base/protocols/smb/consts.zeek
View file

@ -107,13 +107,13 @@ export {
} &redef &default=function(i: count):string { return fmt("unknown-wksta-command-%d", i); };
type rpc_cmd_table: table[count] of string;
## The subcommands for RPC endpoints.
const rpc_sub_cmds: table[string] of rpc_cmd_table = {
["4b324fc8-1670-01d3-1278-5a47bf6ee188"] = srv_cmds,
["6bffd098-a112-3610-9833-46c3f87e345a"] = wksta_cmds,
["6bffd098-a112-3610-9833-46c3f87e345a"] = wksta_cmds,
} &redef &default=function(i: string):rpc_cmd_table { return table() &default=function(j: string):string { return fmt("unknown-uuid-%s", j); }; };
}
module SMB1;
@ -195,37 +195,37 @@ export {
} &default=function(i: count):string { return fmt("unknown-%d", i); };
const trans2_sub_commands: table[count] of string = {
[0x00] = "OPEN2",
[0x01] = "FIND_FIRST2",
[0x02] = "FIND_NEXT2",
[0x03] = "QUERY_FS_INFORMATION",
[0x04] = "SET_FS_INFORMATION",
[0x05] = "QUERY_PATH_INFORMATION",
[0x06] = "SET_PATH_INFORMATION",
[0x07] = "QUERY_FILE_INFORMATION",
[0x08] = "SET_FILE_INFORMATION",
[0x09] = "FSCTL",
[0x0A] = "IOCTL",
[0x0B] = "FIND_NOTIFY_FIRST",
[0x0C] = "FIND_NOTIFY_NEXT",
[0x0D] = "CREATE_DIRECTORY",
[0x0E] = "SESSION_SETUP",
[0x10] = "GET_DFS_REFERRAL",
[0x11] = "REPORT_DFS_INCONSISTENCY",
[0x00] = "OPEN2",
[0x01] = "FIND_FIRST2",
[0x02] = "FIND_NEXT2",
[0x03] = "QUERY_FS_INFORMATION",
[0x04] = "SET_FS_INFORMATION",
[0x05] = "QUERY_PATH_INFORMATION",
[0x06] = "SET_PATH_INFORMATION",
[0x07] = "QUERY_FILE_INFORMATION",
[0x08] = "SET_FILE_INFORMATION",
[0x09] = "FSCTL",
[0x0A] = "IOCTL",
[0x0B] = "FIND_NOTIFY_FIRST",
[0x0C] = "FIND_NOTIFY_NEXT",
[0x0D] = "CREATE_DIRECTORY",
[0x0E] = "SESSION_SETUP",
[0x10] = "GET_DFS_REFERRAL",
[0x11] = "REPORT_DFS_INCONSISTENCY",
} &default=function(i: count):string { return fmt("unknown-trans2-sub-cmd-%d", i); };
const trans_sub_commands: table[count] of string = {
[0x01] = "SET_NMPIPE_STATE",
[0x11] = "RAW_READ_NMPIPE",
[0x21] = "QUERY_NMPIPE_STATE",
[0x22] = "QUERY_NMPIPE_INFO",
[0x23] = "PEEK_NMPIPE",
[0x26] = "TRANSACT_NMPIPE",
[0x31] = "RAW_WRITE_NMPIPE",
[0x36] = "READ_NMPIPE",
[0x37] = "WRITE_NMPIPE",
[0x53] = "WAIT_NMPIPE",
[0x54] = "CALL_NMPIPE",
[0x01] = "SET_NMPIPE_STATE",
[0x11] = "RAW_READ_NMPIPE",
[0x21] = "QUERY_NMPIPE_STATE",
[0x22] = "QUERY_NMPIPE_INFO",
[0x23] = "PEEK_NMPIPE",
[0x26] = "TRANSACT_NMPIPE",
[0x31] = "RAW_WRITE_NMPIPE",
[0x36] = "READ_NMPIPE",
[0x37] = "WRITE_NMPIPE",
[0x53] = "WAIT_NMPIPE",
[0x54] = "CALL_NMPIPE",
} &default=function(i: count):string { return fmt("unknown-trans-sub-cmd-%d", i); };
}

2
scripts/base/protocols/smb/files.zeek
View file

@ -14,7 +14,7 @@ export {
function get_file_handle(c: connection, is_orig: bool): string
{
if ( ! (c$smb_state?$current_file &&
(c$smb_state$current_file?$name ||
(c$smb_state$current_file?$name ||
c$smb_state$current_file?$path)) )
{
# TODO - figure out what are the cases where this happens.

26
scripts/base/protocols/smb/main.zeek
View file

@ -5,7 +5,7 @@
module SMB;
export {
redef enum Log::ID += {
redef enum Log::ID += {
AUTH_LOG,
MAPPING_LOG,
FILES_LOG
@ -13,7 +13,7 @@ export {
global log_policy_files: Log::PolicyHook;
global log_policy_mapping: Log::PolicyHook;
## Abstracted actions for SMB file actions.
type Action: enum {
FILE_READ,
@ -55,7 +55,7 @@ export {
id : conn_id &log;
## Unique ID of the file.
fuid : string &log &optional;
## Action this log record represents.
action : Action &log &optional;
## Path pulled from the tree this file was transferred to or from.
@ -99,14 +99,14 @@ export {
uid : string &log;
## ID of the connection the request was sent over.
id : conn_id &log;
## The command sent by the client.
command : string &log;
## The subcommand sent by the client, if present.
sub_command : string &log &optional;
## Command argument sent by the client, if any.
argument : string &log &optional;
## Server reply to the client's command.
status : string &log &optional;
## Round trip time from the request to the response.
@ -116,13 +116,13 @@ export {
## Authenticated username, if available.
username : string &log &optional;
## If this is related to a tree, this is the tree
## that was used for the current command.
tree : string &log &optional;
## The type of tree (disk share, printer share, named pipe, etc.).
tree_service : string &log &optional;
## If the command referenced a file, store it here.
referenced_file : FileInfo &log &optional;
## If the command referenced a tree, store it here.
@ -138,7 +138,7 @@ export {
current_file : FileInfo &optional;
## A reference to the current tree.
current_tree : TreeInfo &optional;
## Indexed on MID to map responses to requests.
pending_cmds : table[count] of CmdInfo &optional;
## File map to retrieve file information based on the file ID.
@ -161,7 +161,7 @@ export {
redef record connection += {
smb_state : State &optional;
};
## This is an internally used function.
const set_current_file: function(smb_state: State, file_id: count) &redef;
@ -195,7 +195,7 @@ function set_current_file(smb_state: State, file_id: count)
smb_state$fid_map[file_id] = smb_state$current_cmd$referenced_file;
smb_state$fid_map[file_id]$fid = file_id;
}
smb_state$current_cmd$referenced_file = smb_state$fid_map[file_id];
smb_state$current_file = smb_state$current_cmd$referenced_file;
}
@ -203,7 +203,7 @@ function set_current_file(smb_state: State, file_id: count)
function write_file_log(state: State)
{
local f = state$current_file;
if ( f?$name &&
if ( f?$name &&
f$action in logged_file_actions )
{
# Everything in this if statement is to avoid overlogging
@ -225,7 +225,7 @@ function write_file_log(state: State)
else
add state$recent_files[file_ident];
}
Log::write(FILES_LOG, f);
}
}
@ -240,7 +240,7 @@ event file_state_remove(f: fa_file) &priority=-5
{
if ( f$source != "SMB" )
return;
for ( id, c in f$conns )
{
if ( c?$smb_state && c$smb_state?$current_file)

38
scripts/base/protocols/smb/smb1-main.zeek
View file

@ -39,12 +39,12 @@ event smb1_message(c: connection, hdr: SMB1::Header, is_orig: bool) &priority=5
{
smb_state$current_cmd$tree = smb_state$current_tree$path;
}
if ( smb_state$current_tree?$service )
{
smb_state$current_cmd$tree_service = smb_state$current_tree$service;
}
if ( mid !in smb_state$pending_cmds )
{
local tmp_cmd = SMB::CmdInfo($uid=c$uid, $id=c$id, $version="SMB1", $command = SMB1::commands[hdr$command]);
@ -52,10 +52,10 @@ event smb1_message(c: connection, hdr: SMB1::Header, is_orig: bool) &priority=5
local tmp_file = SMB::FileInfo($uid=c$uid, $id=c$id);
tmp_cmd$referenced_file = tmp_file;
tmp_cmd$referenced_tree = smb_state$current_tree;
smb_state$pending_cmds[mid] = tmp_cmd;
}
smb_state$current_cmd = smb_state$pending_cmds[mid];
if ( !is_orig )
@ -97,11 +97,11 @@ event smb1_negotiate_response(c: connection, hdr: SMB1::Header, response: SMB1::
delete c$smb_state$current_cmd$smb1_offered_dialects;
}
}
event smb1_negotiate_response(c: connection, hdr: SMB1::Header, response: SMB1::NegotiateResponse) &priority=-5
{
}
event smb1_tree_connect_andx_request(c: connection, hdr: SMB1::Header, path: string, service: string) &priority=5
{
local tmp_tree = SMB::TreeInfo($uid=c$uid, $id=c$id, $path=path, $service=service);
@ -117,7 +117,7 @@ event smb1_tree_connect_andx_response(c: connection, hdr: SMB1::Header, service:
c$smb_state$current_cmd$referenced_tree$share_type = "PIPE";
c$smb_state$current_cmd$tree_service = service;
if ( native_file_system != "" )
c$smb_state$current_cmd$referenced_tree$native_file_system = native_file_system;
@ -150,13 +150,13 @@ event smb1_nt_create_andx_response(c: connection, hdr: SMB1::Header, file_id: co
# I'm seeing negative data from IPC tree transfers
if ( time_to_double(times$modified) > 0.0 )
c$smb_state$current_cmd$referenced_file$times = times;
# We can identify the file by its file id now so let's stick it
# We can identify the file by its file id now so let's stick it
# in the file map.
c$smb_state$fid_map[file_id] = c$smb_state$current_cmd$referenced_file;
c$smb_state$current_file = c$smb_state$fid_map[file_id];
SMB::write_file_log(c$smb_state);
}
@ -167,7 +167,7 @@ event smb1_read_andx_request(c: connection, hdr: SMB1::Header, file_id: count, o
if ( c$smb_state$current_file?$name )
c$smb_state$current_cmd$argument = c$smb_state$current_file$name;
}
event smb1_read_andx_request(c: connection, hdr: SMB1::Header, file_id: count, offset: count, length: count) &priority=-5
{
if ( c$smb_state$current_tree?$path && !c$smb_state$current_file?$path )
@ -180,12 +180,12 @@ event smb1_write_andx_request(c: connection, hdr: SMB1::Header, file_id: count,
{
SMB::set_current_file(c$smb_state, file_id);
c$smb_state$current_file$action = SMB::FILE_WRITE;
if ( !c$smb_state$current_cmd?$argument &&
if ( !c$smb_state$current_cmd?$argument &&
# TODO: figure out why name isn't getting set sometimes.
c$smb_state$current_file?$name )
c$smb_state$current_cmd$argument = c$smb_state$current_file$name;
}
event smb1_write_andx_request(c: connection, hdr: SMB1::Header, file_id: count, offset: count, data_len: count) &priority=-5
{
if ( c$smb_state$current_tree?$path && !c$smb_state$current_file?$path )
@ -217,7 +217,7 @@ event smb1_close_request(c: connection, hdr: SMB1::Header, file_id: count) &prio
if ( fl?$name )
c$smb_state$current_cmd$argument = fl$name;
delete c$smb_state$fid_map[file_id];
SMB::write_file_log(c$smb_state);
@ -254,7 +254,7 @@ event smb1_session_setup_andx_response(c: connection, hdr: SMB1::Header, respons
{
# No behavior yet.
}
event smb1_transaction_request(c: connection, hdr: SMB1::Header, name: string, sub_cmd: count, parameters: string, data: string)
{
c$smb_state$current_cmd$sub_command = SMB1::trans_sub_commands[sub_cmd];
@ -267,7 +267,7 @@ event smb1_write_andx_request(c: connection, hdr: SMB1::Header, file_id: count,
# TODO: figure out why the uuid isn't getting set sometimes.
return;
}
c$smb_state$pipe_map[file_id] = c$smb_state$current_file$uuid;
}
@ -278,11 +278,11 @@ event smb_pipe_bind_ack_response(c: connection, hdr: SMB1::Header)
# TODO: figure out why the uuid isn't getting set sometimes.
return;
}
c$smb_state$current_cmd$sub_command = "RPC_BIND_ACK";
c$smb_state$current_cmd$argument = SMB::rpc_uuids[c$smb_state$current_file$uuid];
}
event smb_pipe_bind_request(c: connection, hdr: SMB1::Header, uuid: string, version: string)
{
if ( ! c$smb_state?$current_file || ! c$smb_state$current_file?$uuid )

10
scripts/base/protocols/smb/smb2-main.zeek
View file

@ -19,7 +19,7 @@ event smb2_message(c: connection, hdr: SMB2::Header, is_orig: bool) &priority=5
state$pipe_map = table();
c$smb_state = state;
}
local smb_state = c$smb_state;
local tid = hdr$tree_id;
local mid = hdr$message_id;
@ -159,10 +159,10 @@ event smb2_create_response(c: connection, hdr: SMB2::Header, response: SMB2::Cre
if ( time_to_double(response$times$modified) > 0.0 )
c$smb_state$current_file$times = response$times;
# We can identify the file by its file id now so let's stick it
# We can identify the file by its file id now so let's stick it
# in the file map.
c$smb_state$fid_map[response$file_id$persistent+response$file_id$volatile] = c$smb_state$current_file;
c$smb_state$current_file = c$smb_state$fid_map[response$file_id$persistent+response$file_id$volatile];
}
@ -193,7 +193,7 @@ event smb2_read_request(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, o
}
event smb2_read_request(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, offset: count, length: count) &priority=-5
{
{
SMB::write_file_log(c$smb_state);
}
@ -249,7 +249,7 @@ event smb2_file_rename(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, ds
if ( c$smb_state$current_file?$name )
c$smb_state$current_file$prev_name = c$smb_state$current_file$name;
c$smb_state$current_file$name = dst_filename;
switch ( c$smb_state$current_tree$share_type )

2
scripts/base/protocols/ssh/main.zeek
View file

@ -355,7 +355,7 @@ event ssh_server_host_key(c: connection, hash: string) &priority=5
c$ssh$host_key = hash;
}
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=20
event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) &priority=20
{
if ( atype == Analyzer::ANALYZER_SSH )
{

4
scripts/base/protocols/ssl/main.zeek
View file

@ -474,7 +474,7 @@ hook finalize_ssl(c: connection)
finish(c, F);
}
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=5
event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) &priority=5
{
if ( atype == Analyzer::ANALYZER_SSL || atype == Analyzer::ANALYZER_DTLS )
{
@ -494,7 +494,7 @@ event ssl_plaintext_data(c: connection, is_orig: bool, record_version: count, co
Weird::weird(wi);
}
event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count,
reason: string) &priority=5
{
if ( c?$ssl && ( atype == Analyzer::ANALYZER_SSL || atype == Analyzer::ANALYZER_DTLS ) )

2
scripts/base/protocols/syslog/consts.zeek
View file

@ -31,7 +31,7 @@ export {
[23] = "LOCAL7",
[999] = "UNSPECIFIED",
} &default=function(c: count): string { return fmt("?-%d", c); };
## Mapping between the constants and string values for syslog severities.
const severity_codes: table[count] of string = {
[0] = "EMERG",

4
scripts/base/protocols/syslog/main.zeek
View file

@ -1,4 +1,4 @@
##! Core script support for logging syslog messages. This script represents
##! Core script support for logging syslog messages. This script represents
##! one syslog message as one logged record.
@load ./consts
@ -52,7 +52,7 @@ event syslog_message(c: connection, facility: count, severity: count, msg: strin
info$facility=facility_codes[facility];
info$severity=severity_codes[severity];
info$message=msg;
c$syslog = info;
}

12
scripts/base/protocols/tunnels/dpd.sig
View file

@ -1,14 +1,2 @@
# Provide DPD signatures for tunneling protocols that otherwise
# wouldn't be detected at all.
signature dpd_ayiya {
ip-proto = udp
payload /^..\x11\x29/
enable "ayiya"
}
signature dpd_teredo {
ip-proto = udp
payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f].{7}((\x20\x01\x00\x00)).{28})|([\x60-\x6f].{23}((\x20\x01\x00\x00))).{12}/
enable "teredo"
}

8
scripts/base/utils/conn-ids.zeek
View file

@ -3,16 +3,16 @@
module GLOBAL;
export {
## Takes a conn_id record and returns a string representation with the
## Takes a conn_id record and returns a string representation with the
## general data flow appearing to be from the connection originator
## on the left to the responder on the right.
global id_string: function(id: conn_id): string;
## Takes a conn_id record and returns a string representation with the
## Takes a conn_id record and returns a string representation with the
## general data flow appearing to be from the connection responder
## on the right to the originator on the left.
global reverse_id_string: function(id: conn_id): string;
## Calls :zeek:id:`id_string` or :zeek:id:`reverse_id_string` if the
## second argument is T or F, respectively.
global directed_id_string: function(id: conn_id, is_orig: bool): string;

2
scripts/base/utils/directions-and-hosts.zeek
View file

@ -58,7 +58,7 @@ type Host: enum {
function addr_matches_host(ip: addr, h: Host): bool
{
if ( h == NO_HOSTS ) return F;
return ( h == ALL_HOSTS ||
(h == LOCAL_HOSTS && Site::is_local_addr(ip)) ||
(h == REMOTE_HOSTS && !Site::is_local_addr(ip)) );

5
scripts/base/utils/numbers.zeek
View file

@ -1,8 +1,7 @@
## Extract an integer from a string.
##
##
## s: The string to search for a number.
##
##
## get_first: Provide `F` if you would like the last number found.
##
## Returns: The request integer from the given string or 0 if

2
scripts/base/utils/patterns.zeek
View file

@ -27,7 +27,7 @@ function set_to_regex(ss: set[string], pat: string): pattern
for ( s in ss )
{
local tmp_pattern = convert_for_pattern(s);
return_pat = ( i == 0 ) ?
return_pat = ( i == 0 ) ?
tmp_pattern : cat(tmp_pattern, "|", return_pat);
++i;
}

2
scripts/base/utils/strings.zeek
View file

@ -25,7 +25,7 @@ function join_string_set(ss: set[string], j: string): string
{
if ( i > 0 )
output = cat(output, j);
output = cat(output, s);
++i;
}

8
scripts/base/utils/thresholds.zeek
View file

@ -16,13 +16,13 @@ export {
## for.
index: count &default=0;
};
## The thresholds you would like to use as defaults with the
## The thresholds you would like to use as defaults with the
## :zeek:id:`default_check_threshold` function.
const default_notice_thresholds: vector of count = {
30, 100, 1000, 10000, 100000, 1000000, 10000000,
} &redef;
## This will check if a :zeek:type:`TrackCount` variable has crossed any
## thresholds in a given set.
##
@ -33,7 +33,7 @@ export {
##
## Returns: T if a threshold has been crossed, else F.
global check_threshold: function(v: vector of count, tracker: TrackCount): bool;
## This will use the :zeek:id:`default_notice_thresholds` variable to
## check a :zeek:type:`TrackCount` variable to see if it has crossed
## another threshold.

3
scripts/policy/files/unified2/main.zeek
View file

@ -1,4 +1,3 @@
@load base/utils/dir
@load base/utils/paths
@ -255,7 +254,7 @@ event file_new(f: fa_file)
if ( |parts| == 3 )
file_dir = parts[0];
if ( (watch_file != "" && f$source == watch_file) ||
if ( (watch_file != "" && f$source == watch_file) ||
(watch_dir != "" && compress_path(watch_dir) == file_dir) )
{
Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);

5
scripts/policy/frameworks/cluster/agent/__load__.zeek
View file

@ -1,5 +1,4 @@
# The entry point for the cluster agent. It only runs bootstrap logic for
# launching via the Supervisor. If we're not running the Supervisor, this does
# nothing.
##! The entry point for the cluster agent. It runs bootstrap logic for launching
##! the agent process via Zeek's Supervisor.
@load ./boot

88
scripts/policy/frameworks/cluster/agent/api.zeek
View file

@ -1,24 +1,108 @@
##! The event API of cluster agents. Most endpoints consist of event pairs,
##! where the agent answers a request event with a corresponding response
##! event. Such event pairs share the same name prefix and end in "_request" and
##! "_response", respectively.
@load base/frameworks/supervisor/control
@load policy/frameworks/cluster/controller/types
module ClusterAgent::API;
export {
## A simple versioning scheme, used to track basic compatibility of
## controller and agent.
const version = 1;
# Agent API events
## The controller sends this event to convey a new cluster configuration
## to the agent. Once processed, the agent responds with the response
## event.
##
## reqid: a request identifier string, echoed in the response event.
##
## config: a :zeek:see:`ClusterController::Types::Configuration` record
## describing the cluster topology. Note that this contains the full
## topology, not just the part pertaining to this agent. That's because
## the cluster framework requires full cluster visibility to establish
## the needed peerings.
##
global set_configuration_request: event(reqid: string,
config: ClusterController::Types::Configuration);
## Response to a set_configuration_request event. The agent sends
## this back to the controller.
##
## reqid: the request identifier used in the request event.
##
## result: the result record.
##
global set_configuration_response: event(reqid: string,
result: ClusterController::Types::Result);
## The controller sends this event to confirm to the agent that it is
## part of the current cluster topology. The agent acknowledges with the
## corresponding response event.
##
## reqid: a request identifier string, echoed in the response event.
##
global agent_welcome_request: event(reqid: string);
## Response to an agent_welcome_request event. The agent sends this
## back to the controller.
##
## reqid: the request identifier used in the request event.
##
## result: the result record.
##
global agent_welcome_response: event(reqid: string,
result: ClusterController::Types::Result);
## The controller sends this event to convey that the agent is not
## currently required. This status may later change, depending on
## updates from the client, so the Broker-level peering can remain
## active. The agent releases any cluster-related resources (including
## shutdown of existing Zeek cluster nodes) when processing the request,
## and confirms via the response event. Shutting down an agent at this
## point has no operational impact on the running cluster.
##
## reqid: a request identifier string, echoed in the response event.
##
global agent_standby_request: event(reqid: string);
## Response to an agent_standby_request event. The agent sends this
## back to the controller.
##
## reqid: the request identifier used in the request event.
##
## result: the result record.
##
global agent_standby_response: event(reqid: string,
result: ClusterController::Types::Result);
# Notification events, agent -> controller
# Report agent being available.
## The agent sends this event upon peering as a "check-in", informing
## the controller that an agent of the given name is now available to
## communicate with. It is a controller-level equivalent of
## `:zeek:see:`Broker::peer_added`.
##
## instance: an instance name, really the agent's name as per :zeek:see:`ClusterAgent::name`.
##
## host: the IP address of the agent. (This may change in the future.)
##
## api_version: the API version of this agent.
##
global notify_agent_hello: event(instance: string, host: addr,
api_version: count);
# The following are not yet implemented.
# Report node state changes.
global notify_change: event(instance: string,
n: ClusterController::Types::Node,
@ -30,4 +114,4 @@ export {
# Report informational message.
global notify_log: event(instance: string, msg: string, node: string &default="");
}
}

6
scripts/policy/frameworks/cluster/agent/boot.zeek
View file

@ -1,3 +1,9 @@
##! The cluster agent boot logic runs in Zeek's supervisor and instructs it to
##! launch an agent process. The agent's main logic resides in main.zeek,
##! similarly to other frameworks. The new process will execute that script.
##!
##! If the current process is not the Zeek supervisor, this does nothing.
@load ./config
# The agent needs the supervisor to listen for node management requests. We

80
scripts/policy/frameworks/cluster/agent/config.zeek
View file

@ -1,51 +1,83 @@
##! Configuration settings for a cluster agent.
@load policy/frameworks/cluster/controller/types
module ClusterAgent;
export {
# The name this agent uses to represent the cluster instance
# it manages. When the environment variable isn't set and there's,
# no redef, this falls back to "agent-<hostname>".
## The name this agent uses to represent the cluster instance it
## manages. Defaults to the value of the ZEEK_AGENT_NAME environment
## variable. When that is unset and you don't redef the value,
## the implementation defaults to "agent-<hostname>".
const name = getenv("ZEEK_AGENT_NAME") &redef;
# Agent stdout/stderr log files to produce in Zeek's working
# directory. If empty, no such logs will result. The actual
# log files have the agent's name (as per above) dot-prefixed.
## Agent stdout log configuration. If the string is non-empty, Zeek will
## produce a free-form log (i.e., not one governed by Zeek's logging
## framework) in Zeek's working directory. The final log's name is
## "<name>.<suffix>", where the name is taken from :zeek:see:`ClusterAgent::name`,
## and the suffix is defined by the following variable. If left empty,
## no such log results.
##
## Note that the agent also establishes a "proper" Zeek log via the
## :zeek:see:`ClusterController::Log` module.
const stdout_file_suffix = "agent.stdout" &redef;
## Agent stderr log configuration. Like :zeek:see:`ClusterAgent::stdout_file_suffix`,
## but for the stderr stream.
const stderr_file_suffix = "agent.stderr" &redef;
# The address and port the agent listens on. When
# undefined, falls back to configurable default values.
## The network address the agent listens on. This only takes effect if
## the agent isn't configured to connect to the controller (see
## :zeek:see:`ClusterAgent::controller`). By default this uses the value of the
## ZEEK_AGENT_ADDR environment variable, but you may also redef to
## a specific value. When empty, the implementation falls back to
## :zeek:see:`ClusterAgent::default_address`.
const listen_address = getenv("ZEEK_AGENT_ADDR") &redef;
## The fallback listen address if :zeek:see:`ClusterAgent::listen_address`
## remains empty. Unless redefined, this uses Broker's own default listen
## address.
const default_address = Broker::default_listen_address &redef;
## The network port the agent listens on. Counterpart to
## :zeek:see:`ClusterAgent::listen_address`, defaulting to the ZEEK_AGENT_PORT
## environment variable.
const listen_port = getenv("ZEEK_AGENT_PORT") &redef;
## The fallback listen port if :zeek:see:`ClusterAgent::listen_port` remains empty.
const default_port = 2151/tcp &redef;
# The agent communicates under to following topic prefix,
# suffixed with "/<name>" (see above):
## The agent's Broker topic prefix. For its own communication, the agent
## suffixes this with "/<name>", based on :zeek:see:`ClusterAgent::name`.
const topic_prefix = "zeek/cluster-control/agent" &redef;
# The coordinates of the controller. When defined, it means
# agents peer with (connect to) the controller; otherwise the
# controller knows all agents and peers with them.
## The network coordinates of the controller. When defined, the agent
## peers with (and connects to) the controller; otherwise the controller
## will peer (and connect to) the agent, listening as defined by
## :zeek:see:`ClusterAgent::listen_address` and :zeek:see:`ClusterAgent::listen_port`.
const controller: Broker::NetworkInfo = [
$address="0.0.0.0", $bound_port=0/unknown] &redef;
# Agent and controller currently log only, not via the data cluster's
# logger. (This might get added later.) For now, this means that
# if both write to the same log file, it gets garbled. The following
# lets you specify the working directory specifically for the agent.
## An optional custom output directory for the agent's stdout and stderr
## logs. Agent and controller currently only log locally, not via the
## data cluster's logger node. (This might change in the future.) This
## means that if both write to the same log file, the output gets
## garbled.
const directory = "" &redef;
# Working directory for data cluster nodes. When relative, note
# that this will apply from the working directory of the agent,
# since it creates data cluster nodes.
## The working directory for data cluster nodes created by this
## agent. If you make this a relative path, note that the path is
## relative to the agent's working directory, since it creates data
## cluster nodes.
const cluster_directory = "" &redef;
# The following functions return the effective network endpoint
# information for this agent, in two related forms.
## Returns a :zeek:see:`ClusterController::Types::Instance` describing this
## instance (its agent name plus listening address/port, as applicable).
global instance: function(): ClusterController::Types::Instance;
## Returns a :zeek:see:`Broker::EndpointInfo` record for this instance.
## Similar to :zeek:see:`ClusterAgent::instance`, but with slightly different
## data format.
global endpoint_info: function(): Broker::EndpointInfo;
}
@ -53,8 +85,8 @@ function instance(): ClusterController::Types::Instance
{
local epi = endpoint_info();
return ClusterController::Types::Instance($name=epi$id,
$host=to_addr(epi$network$address),
$listen_port=epi$network$bound_port);
$host=to_addr(epi$network$address),
$listen_port=epi$network$bound_port);
}
function endpoint_info(): Broker::EndpointInfo

96
scripts/policy/frameworks/cluster/agent/main.zeek
View file

@ -1,3 +1,8 @@
##! This is the main "runtime" of a cluster agent. Zeek does not load this
##! directly; rather, the agent's bootstrapping module (in ./boot.zeek)
##! specifies it as the script to run in the node newly created via Zeek's
##! supervisor.
@load base/frameworks/broker
@load policy/frameworks/cluster/controller/config
@ -6,21 +11,24 @@
@load ./api
module ClusterAgent::Runtime;
redef ClusterController::role = ClusterController::Types::AGENT;
# The global configuration as passed to us by the controller
global global_config: ClusterController::Types::Configuration;
global g_config: ClusterController::Types::Configuration;
# A map to make other instance info accessible
global instances: table[string] of ClusterController::Types::Instance;
global g_instances: table[string] of ClusterController::Types::Instance;
# A map for the nodes we run on this instance, via this agent.
global nodes: table[string] of ClusterController::Types::Node;
global g_nodes: table[string] of ClusterController::Types::Node;
# The node map employed by the supervisor to describe the cluster
# topology to newly forked nodes. We refresh it when we receive
# new configurations.
global data_cluster: table[string] of Supervisor::ClusterEndpoint;
global g_data_cluster: table[string] of Supervisor::ClusterEndpoint;
event SupervisorControl::create_response(reqid: string, result: string)
{
@ -86,43 +94,43 @@ event ClusterAgent::API::set_configuration_request(reqid: string, config: Cluste
# Adopt the global configuration provided.
# XXX this can later handle validation and persistence
# XXX should do this transactionally, only set when all else worked
global_config = config;
g_config = config;
# Refresh the instances table:
instances = table();
g_instances = table();
for ( inst in config$instances )
instances[inst$name] = inst;
g_instances[inst$name] = inst;
# Terminate existing nodes
for ( nodename in nodes )
for ( nodename in g_nodes )
supervisor_destroy(nodename);
nodes = table();
g_nodes = table();
# Refresh the data cluster and nodes tables
data_cluster = table();
g_data_cluster = table();
for ( node in config$nodes )
{
if ( node$instance == ClusterAgent::name )
nodes[node$name] = node;
g_nodes[node$name] = node;
local cep = Supervisor::ClusterEndpoint(
$role = node$role,
$host = instances[node$instance]$host,
$host = g_instances[node$instance]$host,
$p = node$p);
if ( node?$interface )
cep$interface = node$interface;
data_cluster[node$name] = cep;
g_data_cluster[node$name] = cep;
}
# Apply the new configuration via the supervisor
for ( nodename in nodes )
for ( nodename in g_nodes )
{
node = nodes[nodename];
node = g_nodes[nodename];
nc = Supervisor::NodeConfig($name=nodename);
if ( ClusterAgent::cluster_directory != "" )
@ -140,7 +148,7 @@ event ClusterAgent::API::set_configuration_request(reqid: string, config: Cluste
# XXX could use options to enable per-node overrides for
# directory, stdout, stderr, others?
nc$cluster = data_cluster;
nc$cluster = g_data_cluster;
supervisor_create(nc);
}
@ -149,22 +157,59 @@ event ClusterAgent::API::set_configuration_request(reqid: string, config: Cluste
# events asynchonously. The only indication of error will be
# notification events to the controller.
if ( reqid != "" )
{
local res = ClusterController::Types::Result(
$reqid = reqid,
$instance = ClusterAgent::name);
ClusterController::Log::info(fmt("tx ClusterAgent::API::set_configuration_response %s",
ClusterController::Types::result_to_string(res)));
event ClusterAgent::API::set_configuration_response(reqid, res);
}
}
event ClusterAgent::API::agent_welcome_request(reqid: string)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::agent_welcome_request %s", reqid));
local res = ClusterController::Types::Result(
$reqid = reqid,
$instance = ClusterAgent::name);
ClusterController::Log::info(fmt("tx ClusterAgent::API::set_configuration_response %s", reqid));
event ClusterAgent::API::set_configuration_response(reqid, res);
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_welcome_response %s",
ClusterController::Types::result_to_string(res)));
event ClusterAgent::API::agent_welcome_response(reqid, res);
}
event ClusterAgent::API::agent_standby_request(reqid: string)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::agent_standby_request %s", reqid));
# We shut down any existing cluster nodes via an empty configuration,
# and fall silent. We do not unpeer/disconnect (assuming we earlier
# peered/connected -- otherwise there's nothing we can do here via
# Broker anyway), mainly to keep open the possibility of running
# cluster nodes again later.
event ClusterAgent::API::set_configuration_request("", ClusterController::Types::Configuration());
local res = ClusterController::Types::Result(
$reqid = reqid,
$instance = ClusterAgent::name);
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_standby_response %s",
ClusterController::Types::result_to_string(res)));
event ClusterAgent::API::agent_standby_response(reqid, res);
}
event Broker::peer_added(peer: Broker::EndpointInfo, msg: string)
{
# This does not (cannot?) immediately verify that the new peer
# is in fact a controller, so we might send this redundantly.
# Controllers handle the hello event accordingly.
# is in fact a controller, so we might send this in vain.
# Controllers register the agent upon receipt of the event.
local epi = ClusterAgent::endpoint_info();
# XXX deal with unexpected peers, unless we're okay with it
event ClusterAgent::API::notify_agent_hello(epi$id,
to_addr(epi$network$address), ClusterAgent::API::version);
}
@ -185,13 +230,16 @@ event zeek_init()
Broker::peer(supervisor_addr, Broker::default_port, Broker::default_listen_retry);
# Agents need receive communication targeted at it, and any responses
# from the supervisor.
# from the supervisor.
Broker::subscribe(agent_topic);
Broker::subscribe(SupervisorControl::topic_prefix);
# Auto-publish a bunch of events. Glob patterns or module-level
# auto-publish would be helpful here.
Broker::auto_publish(agent_topic, ClusterAgent::API::set_configuration_response);
Broker::auto_publish(agent_topic, ClusterAgent::API::agent_welcome_response);
Broker::auto_publish(agent_topic, ClusterAgent::API::agent_standby_response);
Broker::auto_publish(agent_topic, ClusterAgent::API::notify_agent_hello);
Broker::auto_publish(agent_topic, ClusterAgent::API::notify_change);
Broker::auto_publish(agent_topic, ClusterAgent::API::notify_error);
@ -210,8 +258,8 @@ event zeek_init()
{
# We connect to the controller.
Broker::peer(ClusterAgent::controller$address,
ClusterAgent::controller$bound_port,
ClusterController::connect_retry);
ClusterAgent::controller$bound_port,
ClusterController::connect_retry);
}
else
{

5
scripts/policy/frameworks/cluster/controller/__load__.zeek
View file

@ -1,5 +1,4 @@
# The entry point for the cluster controller. It only runs bootstrap logic for
# launching via the Supervisor. If we're not running the Supervisor, this does
# nothing.
##! The entry point for the cluster controller. It runs bootstrap logic for
##! launching the controller process via Zeek's Supervisor.
@load ./boot

88
scripts/policy/frameworks/cluster/controller/api.zeek
View file

@ -1,16 +1,96 @@
##! The event API of cluster controllers. Most endpoints consist of event pairs,
##! where the controller answers a zeek-client request event with a
##! corresponding response event. Such event pairs share the same name prefix
##! and end in "_request" and "_response", respectively.
@load ./types
module ClusterController::API;
export {
## A simple versioning scheme, used to track basic compatibility of
## controller, agents, and zeek-client.
const version = 1;
global get_instances_request: event(reqid: string);
global get_instances_response: event(reqid: string,
instances: vector of ClusterController::Types::Instance);
## zeek-client sends this event to request a list of the currently
## peered agents/instances.
##
## reqid: a request identifier string, echoed in the response event.
##
global get_instances_request: event(reqid: string);
## Response to a get_instances_request event. The controller sends
## this back to the client.
##
## reqid: the request identifier used in the request event.
##
## result: the result record. Its data member is a
## :zeek:see:`ClusterController::Types::Instance` record.
##
global get_instances_response: event(reqid: string,
result: ClusterController::Types::Result);
## zeek-client sends this event to establish a new cluster configuration,
## including the full cluster topology. The controller processes the update
## and relays it to the agents. Once each has responded (or a timeout occurs)
## the controller sends a corresponding response event back to the client.
##
## reqid: a request identifier string, echoed in the response event.
##
## config: a :zeek:see:`ClusterController::Types::Configuration` record
## specifying the cluster configuration.
##
global set_configuration_request: event(reqid: string,
config: ClusterController::Types::Configuration);
## Response to a set_configuration_request event. The controller sends
## this back to the client.
##
## reqid: the request identifier used in the request event.
##
## result: a vector of :zeek:see:`ClusterController::Types::Result` records.
## Each member captures one agent's response.
##
global set_configuration_response: event(reqid: string,
result: ClusterController::Types::ResultVec);
}
# Testing events. These don't provide operational value but expose
# internal functionality, triggered by test cases.
## This event causes no further action (other than getting logged) if
## with_state is F. When T, the controller establishes request state, and
## the controller only ever sends the response event when this state times
## out.
##
## reqid: a request identifier string, echoed in the response event when
## with_state is T.
##
## with_state: flag indicating whether the controller should keep (and
## time out) request state for this request.
##
global test_timeout_request: event(reqid: string, with_state: bool);
## Response to a test_timeout_request event. The controller sends this
## back to the client if the original request had the with_state flag.
##
## reqid: the request identifier used in the request event.
##
global test_timeout_response: event(reqid: string,
result: ClusterController::Types::Result);
# Notification events, agent -> controller
## The controller triggers this event when the operational cluster
## instances align with the ones desired by the cluster
## configuration. It's essentially a cluster management readiness
## event. This event is currently only used by the controller and not
## published to other topics.
##
## instances: the set of instance names now ready.
##
global notify_agents_ready: event(instances: set[string]);
}

7
scripts/policy/frameworks/cluster/controller/boot.zeek
View file

@ -1,3 +1,10 @@
##! The cluster controller's boot logic runs in Zeek's supervisor and instructs
##! it to launch the controller process. The controller's main logic resides in
##! main.zeek, similarly to other frameworks. The new process will execute that
##! script.
##!
##! If the current process is not the Zeek supervisor, this does nothing.
@load ./config
event zeek_init()

75
scripts/policy/frameworks/cluster/controller/config.zeek
View file

@ -1,53 +1,78 @@
##! Configuration settings for the cluster controller.
@load policy/frameworks/cluster/agent/config
module ClusterController;
export {
# The name of this controller in the cluster.
# Without the environment variable and no redef, this
# falls back to "controller-<hostname>".
## The name of this controller. Defaults to the value of the
## ZEEK_CONTROLLER_NAME environment variable. When that is unset and the
## user doesn't redef the value, the implementation defaults to
## "controller-<hostname>".
const name = getenv("ZEEK_CONTROLLER_NAME") &redef;
# Controller stdout/stderr log files to produce in Zeek's
# working directory. If empty, no such logs will result.
## The controller's stdout log name. If the string is non-empty, Zeek will
## produce a free-form log (i.e., not one governed by Zeek's logging
## framework) in Zeek's working directory. If left empty, no such log
## results.
##
## Note that the controller also establishes a "proper" Zeek log via the
## :zeek:see:`ClusterController::Log` module.
const stdout_file = "controller.stdout" &redef;
## The controller's stderr log name. Like :zeek:see:`ClusterController::stdout_file`,
## but for the stderr stream.
const stderr_file = "controller.stderr" &redef;
# The address and port the controller listens on. When
# undefined, falls back to the default_address, which you can
# likewise customize.
## The network address the controller listens on. By default this uses
## the value of the ZEEK_CONTROLLER_ADDR environment variable, but you
## may also redef to a specific value. When empty, the implementation
## falls back to :zeek:see:`ClusterController::default_address`.
const listen_address = getenv("ZEEK_CONTROLLER_ADDR") &redef;
## The fallback listen address if :zeek:see:`ClusterController::listen_address`
## remains empty. Unless redefined, this uses Broker's own default
## listen address.
const default_address = Broker::default_listen_address &redef;
## The network port the controller listens on. Counterpart to
## :zeek:see:`ClusterController::listen_address`, defaulting to the
## ZEEK_CONTROLLER_PORT environment variable.
const listen_port = getenv("ZEEK_CONTROLLER_PORT") &redef;
## The fallback listen port if :zeek:see:`ClusterController::listen_port`
## remains empty.
const default_port = 2150/tcp &redef;
# A more aggressive default retry interval (vs default 30s)
## The controller's connect retry interval. Defaults to a more
## aggressive value compared to Broker's 30s.
const connect_retry = 1sec &redef;
# The controller listens for messages on this topic:
## The controller's Broker topic. Clients send requests to this topic.
const topic = "zeek/cluster-control/controller" &redef;
# The set of agents to interact with. When this is non-empty
# at startup, the controller contacts the agents; when it is
# empty, it waits for agents to connect. They key is a name of
# each instance. This should match the $name member of the
# instance records.
const instances: table[string] of ClusterController::Types::Instance = { } &redef;
# The role of this node in cluster management. Agent and
# controller both redef this. Used during logging.
## The role of this process in cluster management. Agent and controller
## both redefine this. Used during logging.
const role = ClusterController::Types::NONE &redef;
# Agent and controller currently log only, not via the data cluster's
# logger. (This might get added later.) For now, this means that
# if both write to the same log file, it gets garbled. The following
# lets you specify the working directory specifically for the agent.
## The timeout for request state. Such state (see the :zeek:see:`ClusterController::Request`
## module) ties together request and response event pairs. The timeout causes
## its cleanup in the absence of a timely response. It applies both to
## state kept for client requests, as well as state in the agents for
## requests to the supervisor.
const request_timeout = 10sec &redef;
## An optional custom output directory for the controller's stdout and
## stderr logs. Agent and controller currently only log locally, not via
## the data cluster's logger node. (This might change in the future.)
## This means that if both write to the same log file, the output gets
## garbled.
const directory = "" &redef;
# The following functions return the effective network endpoint
# information for this controller, in two related forms.
## Returns a :zeek:see:`Broker::NetworkInfo` record describing the controller.
global network_info: function(): Broker::NetworkInfo;
## Returns a :zeek:see:`Broker::EndpointInfo` record describing the controller.
global endpoint_info: function(): Broker::EndpointInfo;
}

30
scripts/policy/frameworks/cluster/controller/log.zeek
View file

@ -1,3 +1,8 @@
##! This module implements straightforward logging abilities for cluster
##! controller and agent. It uses Zeek's logging framework, and works only for
##! nodes managed by the supervisor. In this setting Zeek's logging framework
##! operates locally, i.e., this logging does not involve any logger nodes.
@load ./config
module ClusterController::Log;
@ -9,6 +14,7 @@ export {
## A default logging policy hook for the stream.
global log_policy: Log::PolicyHook;
## The controller/agent log supports four different log levels.
type Level: enum {
DEBUG,
INFO,
@ -16,7 +22,7 @@ export {
ERROR,
};
## The record type which contains the column fields of the cluster log.
## The record type containing the column fields of the agent/controller log.
type Info: record {
## The time at which a cluster message was generated.
ts: time;
@ -30,10 +36,32 @@ export {
message: string;
} &log;
## The log level in use for this node.
global log_level = DEBUG &redef;
## A debug-level log message writer.
##
## message: the message to log.
##
global debug: function(message: string);
## An info-level log message writer.
##
## message: the message to log.
##
global info: function(message: string);
## A warning-level log message writer.
##
## message: the message to log.
##
global warning: function(message: string);
## An error-level log message writer. (This only logs a message, it does not
## terminate Zeek or have other runtime effects.)
##
## message: the message to log.
##
global error: function(message: string);
}

505
scripts/policy/frameworks/cluster/controller/main.zeek
View file

@ -1,3 +1,8 @@
##! This is the main "runtime" of the cluster controller. Zeek does not load
##! this directly; rather, the controller's bootstrapping module (in ./boot.zeek)
##! specifies it as the script to run in the node newly created via Zeek's
##! supervisor.
@load base/frameworks/broker
@load policy/frameworks/cluster/agent/config
@ -6,55 +11,255 @@
@load ./api
@load ./log
@load ./request
@load ./util
module ClusterController::Runtime;
redef ClusterController::role = ClusterController::Types::CONTROLLER;
global check_instances_ready: function();
global add_instance: function(inst: ClusterController::Types::Instance);
global drop_instance: function(inst: ClusterController::Types::Instance);
global null_config: function(): ClusterController::Types::Configuration;
global is_null_config: function(config: ClusterController::Types::Configuration): bool;
# Checks whether the given instance is one that we know with different
# communication settings: a a different peering direction, a different listening
# port, etc. Used as a predicate to indicate when we need to drop the existing
# one from our internal state.
global is_instance_connectivity_change: function
(inst: ClusterController::Types::Instance): bool;
# The set of agents the controller interacts with to manage to currently
# configured cluster. This may be a subset of all the agents known to the
# controller, as tracked by the g_instances_known set. They key is the instance
# name and should match the $name member of the corresponding instance record.
global g_instances: table[string] of ClusterController::Types::Instance = table();
# The set of instances that have checked in with the controller. This is a
# superset of g_instances, since it covers any agent that has sent us a
# notify_agent_hello event.
global g_instances_known: set[string] = set();
# A corresponding set of instances/agents that we track in order to understand
# when all of the above instances have sent agent_welcome_response events. (An
# alternative would be to use a record that adds a single state bit for each
# instance, and store that above.)
global g_instances_ready: set[string] = set();
# The request ID of the most recent configuration update that's come in from
# a client. We track it here until we know we are ready to communicate with all
# agents required by the update.
global g_config_reqid_pending: string = "";
# The most recent configuration we have successfully deployed. This is also
# the one we send whenever the client requests it.
global g_config_current: ClusterController::Types::Configuration;
function send_config_to_agents(req: ClusterController::Request::Request,
config: ClusterController::Types::Configuration)
{
for ( name in g_instances )
{
if ( name !in g_instances_ready )
next;
local agent_topic = ClusterAgent::topic_prefix + "/" + name;
local areq = ClusterController::Request::create();
areq$parent_id = req$id;
# We track the requests sent off to each agent. As the
# responses come in, we can check them off as completed,
# and once all are, we respond back to the client.
req$set_configuration_state$requests += areq;
# We could also broadcast just once on the agent prefix, but
# explicit request/response pairs for each agent seems cleaner.
ClusterController::Log::info(fmt("tx ClusterAgent::API::set_configuration_request %s to %s", areq$id, name));
Broker::publish(agent_topic, ClusterAgent::API::set_configuration_request, areq$id, config);
}
}
# This is the &on_change handler for the g_instances_ready set, meaning
# it runs whenever a required agent has confirmed it's ready.
function check_instances_ready()
{
local cur_instances: set[string];
for ( inst in g_instances )
add cur_instances[inst];
if ( cur_instances == g_instances_ready )
event ClusterController::API::notify_agents_ready(cur_instances);
}
function add_instance(inst: ClusterController::Types::Instance)
{
g_instances[inst$name] = inst;
if ( inst?$listen_port )
Broker::peer(cat(inst$host), inst$listen_port,
ClusterController::connect_retry);
if ( inst$name in g_instances_known )
{
# The agent has already peered with us. Send welcome to indicate
# it's part of cluster management. Once it responds, we update
# the set of ready instances and proceed as feasible with config
# deployments.
local req = ClusterController::Request::create();
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_welcome_request to %s", inst$name));
Broker::publish(ClusterAgent::topic_prefix + "/" + inst$name,
ClusterAgent::API::agent_welcome_request, req$id);
}
}
function drop_instance(inst: ClusterController::Types::Instance)
{
if ( inst$name !in g_instances )
return;
# Send the agent a standby so it shuts down its cluster nodes & state
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_standby_request to %s", inst$name));
Broker::publish(ClusterAgent::topic_prefix + "/" + inst$name,
ClusterAgent::API::agent_standby_request, "");
delete g_instances[inst$name];
if ( inst$name in g_instances_ready )
delete g_instances_ready[inst$name];
# The agent remains in g_instances_known, to track that we're able
# to communicate with it in case it's required again.
ClusterController::Log::info(fmt("dropped instance %s", inst$name));
}
function null_config(): ClusterController::Types::Configuration
{
return ClusterController::Types::Configuration($id="");
}
function is_null_config(config: ClusterController::Types::Configuration): bool
{
return config$id == "";
}
function is_instance_connectivity_change(inst: ClusterController::Types::Instance): bool
{
# If we're not tracking this instance as part of a cluster config, it's
# not a change. (More precisely: we cannot say whether it's changed.)
if ( inst$name !in g_instances )
return F;
# The agent has peered with us and now uses a different host.
# XXX 0.0.0.0 is a workaround until we've resolved how agents that peer
# with us obtain their identity. Broker ID?
if ( inst$host != 0.0.0.0 && inst$host != g_instances[inst$name]$host )
return T;
# The agent has a listening port and the one we know does not, or vice
# versa. I.e., this is a change in the intended peering direction.
if ( inst?$listen_port != g_instances[inst$name]?$listen_port )
return T;
# Both have listening ports, but they differ.
if ( inst?$listen_port && g_instances[inst$name]?$listen_port &&
inst$listen_port != g_instances[inst$name]$listen_port )
return T;
return F;
}
event ClusterController::API::notify_agents_ready(instances: set[string])
{
local insts = ClusterController::Util::set_to_vector(instances);
ClusterController::Log::info(fmt("rx ClusterController::API:notify_agents_ready %s", join_string_vec(insts, ",")));
local req = ClusterController::Request::lookup(g_config_reqid_pending);
# If there's no pending request, when it's no longer available, or it
# doesn't have config state, don't do anything else.
if ( ClusterController::Request::is_null(req) || ! req?$set_configuration_state )
return;
# All instances requested in the pending configuration update are now
# known to us. Send them the config. As they send their response events
# we update the client's request state and eventually send the response
# event to the it.
send_config_to_agents(req, req$set_configuration_state$config);
}
event ClusterAgent::API::notify_agent_hello(instance: string, host: addr, api_version: count)
{
# See if we already know about this agent; if not, register
# it.
#
# XXX protection against rogue agents?
ClusterController::Log::info(fmt("rx ClusterAgent::API::notify_agent_hello %s %s", instance, host));
if ( instance in ClusterController::instances )
# When an agent checks in with a mismatching API version, we log the
# fact and drop its state, if any.
if ( api_version != ClusterController::API::version )
{
# Do nothing, unless this known agent checks in with a mismatching
# API version, in which case we kick it out.
if ( api_version != ClusterController::API::version )
{
local inst = ClusterController::instances[instance];
if ( inst?$listen_port )
{
# We peered with this instance, unpeer.
Broker::unpeer(cat(inst$host), inst$listen_port );
# XXX what to do if they connected to us?
}
delete ClusterController::instances[instance];
}
ClusterController::Log::warning(
fmt("instance %s/%s has checked in with incompatible API version %s",
instance, host, api_version));
# Update the instance name in the pointed-to record, in case it
# was previously named otherwise. Not being too picky here allows
# the user some leeway in spelling out the original config.
ClusterController::instances[instance]$name = instance;
if ( instance in g_instances )
drop_instance(g_instances[instance]);
if ( instance in g_instances_known )
delete g_instances_known[instance];
return;
}
if ( api_version != ClusterController::API::version )
{
ClusterController::Log::warning(
fmt("agent %s/%s speaks incompatible agent protocol (%s, need %s), unpeering",
instance, host, api_version, ClusterController::API::version));
}
add g_instances_known[instance];
ClusterController::instances[instance] = ClusterController::Types::Instance($name=instance, $host=host);
ClusterController::Log::info(fmt("instance %s/%s has checked in", instance, host));
if ( instance in g_instances && instance !in g_instances_ready )
{
# We need this instance for our cluster and have full context for
# it from the configuration. Tell agent.
local req = ClusterController::Request::create();
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_welcome_request to %s", instance));
Broker::publish(ClusterAgent::topic_prefix + "/" + instance,
ClusterAgent::API::agent_welcome_request, req$id);
}
}
event ClusterAgent::API::agent_welcome_response(reqid: string, result: ClusterController::Types::Result)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::agent_welcome_response %s", reqid));
local req = ClusterController::Request::lookup(reqid);
if ( ClusterController::Request::is_null(req) )
return;
ClusterController::Request::finish(req$id);
# An agent we've been waiting to hear back from is ready for cluster
# work. Double-check we still want it, otherwise drop it.
if ( ! result$success || result$instance !in g_instances )
{
ClusterController::Log::info(fmt(
"tx ClusterAgent::API::agent_standby_request to %s", result$instance));
Broker::publish(ClusterAgent::topic_prefix + "/" + result$instance,
ClusterAgent::API::agent_standby_request, "");
return;
}
add g_instances_ready[result$instance];
ClusterController::Log::info(fmt("instance %s ready", result$instance));
check_instances_ready();
}
event ClusterAgent::API::notify_change(instance: string, n: ClusterController::Types::Node,
old: ClusterController::Types::State,
new: ClusterController::Types::State)
old: ClusterController::Types::State,
new: ClusterController::Types::State)
{
# XXX TODO
}
@ -96,10 +301,10 @@ event ClusterAgent::API::set_configuration_response(reqid: string, result: Clust
return;
# All set_configuration requests to instances are done, so respond
# back to client. We need to compose the result, aggregating
# the results we got from the requests to the agents. In the
# end we have one Result per instance requested in the
# original set_configuration_request.
# back to client. We need to compose the result, aggregating
# the results we got from the requests to the agents. In the
# end we have one Result per instance requested in the
# original set_configuration_request.
#
# XXX we can likely generalize result aggregation in the request module.
for ( i in req$set_configuration_state$requests )
@ -132,7 +337,13 @@ event ClusterAgent::API::set_configuration_response(reqid: string, result: Clust
ClusterController::Request::finish(r$id);
}
ClusterController::Log::info(fmt("tx ClusterController::API::set_configuration_response %s", req$id));
# We're now done with the original set_configuration request.
# Adopt the configuration as the current one.
g_config_current = req$set_configuration_state$config;
g_config_reqid_pending = "";
ClusterController::Log::info(fmt("tx ClusterController::API::set_configuration_response %s",
ClusterController::Request::to_string(req)));
event ClusterController::API::set_configuration_response(req$id, req$results);
ClusterController::Request::finish(req$id);
}
@ -141,25 +352,24 @@ event ClusterController::API::set_configuration_request(reqid: string, config: C
{
ClusterController::Log::info(fmt("rx ClusterController::API::set_configuration_request %s", reqid));
local res: ClusterController::Types::Result;
local req = ClusterController::Request::create(reqid);
req$set_configuration_state = ClusterController::Request::SetConfigurationState();
# Compare new configuration to the current one and send updates
# to the instances as needed.
if ( config?$instances )
req$set_configuration_state = ClusterController::Request::SetConfigurationState($config = config);
# At the moment there can only be one pending request.
if ( g_config_reqid_pending != "" )
{
# XXX properly handle instance update: connect to new instances provided
# when they are listening, accept connections from new instances that are
# not
for ( inst in config$instances )
{
if ( inst$name !in ClusterController::instances )
{
local res = ClusterController::Types::Result($reqid=reqid, $instance=inst$name);
res$error = fmt("instance %s is unknown, skipping", inst$name);
req$results += res;
}
}
res = ClusterController::Types::Result($reqid=reqid);
res$success = F;
res$error = fmt("request %s still pending", g_config_reqid_pending);
req$results += res;
ClusterController::Log::info(fmt("tx ClusterController::API::set_configuration_response %s",
ClusterController::Request::to_string(req)));
event ClusterController::API::set_configuration_response(req$id, req$results);
ClusterController::Request::finish(req$id);
return;
}
# XXX validate the configuration:
@ -169,82 +379,177 @@ event ClusterController::API::set_configuration_request(reqid: string, config: C
# - Do node types with optional fields have required values?
# ...
# Transmit the configuration on to the agents. They need to be aware of
# each other's location and nodes, so the data cluster nodes can connect
# (for example, so a worker on instance 1 can connect to a logger on
# instance 2).
for ( name in ClusterController::instances )
# The incoming request is now the pending one. It gets cleared when all
# agents have processed their config updates successfully, or their
# responses time out.
g_config_reqid_pending = req$id;
# Compare the instance configuration to our current one. If it matches,
# we can proceed to deploying the new data cluster topology. If it does
# not, we need to establish connectivity with agents we connect to, or
# wait until all instances that connect to us have done so. Either triggers
# a notify_agents_ready event, upon which we then deploy the data cluster.
# The current & new set of instance names.
local insts_current: set[string];
local insts_new: set[string];
# A set of current instances not contained in the new config.
# Those will need to get dropped.
local insts_to_drop: set[string];
# The opposite: new instances not yet in our current set. Those we will need
# to establish contact with (or they with us).
local insts_to_add: set[string];
# The overlap: instances in both the current and new set. For those we verify
# that we're actually dealign with the same entities, and might need to re-
# connect if not.
local insts_to_keep: set[string];
# Alternative representation of insts_to_add, directly providing the instances.
local insts_to_peer: table[string] of ClusterController::Types::Instance;
# Helpful locals.
local inst_name: string;
local inst: ClusterController::Types::Instance;
for ( inst_name in g_instances )
add insts_current[inst_name];
for ( inst in config$instances )
add insts_new[inst$name];
# Populate TODO lists for instances we need to drop, check, or add.
insts_to_drop = insts_current - insts_new;
insts_to_add = insts_new - insts_current;
insts_to_keep = insts_new & insts_current;
for ( inst in config$instances )
{
local agent_topic = ClusterAgent::topic_prefix + "/" + name;
local areq = ClusterController::Request::create();
areq$parent_id = reqid;
if ( inst$name in insts_to_add )
{
insts_to_peer[inst$name] = inst;
next;
}
# We track the requests sent off to each agent. As the
# responses come in, we can check them off as completed,
# and once all are, we respond back to the client.
req$set_configuration_state$requests += areq;
# Focus on the keepers: check for change in identity/location.
if ( inst$name !in insts_to_keep )
next;
# XXX could also broadcast just once on the agent prefix, but
# explicit request/response pairs for each agent seems cleaner.
ClusterController::Log::info(fmt("tx ClusterAgent::API::set_configuration_request %s to %s",
areq$id, name));
Broker::publish(agent_topic, ClusterAgent::API::set_configuration_request, areq$id, config);
if ( is_instance_connectivity_change(inst) )
{
# The endpoint looks different. We drop the current one
# and need to re-establish connectivity with the new
# one.
add insts_to_drop[inst$name];
add insts_to_add[inst$name];
}
}
# Response event gets sent via the agents' reponse event.
# Process our TODO lists. Handle drops first, then additions, in
# case we need to re-establish connectivity with an agent.
for ( inst_name in insts_to_drop )
drop_instance(g_instances[inst_name]);
for ( inst_name in insts_to_peer )
add_instance(insts_to_peer[inst_name]);
# Updates to out instance tables are complete, now check if we're already
# able to send the config to the agents:
check_instances_ready();
}
event ClusterController::API::get_instances_request(reqid: string)
{
ClusterController::Log::info(fmt("rx ClusterController::API::set_instances_request %s", reqid));
local res = ClusterController::Types::Result($reqid = reqid);
local insts: vector of ClusterController::Types::Instance;
for ( i in ClusterController::instances )
insts += ClusterController::instances[i];
for ( i in g_instances )
insts += g_instances[i];
res$data = insts;
ClusterController::Log::info(fmt("tx ClusterController::API::get_instances_response %s", reqid));
event ClusterController::API::get_instances_response(reqid, insts);
event ClusterController::API::get_instances_response(reqid, res);
}
event ClusterController::Request::request_expired(req: ClusterController::Request::Request)
{
# Various handlers for timed-out request state. We use the state members
# to identify how to respond. No need to clean up the request itself,
# since we're getting here via the request module's expiration
# mechanism that handles the cleanup.
local res: ClusterController::Types::Result;
if ( req?$set_configuration_state )
{
# This timeout means we no longer have a pending request.
g_config_reqid_pending = "";
res = ClusterController::Types::Result($reqid=req$id);
res$success = F;
res$error = "request timed out";
req$results += res;
ClusterController::Log::info(fmt("tx ClusterController::API::set_configuration_response %s",
ClusterController::Request::to_string(req)));
event ClusterController::API::set_configuration_response(req$id, req$results);
}
if ( req?$test_state )
{
res = ClusterController::Types::Result($reqid=req$id);
res$success = F;
res$error = "request timed out";
ClusterController::Log::info(fmt("tx ClusterController::API::test_timeout_response %s", req$id));
event ClusterController::API::test_timeout_response(req$id, res);
}
}
event ClusterController::API::test_timeout_request(reqid: string, with_state: bool)
{
ClusterController::Log::info(fmt("rx ClusterController::API::test_timeout_request %s %s", reqid, with_state));
if ( with_state )
{
# This state times out and triggers a timeout response in the
# above request_expired event handler.
local req = ClusterController::Request::create(reqid);
req$test_state = ClusterController::Request::TestState();
}
}
event zeek_init()
{
# Controller always listens -- it needs to be able to respond
# to the Zeek client. This port is also used by the agents
# if they connect to the client.
# Initialize null config at startup. We will replace it once we have
# persistence, and again whenever we complete a client's
# set_configuration request.
g_config_current = null_config();
# The controller always listens -- it needs to be able to respond to the
# Zeek client. This port is also used by the agents if they connect to
# the client. The client doesn't automatically establish or accept
# connectivity to agents: agents are defined and communicated with as
# defined via configurations defined by the client.
local cni = ClusterController::network_info();
Broker::listen(cat(cni$address), cni$bound_port);
Broker::subscribe(ClusterAgent::topic_prefix);
Broker::subscribe(ClusterController::topic);
# Events sent to the client:
Broker::auto_publish(ClusterController::topic,
ClusterController::API::get_instances_response);
Broker::auto_publish(ClusterController::topic,
ClusterController::API::set_configuration_response);
if ( |ClusterController::instances| > 0 )
{
# We peer with the agents -- otherwise, the agents peer
# with (i.e., connect to) us.
for ( i in ClusterController::instances )
{
local inst = ClusterController::instances[i];
if ( ! inst?$listen_port )
{
# XXX config error -- this must be there
next;
}
Broker::peer(cat(inst$host), inst$listen_port,
ClusterController::connect_retry);
}
}
# If ClusterController::instances is empty, agents peer with
# us and we do nothing. We'll build up state as the
# notify_agent_hello() events come int.
Broker::auto_publish(ClusterController::topic,
ClusterController::API::test_timeout_response);
ClusterController::Log::info("controller is live");
}

118
scripts/policy/frameworks/cluster/controller/request.zeek
View file

@ -1,23 +1,33 @@
##! This module implements a request state abstraction that both cluster
##! controller and agent use to tie responses to received request events and be
##! able to time-out such requests.
@load ./types
@load ./config
module ClusterController::Request;
export {
## Request records track each request's state.
type Request: record {
## Each request has a hopfully unique ID provided by the requester.
id: string;
## For requests that result based upon another request (such as when
## the controller sends requests to agents based on a request it
## received by the client), this specifies that original, "parent"
## request.
parent_id: string &optional;
};
# API-specific state. XXX we may be able to generalize after this
# has settled a bit more.
# API-specific state. XXX we may be able to generalize after this has
# settled a bit more. It would also be nice to move request-specific
# state out of this module -- we could for example redef Request in
# main.zeek as needed.
# State specific to the set_configuration request/response events
type SetConfigurationState: record {
requests: vector of Request &default=vector();
};
# State specific to the set_nodes request/response events
type SetNodesState: record {
config: ClusterController::Types::Configuration;
requests: vector of Request &default=vector();
};
@ -26,51 +36,105 @@ export {
node: string;
};
# State for testing events
type TestState: record {
};
# The redef is a workaround so we can use the Request type
# while it is still being defined
# while it is still being defined.
redef record Request += {
results: ClusterController::Types::ResultVec &default=vector();
finished: bool &default=F;
set_configuration_state: SetConfigurationState &optional;
set_nodes_state: SetNodesState &optional;
supervisor_state: SupervisorState &optional;
test_state: TestState &optional;
};
## A token request that serves as a null/nonexistant request.
global null_req = Request($id="", $finished=T);
## This function establishes request state.
##
## reqid: the identifier to use for the request.
##
global create: function(reqid: string &default=unique_id("")): Request;
## This function looks up the request for a given request ID and returns
## it. When no such request exists, returns ClusterController::Request::null_req.
##
## reqid: the ID of the request state to retrieve.
##
global lookup: function(reqid: string): Request;
## This function marks a request as complete and causes Zeek to release
## its internal state. When the request does not exist, this does
## nothing.
##
## reqid: the ID of the request state to releaase.
##
global finish: function(reqid: string): bool;
## This event fires when a request times out (as per the
## ClusterController::request_timeout) before it has been finished via
## ClusterController::Request::finish().
##
## req: the request state that is expiring.
##
global request_expired: event(req: Request);
## This function is a helper predicate to indicate whether a given
## request is null.
##
## request: a Request record to check.
##
## Returns: T if the given request matches the null_req instance, F otherwise.
##
global is_null: function(request: Request): bool;
## For troubleshooting, this function renders a request record to a string.
##
## request: the request to render.
##
global to_string: function(request: Request): string;
}
# XXX this needs a mechanism for expiring stale requests
global requests: table[string] of Request;
function requests_expire_func(reqs: table[string] of Request, reqid: string): interval
{
event ClusterController::Request::request_expired(reqs[reqid]);
return 0secs;
}
# This is the global request-tracking table. The table maps from request ID
# strings to corresponding Request records. Entries time out after the
# ClusterController::request_timeout interval. Upon expiration, a
# request_expired event triggers that conveys the request state.
global g_requests: table[string] of Request
&create_expire=ClusterController::request_timeout
&expire_func=requests_expire_func;
function create(reqid: string): Request
{
local ret = Request($id=reqid);
requests[reqid] = ret;
g_requests[reqid] = ret;
return ret;
}
function lookup(reqid: string): Request
{
if ( reqid in requests )
return requests[reqid];
if ( reqid in g_requests )
return g_requests[reqid];
return null_req;
}
function finish(reqid: string): bool
{
if ( reqid !in requests )
if ( reqid !in g_requests )
return F;
local req = requests[reqid];
delete requests[reqid];
local req = g_requests[reqid];
delete g_requests[reqid];
req$finished = T;
@ -84,3 +148,23 @@ function is_null(request: Request): bool
return F;
}
function to_string(request: Request): string
{
local results: string_vec;
local res: ClusterController::Types::Result;
local parent_id = "";
if ( request?$parent_id )
parent_id = fmt(" (via %s)", request$parent_id);
for ( idx in request$results )
{
res = request$results[idx];
results[|results|] = ClusterController::Types::result_to_string(res);
}
return fmt("[request %s%s %s, results: %s]", request$id, parent_id,
request$finished ? "finished" : "pending",
join_string_vec(results, ","));
}

97
scripts/policy/frameworks/cluster/controller/types.zeek
View file

@ -1,4 +1,6 @@
# Types for the Cluster Controller framework. These are used by both agent and controller.
##! This module holds the basic types needed for the Cluster Controller
##! framework. These are used by both agent and controller, and several
##! have corresponding equals in the zeek-client implementation.
module ClusterController::Types;
@ -14,67 +16,96 @@ export {
## A Zeek-side option with value.
type Option: record {
name: string; # Name of option
value: string; # Value of option
name: string; ##< Name of option
value: string; ##< Value of option
};
## Configuration describing a Zeek instance running a Cluster
## Agent. Normally, there'll be one instance per cluster
## system: a single physical system.
type Instance: record {
# Unique, human-readable instance name
## Unique, human-readable instance name
name: string;
# IP address of system
## IP address of system
host: addr;
# Agent listening port. Not needed if agents connect to controller.
## Agent listening port. Not needed if agents connect to controller.
listen_port: port &optional;
};
type InstanceVec: vector of Instance;
## State that a Cluster Node can be in. State changes trigger an
## API notification (see notify_change()).
type State: enum {
Running, # Running and operating normally
Stopped, # Explicitly stopped
Failed, # Failed to start; and permanently halted
Crashed, # Crashed, will be restarted,
Unknown, # State not known currently (e.g., because of lost connectivity)
Running, ##< Running and operating normally
Stopped, ##< Explicitly stopped
Failed, ##< Failed to start; and permanently halted
Crashed, ##< Crashed, will be restarted,
Unknown, ##< State not known currently (e.g., because of lost connectivity)
};
## Configuration describing a Cluster Node process.
type Node: record {
name: string; # Cluster-unique, human-readable node name
instance: string; # Name of instance where node is to run
p: port; # Port on which this node will listen
role: Supervisor::ClusterRole; # Role of the node.
state: State; # Desired, or current, run state.
scripts: vector of string &optional; # Additional Zeek scripts for node
options: set[Option] &optional; # Zeek options for node
interface: string &optional; # Interface to sniff
cpu_affinity: int &optional; # CPU/core number to pin to
env: table[string] of string &default=table(); # Custom environment vars
name: string; ##< Cluster-unique, human-readable node name
instance: string; ##< Name of instance where node is to run
p: port; ##< Port on which this node will listen
role: Supervisor::ClusterRole; ##< Role of the node.
state: State; ##< Desired, or current, run state.
scripts: vector of string &optional; ##< Additional Zeek scripts for node
options: set[Option] &optional; ##< Zeek options for node
interface: string &optional; ##< Interface to sniff
cpu_affinity: int &optional; ##< CPU/core number to pin to
env: table[string] of string &default=table(); ##< Custom environment vars
};
# Data structure capturing a cluster's complete configuration.
## Data structure capturing a cluster's complete configuration.
type Configuration: record {
id: string &default=unique_id(""); # Unique identifier for a particular configuration
id: string &default=unique_id(""); ##< Unique identifier for a particular configuration
## The instances in the cluster.
## XXX we may be able to make this optional
instances: set[Instance];
instances: set[Instance] &default=set();
## The set of nodes in the cluster, as distributed over the instances.
nodes: set[Node];
nodes: set[Node] &default=set();
};
# Return value for request-response API event pairs
## Return value for request-response API event pairs
type Result: record {
reqid: string; # Request ID of operation this result refers to
instance: string; # Name of associated instance (for context)
success: bool &default=T; # True if successful
data: any &optional; # Addl data returned for successful operation
error: string &default=""; # Descriptive error on failure
node: string &optional; # Name of associated node (for context)
reqid: string; ##< Request ID of operation this result refers to
instance: string &default=""; ##< Name of associated instance (for context)
success: bool &default=T; ##< True if successful
data: any &optional; ##< Addl data returned for successful operation
error: string &default=""; ##< Descriptive error on failure
node: string &optional; ##< Name of associated node (for context)
};
type ResultVec: vector of Result;
global result_to_string: function(res: Result): string;
}
function result_to_string(res: Result): string
{
local result = "";
if ( res$success )
result = "success";
else if ( res$error != "" )
result = fmt("error (%s)", res$error);
else
result = "error";
local details: string_vec;
if ( res$reqid != "" )
details[|details|] = fmt("reqid %s", res$reqid);
if ( res$instance != "" )
details[|details|] = fmt("instance %s", res$instance);
if ( res?$node && res$node != "" )
details[|details|] = fmt("node %s", res$node);
if ( |details| > 0 )
result = fmt("%s (%s)", result, join_string_vec(details, ", "));
return result;
}

25
scripts/policy/frameworks/cluster/controller/util.zeek Normal file
View file

@ -0,0 +1,25 @@
##! Utility functions for the cluster controller framework, available to agent
##! and controller.
module ClusterController::Util;
export {
## Renders a set of strings to an alphabetically sorted vector.
##
## ss: the string set to convert.
##
## Returns: the vector of all strings in ss.
global set_to_vector: function(ss: set[string]): vector of string;
}
function set_to_vector(ss: set[string]): vector of string
{
local res: vector of string;
for ( s in ss )
res[|res|] = s;
sort(res, strcmp);
return res;
}

4
scripts/policy/frameworks/control/controller.zeek
View file

@ -41,7 +41,7 @@ event Control::net_stats_response(s: string) &priority=-10
{
event terminate_event();
}
event Control::configuration_update_response() &priority=-10
{
event terminate_event();
@ -68,7 +68,7 @@ function configurable_ids(): id_table
# We don't want to update non-const globals because that's usually
# where state is stored and those values will frequently be declared
# with &redef so that attributes can be redefined.
#
#
# NOTE: functions are currently not fully supported for serialization and hence
# aren't sent.
if ( t$constant && t$redefinable && t$type_name != "func" )

16
scripts/policy/frameworks/dpd/detect-protocols.zeek
View file

@ -22,7 +22,7 @@ export {
type dir: enum { NONE, INCOMING, OUTGOING, BOTH };
option valids: table[Analyzer::Tag, addr, port] of dir = {
option valids: table[AllAnalyzers::Tag, addr, port] of dir = {
# A couple of ports commonly used for benign HTTP servers.
# For now we want to see everything.
@ -45,7 +45,7 @@ export {
# log files, this also saves memory because for these we don't
# need to remember which servers we already have reported, which
# for some can be a lot.
option suppress_servers: set [Analyzer::Tag] = {
option suppress_servers: set [AllAnalyzers::Tag] = {
# Analyzer::ANALYZER_HTTP
};
@ -61,7 +61,7 @@ export {
# Entry point for other analyzers to report that they recognized
# a certain (sub-)protocol.
global found_protocol: function(c: connection, analyzer: Analyzer::Tag,
global found_protocol: function(c: connection, analyzer: AllAnalyzers::Tag,
protocol: string);
# Table keeping reported (server, port, analyzer) tuples (and their
@ -74,7 +74,7 @@ export {
}
# Table that tracks currently active dynamic analyzers per connection.
global conns: table[conn_id] of set[Analyzer::Tag];
global conns: table[conn_id] of set[AllAnalyzers::Tag];
# Table of reports by other analyzers about the protocol used in a connection.
global protocols: table[conn_id] of set[string];
@ -84,7 +84,7 @@ type protocol : record {
sub: string; # "sub-protocols" reported by other sources
};
function get_protocol(c: connection, a: Analyzer::Tag) : protocol
function get_protocol(c: connection, a: AllAnalyzers::Tag) : protocol
{
local str = "";
if ( c$id in protocols )
@ -101,7 +101,7 @@ function fmt_protocol(p: protocol) : string
return p$sub != "" ? fmt("%s (via %s)", p$sub, p$a) : p$a;
}
function do_notice(c: connection, a: Analyzer::Tag, d: dir)
function do_notice(c: connection, a: AllAnalyzers::Tag, d: dir)
{
if ( d == BOTH )
return;
@ -198,7 +198,7 @@ hook finalize_protocol_detection(c: connection)
report_protocols(c);
}
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count)
event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count)
{
# Don't report anything running on a well-known port.
if ( c$id$resp_p in Analyzer::registered_ports(atype) )
@ -219,7 +219,7 @@ event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count)
}
}
function found_protocol(c: connection, atype: Analyzer::Tag, protocol: string)
function found_protocol(c: connection, atype: AllAnalyzers::Tag, protocol: string)
{
# Don't report anything running on a well-known port.
if ( c$id$resp_p in Analyzer::registered_ports(atype) )

6
scripts/policy/frameworks/dpd/packet-segment-logging.zeek
View file

@ -11,7 +11,7 @@ module DPD;
export {
redef record Info += {
## A chunk of the payload that most likely resulted in the
## protocol violation.
## analyzer violation.
packet_segment: string &optional &log;
};
@ -20,10 +20,10 @@ export {
}
event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count,
reason: string) &priority=4
{
if ( ! c?$dpd ) return;
c$dpd$packet_segment=fmt("%s", sub_bytes(get_current_packet()$data, 0, packet_segment_size));
}

2
scripts/policy/frameworks/files/detect-MHR.zeek
View file

@ -66,7 +66,7 @@ function do_mhr_lookup(hash: string, fi: Notice::FileInfo)
event file_hash(f: fa_file, kind: string, hash: string)
{
if ( kind == "sha1" && f?$info && f$info?$mime_type &&
if ( kind == "sha1" && f?$info && f$info?$mime_type &&
match_file_types in f$info$mime_type )
do_mhr_lookup(hash, Notice::create_file_info(f));
}

5
scripts/policy/frameworks/files/entropy-test-all-files.zeek
View file

@ -1,10 +1,9 @@
module Files;
export {
redef record Files::Info += {
## The information density of the contents of the file,
## expressed as a number of bits per character.
## The information density of the contents of the file,
## expressed as a number of bits per character.
entropy: double &log &optional;
};
}

2
scripts/policy/frameworks/intel/seen/file-hashes.zeek
View file

@ -7,6 +7,6 @@ event file_hash(f: fa_file, kind: string, hash: string)
$indicator_type=Intel::FILE_HASH,
$f=f,
$where=Files::IN_HASH);
Intel::seen(seen);
}

3
scripts/policy/frameworks/intel/whitelist.zeek
View file

@ -22,9 +22,8 @@ hook Intel::extend_match(info: Info, s: Seen, items: set[Item]) &priority=9
break;
}
}
if ( whitelisted )
# Prevent logging
break;
}

4
scripts/policy/frameworks/software/version-changes.zeek
View file

@ -8,14 +8,14 @@
module Software;
export {
redef enum Notice::Type += {
redef enum Notice::Type += {
## For certain software, a version changing may matter. In that
## case, this notice will be generated. Software that matters
## if the version changes can be configured with the
## :zeek:id:`Software::interesting_version_changes` variable.
Software_Version_Change,
};
## Some software is more interesting when the version changes and this
## is a set of all software that should raise a notice when a different
## version is seen on a host.

18
scripts/policy/integration/barnyard2/main.zeek
View file

@ -8,7 +8,7 @@ module Barnyard2;
export {
redef enum Log::ID += { LOG };
global log_policy: Log::PolicyHook;
type Info: record {
@ -19,9 +19,9 @@ export {
## Associated alert data.
alert: AlertData &log;
};
## This can convert a Barnyard :zeek:type:`Barnyard2::PacketID` value to
## a :zeek:type:`conn_id` value in the case that you might need to index
## a :zeek:type:`conn_id` value in the case that you might need to index
## into an existing data structure elsewhere within Zeek.
global pid2cid: function(p: PacketID): conn_id;
}
@ -40,22 +40,22 @@ function pid2cid(p: PacketID): conn_id
event barnyard_alert(id: PacketID, alert: AlertData, msg: string, data: string)
{
Log::write(Barnyard2::LOG, [$ts=network_time(), $pid=id, $alert=alert]);
#local proto_connection_string: string;
#if ( id$src_p == 0/tcp )
# proto_connection_string = fmt("{PROTO:255} %s -> %s", id$src_ip, id$dst_ip);
#else
# proto_connection_string = fmt("{%s} %s:%d -> %s:%d",
# proto_connection_string = fmt("{%s} %s:%d -> %s:%d",
# to_upper(fmt("%s", get_port_transport_proto(id$dst_p))),
# id$src_ip, id$src_p, id$dst_ip, id$dst_p);
#
#local snort_alike_msg = fmt("%.6f [**] [%d:%d:%d] %s [**] [Classification: %s] [Priority: %d] %s",
#local snort_alike_msg = fmt("%.6f [**] [%d:%d:%d] %s [**] [Classification: %s] [Priority: %d] %s",
# sad$ts,
# sad$generator_id,
# sad$signature_id,
# sad$signature_revision,
# msg,
# sad$classification,
# sad$priority_id,
# msg,
# sad$classification,
# sad$priority_id,
# proto_connection_string);
}

2
scripts/policy/integration/barnyard2/types.zeek
View file

@ -23,7 +23,7 @@ export {
dst_p: port;
} &log;
## This is the event that Barnyard2 instances will send if they're
## This is the event that Barnyard2 instances will send if they're
## configured with the bro_alert output plugin.
global barnyard_alert: event(id: Barnyard2::PacketID,
alert: Barnyard2::AlertData,

7
scripts/policy/misc/trim-trace-file.zeek
View file

@ -6,7 +6,7 @@ module TrimTraceFile;
export {
## The interval between times that the output tracefile is rotated.
const trim_interval = 10 mins &redef;
## This event can be generated externally to this script if on-demand
## tracefile rotation is required with the caveat that the script
## doesn't currently attempt to get back on schedule automatically and
@ -19,14 +19,14 @@ event TrimTraceFile::go(first_trim: bool)
{
if ( zeek_is_terminating() || trace_output_file == "" )
return;
if ( ! first_trim )
{
local info = rotate_file_by_name(trace_output_file);
if ( info$old_name != "" )
system(fmt("/bin/rm %s", safe_shell_quote(info$new_name)));
}
schedule trim_interval { TrimTraceFile::go(F) };
}
@ -35,4 +35,3 @@ event zeek_init()
if ( trim_interval > 0 secs )
schedule trim_interval { TrimTraceFile::go(T) };
}

10
scripts/policy/protocols/conn/known-hosts.zeek
View file

@ -1,5 +1,5 @@
##! This script logs hosts that Zeek determines have performed complete TCP
##! handshakes and logs the address once per day (by default). The log that
##! handshakes and logs the address once per day (by default). The log that
##! is output provides an easy way to determine a count of the IP addresses in
##! use on a network per day.
@ -29,11 +29,11 @@ export {
## with keys uniformly distributed over proxy nodes in cluster
## operation.
const use_host_store = T &redef;
## The hosts whose existence should be logged and tracked.
## See :zeek:type:`Host` for possible choices.
option host_tracking = LOCAL_HOSTS;
## Holds the set of all known hosts. Keys in the store are addresses
## and their associated value will always be the "true" boolean.
global host_store: Cluster::StoreInfo;
@ -49,8 +49,8 @@ export {
## :zeek:see:`Known::host_store`.
option host_store_timeout = 15sec;
## The set of all known addresses to store for preventing duplicate
## logging of addresses. It can also be used from other scripts to
## The set of all known addresses to store for preventing duplicate
## logging of addresses. It can also be used from other scripts to
## inspect if an address has been seen in use.
## Maintain the list of known hosts for 24 hours so that the existence
## of each individual address is logged each day.

7
scripts/policy/protocols/conn/known-services.zeek
View file

@ -84,7 +84,7 @@ export {
}
redef record connection += {
# This field is to indicate whether or not the processing for detecting
# This field is to indicate whether or not the processing for detecting
# and logging the service for this connection is complete.
known_services_done: bool &default=F;
};
@ -262,7 +262,7 @@ function known_services_done(c: connection)
}
if ( ! has_active_service(c) )
# If we're here during a protocol_confirmation, it's still premature
# If we're here during a analyzer_confirmation, it's still premature
# to declare there's an actual service, so wait for the connection
# removal to check again (to get more timely reporting we'd have
# schedule some recurring event to poll for handshake/activity).
@ -293,7 +293,7 @@ function known_services_done(c: connection)
event service_info_commit(info);
}
event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=-5
event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) &priority=-5
{
known_services_done(c);
}
@ -314,4 +314,3 @@ event zeek_init() &priority=5
$path="known_services",
$policy=log_policy_services]);
}

10
scripts/policy/protocols/dns/detect-external-names.zeek
View file

@ -1,6 +1,6 @@
##! This script detects names which are not within zones considered to be
##! local but resolving to addresses considered local.
##! The :zeek:id:`Site::local_zones` variable **must** be set appropriately for
##! local but resolving to addresses considered local.
##! The :zeek:id:`Site::local_zones` variable **must** be set appropriately for
##! this detection.
@load base/frameworks/notice
@ -9,7 +9,7 @@
module DNS;
export {
redef enum Notice::Type += {
redef enum Notice::Type += {
## Raised when a non-local name is found to be pointing at a
## local host. The :zeek:id:`Site::local_zones` variable
## **must** be set appropriately for this detection.
@ -21,7 +21,7 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori
{
if ( |Site::local_zones| == 0 )
return;
# Check for responses from remote hosts that point at local hosts
# but the name is not considered to be within a "local" zone.
if ( Site::is_local_addr(a) && # referring to a local host
@ -29,7 +29,7 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori
{
NOTICE([$note=External_Name,
$msg=fmt("%s is pointing to a local host - %s.", ans$query, a),
$conn=c,
$conn=c,
$identifier=cat(a,ans$query)]);
}
}

6
scripts/policy/protocols/ftp/detect.zeek
View file

@ -7,7 +7,7 @@ module FTP;
export {
redef enum Notice::Type += {
## Indicates that a successful response to a "SITE EXEC"
## Indicates that a successful response to a "SITE EXEC"
## command/arg pair was seen.
Site_Exec_Success,
};
@ -16,10 +16,10 @@ export {
event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &priority=3
{
local response_xyz = parse_ftp_reply_code(code);
# If a successful SITE EXEC command is executed, raise a notice.
if ( response_xyz$x == 2 &&
c$ftp$cmdarg$cmd == "SITE" &&
c$ftp$cmdarg$cmd == "SITE" &&
/[Ee][Xx][Ee][Cc]/ in c$ftp$cmdarg$arg )
{
NOTICE([$note=Site_Exec_Success, $conn=c,

2
scripts/policy/protocols/http/detect-webapps.zeek
View file

@ -26,7 +26,7 @@ export {
event signature_match(state: signature_state, msg: string, data: string) &priority=5
{
if ( /^webapp-/ !in state$sig_id ) return;
local c = state$conn;
local si: Software::Info;
si = [$name=msg, $unparsed_version=msg, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=WEB_APPLICATION];

6
scripts/policy/protocols/http/header-names.zeek
View file

@ -11,15 +11,15 @@ export {
## The vector of HTTP header names sent by the client. No
## header values are included here, just the header names.
client_header_names: vector of string &log &optional;
## The vector of HTTP header names sent by the server. No
## header values are included here, just the header names.
server_header_names: vector of string &log &optional;
};
## A boolean value to determine if client header names are to be logged.
option log_client_header_names = T;
## A boolean value to determine if server header names are to be logged.
option log_server_header_names = F;
}

Some files were not shown because too many files have changed in this diff Show more