mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 14:48:21 +00:00
Merge remote-tracking branch 'origin/topic/johanna/new-style-analyzer-log'
* origin/topic/johanna/new-style-analyzer-log: NEWS entries for analyzer log changes Move detect-protocol from frameworks/dpd to frameworks/analyzer Introduce new c$failed_analyzers field Settle on analyzer.log for the dpd.log replacement dpd->analyzer.log change - rename files Analyzer failure logging: tweaks and test fixes Introduce analyzer-failed.log, as a replacement for dpd.log Rename analyzer.log to analyzer.debug log; move to policy Move dpd.log to policy script
This commit is contained in:
commit
e6755325e1
127 changed files with 5151 additions and 4742 deletions
207
scripts/policy/frameworks/analyzer/debug-logging.zeek
Normal file
207
scripts/policy/frameworks/analyzer/debug-logging.zeek
Normal file
|
@ -0,0 +1,207 @@
|
|||
##! Logging analyzer confirmations and violations into analyzer-debug.log
|
||||
|
||||
@load base/frameworks/config
|
||||
@load base/frameworks/logging
|
||||
@load base/frameworks/analyzer
|
||||
|
||||
module Analyzer::DebugLogging;
|
||||
|
||||
export {
|
||||
## Add the analyzer logging stream identifier.
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
## A default logging policy hook for the stream.
|
||||
global log_policy: Log::PolicyHook;
|
||||
|
||||
## The record type defining the columns to log in the analyzer logging stream.
|
||||
type Info: record {
|
||||
## Timestamp of confirmation or violation.
|
||||
ts: time &log;
|
||||
## What caused this log entry to be produced. This can
|
||||
## currently be "violation", "confirmation", or "disabled".
|
||||
cause: string &log;
|
||||
## The kind of analyzer involved. Currently "packet", "file"
|
||||
## or "protocol".
|
||||
analyzer_kind: string &log;
|
||||
## The name of the analyzer as produced by :zeek:see:`Analyzer::name`
|
||||
## for the analyzer's tag.
|
||||
analyzer_name: string &log;
|
||||
## Connection UID if available.
|
||||
uid: string &log &optional;
|
||||
## File UID if available.
|
||||
fuid: string &log &optional;
|
||||
## Connection identifier if available
|
||||
id: conn_id &log &optional;
|
||||
|
||||
## Failure or violation reason, if available.
|
||||
failure_reason: string &log &optional;
|
||||
|
||||
## Data causing failure or violation if available. Truncated
|
||||
## to :zeek:see:`Analyzer::DebugLogging::failure_data_max_size`.
|
||||
failure_data: string &log &optional;
|
||||
};
|
||||
|
||||
## Enable logging of analyzer violations and optionally confirmations
|
||||
## when :zeek:see:`Analyzer::DebugLogging::include_confirmations` is set.
|
||||
option enable = T;
|
||||
|
||||
## Enable analyzer_confirmation. They are usually less interesting
|
||||
## outside of development of analyzers or troubleshooting scenarios.
|
||||
## Setting this option may also generated multiple log entries per
|
||||
## connection, minimally one for each conn.log entry with a populated
|
||||
## service field.
|
||||
option include_confirmations = T;
|
||||
|
||||
## Enable tracking of analyzers getting disabled. This is mostly
|
||||
## interesting for troubleshooting of analyzers in DPD scenarios.
|
||||
## Setting this option may also generated multiple log entries per
|
||||
## connection.
|
||||
option include_disabling = T;
|
||||
|
||||
## If a violation contains information about the data causing it,
|
||||
## include at most this many bytes of it in the log.
|
||||
option failure_data_max_size = 40;
|
||||
|
||||
## Set of analyzers for which to not log confirmations or violations.
|
||||
option ignore_analyzers: set[AllAnalyzers::Tag] = set();
|
||||
}
|
||||
|
||||
|
||||
event zeek_init() &priority=5
|
||||
{
|
||||
Log::create_stream(LOG, [$columns=Info, $path="analyzer_debug", $policy=log_policy,
|
||||
$event_groups=set("Analyzer::DebugLogging")]);
|
||||
|
||||
local enable_handler = function(id: string, new_value: bool): bool {
|
||||
if ( new_value )
|
||||
Log::enable_stream(LOG);
|
||||
else
|
||||
Log::disable_stream(LOG);
|
||||
|
||||
return new_value;
|
||||
};
|
||||
|
||||
Option::set_change_handler("Analyzer::DebugLogging::enable", enable_handler);
|
||||
|
||||
local include_confirmations_handler = function(id: string, new_value: bool): bool {
|
||||
if ( new_value )
|
||||
enable_event_group("Analyzer::DebugLogging::include_confirmations");
|
||||
else
|
||||
disable_event_group("Analyzer::DebugLogging::include_confirmations");
|
||||
|
||||
return new_value;
|
||||
};
|
||||
|
||||
Option::set_change_handler("Analyzer::DebugLogging::include_confirmations",
|
||||
include_confirmations_handler);
|
||||
|
||||
local include_disabling_handler = function(id: string, new_value: bool): bool {
|
||||
if ( new_value )
|
||||
enable_event_group("Analyzer::DebugLogging::include_disabling");
|
||||
else
|
||||
disable_event_group("Analyzer::DebugLogging::include_disabling");
|
||||
|
||||
return new_value;
|
||||
};
|
||||
|
||||
Option::set_change_handler("Analyzer::DebugLogging::include_disabling",
|
||||
include_disabling_handler);
|
||||
|
||||
# Call the handlers directly with the current values to avoid config
|
||||
# framework interactions like creating entries in config.log.
|
||||
enable_handler("Analyzer::DebugLogging::enable", Analyzer::DebugLogging::enable);
|
||||
include_confirmations_handler("Analyzer::DebugLogging::include_confirmations",
|
||||
Analyzer::DebugLogging::include_confirmations);
|
||||
include_disabling_handler("Analyzer::DebugLogging::include_disabling",
|
||||
Analyzer::DebugLogging::include_disabling);
|
||||
|
||||
}
|
||||
|
||||
function populate_from_conn(rec: Info, c: connection)
|
||||
{
|
||||
rec$id = c$id;
|
||||
rec$uid = c$uid;
|
||||
}
|
||||
|
||||
function populate_from_file(rec: Info, f: fa_file)
|
||||
{
|
||||
rec$fuid = f$id;
|
||||
# If the confirmation didn't have a connection, but the
|
||||
# fa_file object has exactly one, use it.
|
||||
if ( ! rec?$uid && f?$conns && |f$conns| == 1 )
|
||||
{
|
||||
for ( _, c in f$conns )
|
||||
{
|
||||
rec$id = c$id;
|
||||
rec$uid = c$uid;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
event analyzer_confirmation_info(atype: AllAnalyzers::Tag, info: AnalyzerConfirmationInfo) &group="Analyzer::DebugLogging::include_confirmations"
|
||||
{
|
||||
if ( atype in ignore_analyzers )
|
||||
return;
|
||||
|
||||
local rec = Info(
|
||||
$ts=network_time(),
|
||||
$cause="confirmation",
|
||||
$analyzer_kind=Analyzer::kind(atype),
|
||||
$analyzer_name=Analyzer::name(atype),
|
||||
);
|
||||
|
||||
if ( info?$c )
|
||||
populate_from_conn(rec, info$c);
|
||||
|
||||
if ( info?$f )
|
||||
populate_from_file(rec, info$f);
|
||||
|
||||
Log::write(LOG, rec);
|
||||
}
|
||||
|
||||
event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo) &priority=6
|
||||
{
|
||||
if ( atype in ignore_analyzers )
|
||||
return;
|
||||
|
||||
local rec = Info(
|
||||
$ts=network_time(),
|
||||
$cause="violation",
|
||||
$analyzer_kind=Analyzer::kind(atype),
|
||||
$analyzer_name=Analyzer::name(atype),
|
||||
$failure_reason=info$reason,
|
||||
);
|
||||
|
||||
if ( info?$c )
|
||||
populate_from_conn(rec, info$c);
|
||||
|
||||
if ( info?$f )
|
||||
populate_from_file(rec, info$f);
|
||||
|
||||
if ( info?$data )
|
||||
{
|
||||
if ( failure_data_max_size > 0 )
|
||||
rec$failure_data = info$data[0:failure_data_max_size];
|
||||
else
|
||||
rec$failure_data = info$data;
|
||||
}
|
||||
|
||||
Log::write(LOG, rec);
|
||||
}
|
||||
|
||||
hook Analyzer::disabling_analyzer(c: connection, atype: AllAnalyzers::Tag, aid: count) &priority=-1000 &group="Analyzer::DebugLogging::include_disabling"
|
||||
{
|
||||
if ( atype in ignore_analyzers )
|
||||
return;
|
||||
|
||||
local rec = Info(
|
||||
$ts=network_time(),
|
||||
$cause="disabled",
|
||||
$analyzer_kind=Analyzer::kind(atype),
|
||||
$analyzer_name=Analyzer::name(atype),
|
||||
);
|
||||
|
||||
populate_from_conn(rec, c);
|
||||
|
||||
Log::write(LOG, rec);
|
||||
}
|
90
scripts/policy/frameworks/analyzer/deprecated-dpd-log.zeek
Normal file
90
scripts/policy/frameworks/analyzer/deprecated-dpd-log.zeek
Normal file
|
@ -0,0 +1,90 @@
|
|||
##! Creates the now deprecated dpd.logfile.
|
||||
# Remove in v8.1
|
||||
|
||||
module DPD;
|
||||
|
||||
export {
|
||||
## Add the DPD logging stream identifier.
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
## A default logging policy hook for the stream.
|
||||
global log_policy: Log::PolicyHook;
|
||||
|
||||
## The record type defining the columns to log in the DPD logging stream.
|
||||
type Info: record {
|
||||
## Timestamp for when protocol analysis failed.
|
||||
ts: time &log;
|
||||
## Connection unique ID.
|
||||
uid: string &log;
|
||||
## Connection ID containing the 4-tuple which identifies endpoints.
|
||||
id: conn_id &log;
|
||||
## Transport protocol for the violation.
|
||||
proto: transport_proto &log;
|
||||
## The analyzer that generated the violation.
|
||||
analyzer: string &log;
|
||||
## The textual reason for the analysis failure.
|
||||
failure_reason: string &log;
|
||||
};
|
||||
}
|
||||
|
||||
redef record connection += {
|
||||
dpd: Info &optional;
|
||||
};
|
||||
|
||||
event zeek_init() &priority=5
|
||||
{
|
||||
Log::create_stream(DPD::LOG, [$columns=Info, $path="dpd", $policy=log_policy]);
|
||||
}
|
||||
|
||||
# before the same event in dpd.zeek
|
||||
event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo) &priority=15
|
||||
{
|
||||
if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
|
||||
return;
|
||||
|
||||
if ( ! info?$c )
|
||||
return;
|
||||
|
||||
local c = info$c;
|
||||
local analyzer = Analyzer::name(atype);
|
||||
# If the service hasn't been confirmed yet, or already failed,
|
||||
# don't generate a log message for the protocol violation.
|
||||
if ( analyzer !in c$service || analyzer in c$service_violation )
|
||||
return;
|
||||
|
||||
local dpd: Info;
|
||||
dpd$ts = network_time();
|
||||
dpd$uid = c$uid;
|
||||
dpd$id = c$id;
|
||||
dpd$proto = get_port_transport_proto(c$id$orig_p);
|
||||
dpd$analyzer = analyzer;
|
||||
|
||||
# Encode data into the reason if there's any as done for the old
|
||||
# analyzer_violation event, previously.
|
||||
local reason = info$reason;
|
||||
if ( info?$data )
|
||||
{
|
||||
local ellipsis = |info$data| > 40 ? "..." : "";
|
||||
local data = info$data[0:40];
|
||||
reason = fmt("%s [%s%s]", reason, data, ellipsis);
|
||||
}
|
||||
|
||||
dpd$failure_reason = reason;
|
||||
c$dpd = dpd;
|
||||
}
|
||||
|
||||
event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo ) &priority=-5
|
||||
{
|
||||
if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
|
||||
return;
|
||||
|
||||
if ( ! info?$c )
|
||||
return;
|
||||
|
||||
if ( info$c?$dpd )
|
||||
{
|
||||
Log::write(DPD::LOG, info$c$dpd);
|
||||
delete info$c$dpd;
|
||||
}
|
||||
}
|
||||
|
238
scripts/policy/frameworks/analyzer/detect-protocols.zeek
Normal file
238
scripts/policy/frameworks/analyzer/detect-protocols.zeek
Normal file
|
@ -0,0 +1,238 @@
|
|||
##! Finds connections with protocols on non-standard ports with DPD.
|
||||
|
||||
@load base/frameworks/notice
|
||||
@load base/utils/site
|
||||
@load base/utils/conn-ids
|
||||
@load base/protocols/conn/removal-hooks
|
||||
|
||||
module ProtocolDetector;
|
||||
|
||||
export {
|
||||
redef enum Notice::Type += {
|
||||
Protocol_Found,
|
||||
Server_Found,
|
||||
};
|
||||
|
||||
# Table of (protocol, resp_h, resp_p) tuples known to be uninteresting
|
||||
# in the given direction. For all other protocols detected on
|
||||
# non-standard ports, we raise a Protocol_Found notice. (More specific
|
||||
# filtering can then be done via notice_filters.)
|
||||
#
|
||||
# Use 0.0.0.0 for to wildcard-match any resp_h.
|
||||
|
||||
type dir: enum { NONE, INCOMING, OUTGOING, BOTH };
|
||||
|
||||
option valids: table[AllAnalyzers::Tag, addr, port] of dir = {
|
||||
# A couple of ports commonly used for benign HTTP servers.
|
||||
|
||||
# For now we want to see everything.
|
||||
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 81/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 82/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 83/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 88/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 8001/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 8090/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 8081/tcp] = OUTGOING,
|
||||
#
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 6346/tcp] = BOTH, # Gnutella
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 6347/tcp] = BOTH, # Gnutella
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 6348/tcp] = BOTH, # Gnutella
|
||||
};
|
||||
|
||||
# Set of analyzers for which we suppress Server_Found notices
|
||||
# (but not Protocol_Found). Along with avoiding clutter in the
|
||||
# log files, this also saves memory because for these we don't
|
||||
# need to remember which servers we already have reported, which
|
||||
# for some can be a lot.
|
||||
option suppress_servers: set [AllAnalyzers::Tag] = {
|
||||
# Analyzer::ANALYZER_HTTP
|
||||
};
|
||||
|
||||
# We consider a connection to use a protocol X if the analyzer for X
|
||||
# is still active (i) after an interval of minimum_duration, or (ii)
|
||||
# after a payload volume of minimum_volume, or (iii) at the end of the
|
||||
# connection.
|
||||
option minimum_duration = 30 secs;
|
||||
option minimum_volume = 4e3; # bytes
|
||||
|
||||
# How often to check the size of the connection.
|
||||
const check_interval = 5 secs;
|
||||
|
||||
# Entry point for other analyzers to report that they recognized
|
||||
# a certain (sub-)protocol.
|
||||
global found_protocol: function(c: connection, analyzer: AllAnalyzers::Tag,
|
||||
protocol: string);
|
||||
|
||||
# Table keeping reported (server, port, analyzer) tuples (and their
|
||||
# reported sub-protocols).
|
||||
global servers: table[addr, port, string] of set[string]
|
||||
&read_expire = 14 days;
|
||||
|
||||
## Non-standard protocol port detection finalization hook.
|
||||
global finalize_protocol_detection: Conn::RemovalHook;
|
||||
}
|
||||
|
||||
# Table that tracks currently active dynamic analyzers per connection.
|
||||
global conns: table[conn_id] of set[AllAnalyzers::Tag];
|
||||
|
||||
# Table of reports by other analyzers about the protocol used in a connection.
|
||||
global protocols: table[conn_id] of set[string];
|
||||
|
||||
type protocol : record {
|
||||
a: string; # analyzer name
|
||||
sub: string; # "sub-protocols" reported by other sources
|
||||
};
|
||||
|
||||
function get_protocol(c: connection, a: AllAnalyzers::Tag) : protocol
|
||||
{
|
||||
local str = "";
|
||||
if ( c$id in protocols )
|
||||
{
|
||||
for ( p in protocols[c$id] )
|
||||
str = |str| > 0 ? fmt("%s/%s", str, p) : p;
|
||||
}
|
||||
|
||||
return [$a=Analyzer::name(a), $sub=str];
|
||||
}
|
||||
|
||||
function fmt_protocol(p: protocol) : string
|
||||
{
|
||||
return p$sub != "" ? fmt("%s (via %s)", p$sub, p$a) : p$a;
|
||||
}
|
||||
|
||||
function do_notice(c: connection, a: AllAnalyzers::Tag, d: dir)
|
||||
{
|
||||
if ( d == BOTH )
|
||||
return;
|
||||
|
||||
if ( d == INCOMING && Site::is_local_addr(c$id$resp_h) )
|
||||
return;
|
||||
|
||||
if ( d == OUTGOING && ! Site::is_local_addr(c$id$resp_h) )
|
||||
return;
|
||||
|
||||
local p = get_protocol(c, a);
|
||||
local s = fmt_protocol(p);
|
||||
|
||||
NOTICE([$note=Protocol_Found,
|
||||
$msg=fmt("%s %s on port %s", id_string(c$id), s, c$id$resp_p),
|
||||
$sub=s, $conn=c]);
|
||||
|
||||
# We report multiple Server_Found's per host if we find a new
|
||||
# sub-protocol.
|
||||
local known = [c$id$resp_h, c$id$resp_p, p$a] in servers;
|
||||
|
||||
local newsub = F;
|
||||
if ( known )
|
||||
newsub = (p$sub != "" &&
|
||||
p$sub !in servers[c$id$resp_h, c$id$resp_p, p$a]);
|
||||
|
||||
if ( (! known || newsub) && a !in suppress_servers )
|
||||
{
|
||||
NOTICE([$note=Server_Found,
|
||||
$msg=fmt("%s: %s server on port %s%s", c$id$resp_h, s,
|
||||
c$id$resp_p, (known ? " (update)" : "")),
|
||||
$p=c$id$resp_p, $sub=s, $conn=c, $src=c$id$resp_h]);
|
||||
|
||||
if ( ! known )
|
||||
servers[c$id$resp_h, c$id$resp_p, p$a] = set();
|
||||
|
||||
add servers[c$id$resp_h, c$id$resp_p, p$a][p$sub];
|
||||
}
|
||||
}
|
||||
|
||||
function report_protocols(c: connection)
|
||||
{
|
||||
# We only report the connection if both sides have transferred data.
|
||||
if ( c$resp$size == 0 || c$orig$size == 0 )
|
||||
{
|
||||
delete conns[c$id];
|
||||
delete protocols[c$id];
|
||||
return;
|
||||
}
|
||||
|
||||
local analyzers = conns[c$id];
|
||||
|
||||
for ( a in analyzers )
|
||||
{
|
||||
if ( [a, c$id$resp_h, c$id$resp_p] in valids )
|
||||
do_notice(c, a, valids[a, c$id$resp_h, c$id$resp_p]);
|
||||
else if ( [a, 0.0.0.0, c$id$resp_p] in valids )
|
||||
do_notice(c, a, valids[a, 0.0.0.0, c$id$resp_p]);
|
||||
else
|
||||
do_notice(c, a, NONE);
|
||||
}
|
||||
|
||||
delete conns[c$id];
|
||||
delete protocols[c$id];
|
||||
}
|
||||
|
||||
event ProtocolDetector::check_connection(c: connection)
|
||||
{
|
||||
if ( c$id !in conns )
|
||||
return;
|
||||
|
||||
local duration = network_time() - c$start_time;
|
||||
local size = c$resp$size + c$orig$size;
|
||||
|
||||
if ( duration >= minimum_duration || size >= minimum_volume )
|
||||
report_protocols(c);
|
||||
else
|
||||
{
|
||||
local delay = min_interval(minimum_duration - duration,
|
||||
check_interval);
|
||||
schedule delay { ProtocolDetector::check_connection(c) };
|
||||
}
|
||||
}
|
||||
|
||||
hook finalize_protocol_detection(c: connection)
|
||||
{
|
||||
if ( c$id !in conns )
|
||||
{
|
||||
delete protocols[c$id];
|
||||
return;
|
||||
}
|
||||
|
||||
# Reports all analyzers that have remained to the end.
|
||||
report_protocols(c);
|
||||
}
|
||||
|
||||
event analyzer_confirmation_info(atype: AllAnalyzers::Tag, info: AnalyzerConfirmationInfo)
|
||||
{
|
||||
if ( ! is_protocol_analyzer(atype) )
|
||||
return;
|
||||
|
||||
local c = info$c;
|
||||
|
||||
# Don't report anything running on a well-known port.
|
||||
if ( c$id$resp_p in Analyzer::registered_ports(atype) )
|
||||
return;
|
||||
|
||||
if ( c$id in conns )
|
||||
{
|
||||
local analyzers = conns[c$id];
|
||||
add analyzers[atype];
|
||||
}
|
||||
else
|
||||
{
|
||||
conns[c$id] = set(atype);
|
||||
Conn::register_removal_hook(c, finalize_protocol_detection);
|
||||
|
||||
local delay = min_interval(minimum_duration, check_interval);
|
||||
schedule delay { ProtocolDetector::check_connection(c) };
|
||||
}
|
||||
}
|
||||
|
||||
function found_protocol(c: connection, atype: AllAnalyzers::Tag, protocol: string)
|
||||
{
|
||||
# Don't report anything running on a well-known port.
|
||||
if ( c$id$resp_p in Analyzer::registered_ports(atype) )
|
||||
return;
|
||||
|
||||
if ( c$id !in protocols )
|
||||
protocols[c$id] = set();
|
||||
|
||||
add protocols[c$id][protocol];
|
||||
Conn::register_removal_hook(c, finalize_protocol_detection);
|
||||
}
|
|
@ -0,0 +1,50 @@
|
|||
##! This script enables logging of packet segment data when a protocol
|
||||
##! parsing violation is encountered. The amount of data from the
|
||||
##! packet logged is set by the :zeek:see:`Analyzer::Logging::packet_segment_size` variable.
|
||||
##! A caveat to logging packet data is that in some cases, the packet may
|
||||
##! not be the packet that actually caused the protocol violation.
|
||||
|
||||
module Analyzer::Logging;
|
||||
|
||||
export {
|
||||
redef record connection += {
|
||||
## A chunk of the payload that most likely resulted in a
|
||||
## analyzer violation.
|
||||
packet_segment: string &optional &log;
|
||||
};
|
||||
|
||||
redef record Analyzer::Logging::Info += {
|
||||
## A chunk of the payload that most likely resulted in the
|
||||
## analyzer violation.
|
||||
packet_segment: string &optional &log;
|
||||
};
|
||||
|
||||
## Size of the packet segment to display in the DPD log.
|
||||
option packet_segment_size: int = 255;
|
||||
}
|
||||
|
||||
# stash the packet segment in the event causing the violation, so that it can be retrieved later.
|
||||
event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo) &priority=4
|
||||
{
|
||||
if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
|
||||
return;
|
||||
|
||||
if ( ! info?$c || ! info?$aid )
|
||||
return;
|
||||
|
||||
info$c$packet_segment = fmt("%s", get_current_packet()$data[:packet_segment_size]);
|
||||
}
|
||||
|
||||
hook Analyzer::Logging::log_policy(rec: Analyzer::Logging::Info, id: Log::ID, filter: Log::Filter)
|
||||
{
|
||||
if ( id != Analyzer::Logging::LOG )
|
||||
return;
|
||||
|
||||
if ( ! rec?$id || ! connection_exists(rec$id) )
|
||||
return;
|
||||
|
||||
local c = lookup_connection(rec$id);
|
||||
|
||||
if ( c?$packet_segment )
|
||||
rec$packet_segment = c$packet_segment;
|
||||
}
|
|
@ -1,238 +1,3 @@
|
|||
##! Finds connections with protocols on non-standard ports with DPD.
|
||||
@deprecated("frameworks/dpd/detect-protocols.zeek moved to frameworks/analyzer/detect-protocols.zeek. Please switch to frameworks/analyzer/detect-protocols.zeek. Remove in 8.1")
|
||||
|
||||
@load base/frameworks/notice
|
||||
@load base/utils/site
|
||||
@load base/utils/conn-ids
|
||||
@load base/protocols/conn/removal-hooks
|
||||
|
||||
module ProtocolDetector;
|
||||
|
||||
export {
|
||||
redef enum Notice::Type += {
|
||||
Protocol_Found,
|
||||
Server_Found,
|
||||
};
|
||||
|
||||
# Table of (protocol, resp_h, resp_p) tuples known to be uninteresting
|
||||
# in the given direction. For all other protocols detected on
|
||||
# non-standard ports, we raise a Protocol_Found notice. (More specific
|
||||
# filtering can then be done via notice_filters.)
|
||||
#
|
||||
# Use 0.0.0.0 for to wildcard-match any resp_h.
|
||||
|
||||
type dir: enum { NONE, INCOMING, OUTGOING, BOTH };
|
||||
|
||||
option valids: table[AllAnalyzers::Tag, addr, port] of dir = {
|
||||
# A couple of ports commonly used for benign HTTP servers.
|
||||
|
||||
# For now we want to see everything.
|
||||
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 81/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 82/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 83/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 88/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 8001/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 8090/tcp] = OUTGOING,
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 8081/tcp] = OUTGOING,
|
||||
#
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 6346/tcp] = BOTH, # Gnutella
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 6347/tcp] = BOTH, # Gnutella
|
||||
# [Analyzer::ANALYZER_HTTP, 0.0.0.0, 6348/tcp] = BOTH, # Gnutella
|
||||
};
|
||||
|
||||
# Set of analyzers for which we suppress Server_Found notices
|
||||
# (but not Protocol_Found). Along with avoiding clutter in the
|
||||
# log files, this also saves memory because for these we don't
|
||||
# need to remember which servers we already have reported, which
|
||||
# for some can be a lot.
|
||||
option suppress_servers: set [AllAnalyzers::Tag] = {
|
||||
# Analyzer::ANALYZER_HTTP
|
||||
};
|
||||
|
||||
# We consider a connection to use a protocol X if the analyzer for X
|
||||
# is still active (i) after an interval of minimum_duration, or (ii)
|
||||
# after a payload volume of minimum_volume, or (iii) at the end of the
|
||||
# connection.
|
||||
option minimum_duration = 30 secs;
|
||||
option minimum_volume = 4e3; # bytes
|
||||
|
||||
# How often to check the size of the connection.
|
||||
const check_interval = 5 secs;
|
||||
|
||||
# Entry point for other analyzers to report that they recognized
|
||||
# a certain (sub-)protocol.
|
||||
global found_protocol: function(c: connection, analyzer: AllAnalyzers::Tag,
|
||||
protocol: string);
|
||||
|
||||
# Table keeping reported (server, port, analyzer) tuples (and their
|
||||
# reported sub-protocols).
|
||||
global servers: table[addr, port, string] of set[string]
|
||||
&read_expire = 14 days;
|
||||
|
||||
## Non-standard protocol port detection finalization hook.
|
||||
global finalize_protocol_detection: Conn::RemovalHook;
|
||||
}
|
||||
|
||||
# Table that tracks currently active dynamic analyzers per connection.
|
||||
global conns: table[conn_id] of set[AllAnalyzers::Tag];
|
||||
|
||||
# Table of reports by other analyzers about the protocol used in a connection.
|
||||
global protocols: table[conn_id] of set[string];
|
||||
|
||||
type protocol : record {
|
||||
a: string; # analyzer name
|
||||
sub: string; # "sub-protocols" reported by other sources
|
||||
};
|
||||
|
||||
function get_protocol(c: connection, a: AllAnalyzers::Tag) : protocol
|
||||
{
|
||||
local str = "";
|
||||
if ( c$id in protocols )
|
||||
{
|
||||
for ( p in protocols[c$id] )
|
||||
str = |str| > 0 ? fmt("%s/%s", str, p) : p;
|
||||
}
|
||||
|
||||
return [$a=Analyzer::name(a), $sub=str];
|
||||
}
|
||||
|
||||
function fmt_protocol(p: protocol) : string
|
||||
{
|
||||
return p$sub != "" ? fmt("%s (via %s)", p$sub, p$a) : p$a;
|
||||
}
|
||||
|
||||
function do_notice(c: connection, a: AllAnalyzers::Tag, d: dir)
|
||||
{
|
||||
if ( d == BOTH )
|
||||
return;
|
||||
|
||||
if ( d == INCOMING && Site::is_local_addr(c$id$resp_h) )
|
||||
return;
|
||||
|
||||
if ( d == OUTGOING && ! Site::is_local_addr(c$id$resp_h) )
|
||||
return;
|
||||
|
||||
local p = get_protocol(c, a);
|
||||
local s = fmt_protocol(p);
|
||||
|
||||
NOTICE([$note=Protocol_Found,
|
||||
$msg=fmt("%s %s on port %s", id_string(c$id), s, c$id$resp_p),
|
||||
$sub=s, $conn=c]);
|
||||
|
||||
# We report multiple Server_Found's per host if we find a new
|
||||
# sub-protocol.
|
||||
local known = [c$id$resp_h, c$id$resp_p, p$a] in servers;
|
||||
|
||||
local newsub = F;
|
||||
if ( known )
|
||||
newsub = (p$sub != "" &&
|
||||
p$sub !in servers[c$id$resp_h, c$id$resp_p, p$a]);
|
||||
|
||||
if ( (! known || newsub) && a !in suppress_servers )
|
||||
{
|
||||
NOTICE([$note=Server_Found,
|
||||
$msg=fmt("%s: %s server on port %s%s", c$id$resp_h, s,
|
||||
c$id$resp_p, (known ? " (update)" : "")),
|
||||
$p=c$id$resp_p, $sub=s, $conn=c, $src=c$id$resp_h]);
|
||||
|
||||
if ( ! known )
|
||||
servers[c$id$resp_h, c$id$resp_p, p$a] = set();
|
||||
|
||||
add servers[c$id$resp_h, c$id$resp_p, p$a][p$sub];
|
||||
}
|
||||
}
|
||||
|
||||
function report_protocols(c: connection)
|
||||
{
|
||||
# We only report the connection if both sides have transferred data.
|
||||
if ( c$resp$size == 0 || c$orig$size == 0 )
|
||||
{
|
||||
delete conns[c$id];
|
||||
delete protocols[c$id];
|
||||
return;
|
||||
}
|
||||
|
||||
local analyzers = conns[c$id];
|
||||
|
||||
for ( a in analyzers )
|
||||
{
|
||||
if ( [a, c$id$resp_h, c$id$resp_p] in valids )
|
||||
do_notice(c, a, valids[a, c$id$resp_h, c$id$resp_p]);
|
||||
else if ( [a, 0.0.0.0, c$id$resp_p] in valids )
|
||||
do_notice(c, a, valids[a, 0.0.0.0, c$id$resp_p]);
|
||||
else
|
||||
do_notice(c, a, NONE);
|
||||
}
|
||||
|
||||
delete conns[c$id];
|
||||
delete protocols[c$id];
|
||||
}
|
||||
|
||||
event ProtocolDetector::check_connection(c: connection)
|
||||
{
|
||||
if ( c$id !in conns )
|
||||
return;
|
||||
|
||||
local duration = network_time() - c$start_time;
|
||||
local size = c$resp$size + c$orig$size;
|
||||
|
||||
if ( duration >= minimum_duration || size >= minimum_volume )
|
||||
report_protocols(c);
|
||||
else
|
||||
{
|
||||
local delay = min_interval(minimum_duration - duration,
|
||||
check_interval);
|
||||
schedule delay { ProtocolDetector::check_connection(c) };
|
||||
}
|
||||
}
|
||||
|
||||
hook finalize_protocol_detection(c: connection)
|
||||
{
|
||||
if ( c$id !in conns )
|
||||
{
|
||||
delete protocols[c$id];
|
||||
return;
|
||||
}
|
||||
|
||||
# Reports all analyzers that have remained to the end.
|
||||
report_protocols(c);
|
||||
}
|
||||
|
||||
event analyzer_confirmation_info(atype: AllAnalyzers::Tag, info: AnalyzerConfirmationInfo)
|
||||
{
|
||||
if ( ! is_protocol_analyzer(atype) )
|
||||
return;
|
||||
|
||||
local c = info$c;
|
||||
|
||||
# Don't report anything running on a well-known port.
|
||||
if ( c$id$resp_p in Analyzer::registered_ports(atype) )
|
||||
return;
|
||||
|
||||
if ( c$id in conns )
|
||||
{
|
||||
local analyzers = conns[c$id];
|
||||
add analyzers[atype];
|
||||
}
|
||||
else
|
||||
{
|
||||
conns[c$id] = set(atype);
|
||||
Conn::register_removal_hook(c, finalize_protocol_detection);
|
||||
|
||||
local delay = min_interval(minimum_duration, check_interval);
|
||||
schedule delay { ProtocolDetector::check_connection(c) };
|
||||
}
|
||||
}
|
||||
|
||||
function found_protocol(c: connection, atype: AllAnalyzers::Tag, protocol: string)
|
||||
{
|
||||
# Don't report anything running on a well-known port.
|
||||
if ( c$id$resp_p in Analyzer::registered_ports(atype) )
|
||||
return;
|
||||
|
||||
if ( c$id !in protocols )
|
||||
protocols[c$id] = set();
|
||||
|
||||
add protocols[c$id][protocol];
|
||||
Conn::register_removal_hook(c, finalize_protocol_detection);
|
||||
}
|
||||
@load frameworks/analyzer/detect-protocols.zeek
|
||||
|
|
|
@ -1,9 +1,13 @@
|
|||
@deprecated("Please switch to frameworks/analyzer/packet-segment-logging, which logs to analyzer.log. Remove in 8.1")
|
||||
|
||||
##! This script enables logging of packet segment data when a protocol
|
||||
##! parsing violation is encountered. The amount of data from the
|
||||
##! packet logged is set by the :zeek:see:`DPD::packet_segment_size` variable.
|
||||
##! A caveat to logging packet data is that in some cases, the packet may
|
||||
##! not be the packet that actually caused the protocol violation.
|
||||
|
||||
@load frameworks/analyzer/deprecated-dpd-log
|
||||
|
||||
module DPD;
|
||||
|
||||
export {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue