mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Merge branch 'topic/jgras/dpd-late-match' of https://github.com/J-Gras/zeek
* 'topic/jgras/dpd-late-match' of https://github.com/J-Gras/zeek: Improve dpd_late_match event generation. Improve logging of speculative service. Update test-all-policy script. Add speculative service script. Allow to handle late DPD matches.
This commit is contained in:
Jon Siwek
commit
aeef4bf030
16 changed files with 186 additions and 2 deletions
13
CHANGES
13
CHANGES
|
|
@ -1,4 +1,17 @@
|
|||
|
||||
3.1.0-dev.124 | 2019-09-17 11:16:47 -0700
|
||||
|
||||
* Add speculative service script. (Jan Grashoefer)
|
||||
|
||||
The speculative service script handles dpd_late_match events to extend
|
||||
conn.log with infos about potential protocol identifications.
|
||||
|
||||
* Allow to handle late DPD matches. (Jan Grashoefer)
|
||||
|
||||
If "dpd_match_only_beginning" is disabled, matches of protocol signatures
|
||||
can be handeld using protocol_late_match. To prevent further matching in
|
||||
this case, the new option "dpd_late_match_stop" may be activated.
|
||||
|
||||
3.1.0-dev.118 | 2019-09-17 17:21:58 +0000
|
||||
|
||||
* GH-566: Fix cases where ssh_encrypted_packet event wasn't raised.
|
||||
|
|
|
|||
10
NEWS
10
NEWS
|
|
@ -9,6 +9,16 @@ Zeek 3.1.0
|
|||
New Functionality
|
||||
-----------------
|
||||
|
||||
- Add a new option, ``dpd_late_match_stop``, which can be used in conjuction
|
||||
with the option ``dpd_match_only_beginning`` and the new event
|
||||
``protocol_late_match`` to help annotate the conn.log with a field
|
||||
to speculate on the protocol/service in cases where the DPD buffer
|
||||
was already exhausted and can't analyze the full connection anymore,
|
||||
but where there was still a late signature match. A new script,
|
||||
``policy/protocols/conn/speculative-service.zeek``, was added as an
|
||||
example of how to perform this tuning and add a "speculative_service"
|
||||
field to conn.log, but it's not loaded by default.
|
||||
|
||||
Changed Functionality
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
3.1.0-dev.118
|
||||
3.1.0-dev.124
|
||||
|
|
|
|||
2
doc
2
doc
|
|
@ -1 +1 @@
|
|||
Subproject commit a3dd09cd5acfe6aa848a84eae5fdb18d64d590e1
|
||||
Subproject commit 218907b6500d627904f6ba0767f7ba362bf825f7
|
||||
|
|
@ -4684,6 +4684,18 @@ const dpd_buffer_size = 1024 &redef;
|
|||
## only signatures used for dynamic protocol detection.
|
||||
const dpd_match_only_beginning = T &redef;
|
||||
|
||||
## If true, stops signature matching after a late match. A late match may occur
|
||||
## in case the DPD buffer is exhausted but a protocol signature matched. To
|
||||
## allow late matching, :zeek:see:`dpd_match_only_beginning` must be disabled.
|
||||
##
|
||||
## .. zeek:see:: dpd_reassemble_first_packets dpd_buffer_size
|
||||
## dpd_match_only_beginning
|
||||
##
|
||||
## .. note:: Despite the name, this option stops *all* signature matching, not
|
||||
## only signatures used for dynamic protocol detection but is triggered by
|
||||
## DPD signatures only.
|
||||
const dpd_late_match_stop = F &redef;
|
||||
|
||||
## If true, don't consider any ports for deciding which protocol analyzer to
|
||||
## use.
|
||||
##
|
||||
|
|
|
|||
36
scripts/policy/protocols/conn/speculative-service.zeek
Normal file
36
scripts/policy/protocols/conn/speculative-service.zeek
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
##! This script adds information about matched DPD signatures to the connection
|
||||
##! log.
|
||||
|
||||
@load base/protocols/conn
|
||||
|
||||
module Conn;
|
||||
|
||||
redef record Info += {
|
||||
## Protocol that was determined by a matching signature after the beginning
|
||||
## of a connection. In this situation no analyzer can be attached and hence
|
||||
## the data cannot be analyzed nor the protocol can be confirmed.
|
||||
speculative_service: string &log &optional;
|
||||
};
|
||||
|
||||
redef record connection += {
|
||||
speculative_service: set[string] &default=string_set();
|
||||
};
|
||||
|
||||
redef dpd_match_only_beginning = F;
|
||||
redef dpd_late_match_stop = T;
|
||||
|
||||
event protocol_late_match(c: connection, atype: Analyzer::Tag)
|
||||
{
|
||||
local analyzer = Analyzer::name(atype);
|
||||
add c$speculative_service[analyzer];
|
||||
}
|
||||
|
||||
event connection_state_remove(c: connection)
|
||||
{
|
||||
local sp_service = "";
|
||||
for ( s in c$speculative_service )
|
||||
sp_service = sp_service == "" ? s : cat(sp_service, ",", s);
|
||||
|
||||
if ( sp_service != "" )
|
||||
c$conn$speculative_service = to_lower(sp_service);
|
||||
}
|
||||
|
|
@ -67,6 +67,7 @@
|
|||
@load protocols/conn/mac-logging.zeek
|
||||
@load protocols/conn/vlan-logging.zeek
|
||||
@load protocols/conn/weirds.zeek
|
||||
#@load protocols/conn/speculative-service.zeek
|
||||
@load protocols/dhcp/msg-orig.zeek
|
||||
@load protocols/dhcp/software.zeek
|
||||
@load protocols/dhcp/sub-opts.zeek
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
@load frameworks/control/controller.zeek
|
||||
@load frameworks/files/extract-all-files.zeek
|
||||
@load policy/misc/dump-events.zeek
|
||||
@load policy/protocols/conn/speculative-service.zeek
|
||||
|
||||
@load ./example.zeek
|
||||
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ RecordType* irc_join_info;
|
|||
int dpd_reassemble_first_packets;
|
||||
int dpd_buffer_size;
|
||||
int dpd_match_only_beginning;
|
||||
int dpd_late_match_stop;
|
||||
int dpd_ignore_ports;
|
||||
|
||||
TableVal* likely_server_ports;
|
||||
|
|
@ -406,6 +407,7 @@ void init_net_var()
|
|||
opt_internal_int("dpd_reassemble_first_packets");
|
||||
dpd_buffer_size = opt_internal_int("dpd_buffer_size");
|
||||
dpd_match_only_beginning = opt_internal_int("dpd_match_only_beginning");
|
||||
dpd_late_match_stop = opt_internal_int("dpd_late_match_stop");
|
||||
dpd_ignore_ports = opt_internal_int("dpd_ignore_ports");
|
||||
|
||||
likely_server_ports = internal_val("likely_server_ports")->AsTableVal();
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ extern RecordType* irc_join_info;
|
|||
extern int dpd_reassemble_first_packets;
|
||||
extern int dpd_buffer_size;
|
||||
extern int dpd_match_only_beginning;
|
||||
extern int dpd_late_match_stop;
|
||||
extern int dpd_ignore_ports;
|
||||
|
||||
extern TableVal* likely_server_ports;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "PIA.h"
|
||||
#include "RuleMatcher.h"
|
||||
#include "Event.h"
|
||||
#include "analyzer/protocol/tcp/TCP_Flags.h"
|
||||
#include "analyzer/protocol/tcp/TCP_Reassembler.h"
|
||||
|
||||
|
|
@ -147,6 +148,20 @@ void PIA_UDP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule)
|
|||
DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded");
|
||||
// FIXME: This is where to check whether an analyzer
|
||||
// supports partial connections once we get such.
|
||||
|
||||
if ( protocol_late_match )
|
||||
{
|
||||
// Queue late match event
|
||||
EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal();
|
||||
Ref(tval);
|
||||
|
||||
mgr.QueueEventFast(protocol_late_match, {
|
||||
BuildConnVal(),
|
||||
tval,
|
||||
});
|
||||
}
|
||||
|
||||
pkt_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY;
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -282,6 +297,20 @@ void PIA_TCP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule)
|
|||
DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded");
|
||||
// FIXME: This is where to check whether an analyzer supports
|
||||
// partial connections once we get such.
|
||||
|
||||
if ( protocol_late_match )
|
||||
{
|
||||
// Queue late match event
|
||||
EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal();
|
||||
Ref(tval);
|
||||
|
||||
mgr.QueueEventFast(protocol_late_match, {
|
||||
BuildConnVal(),
|
||||
tval
|
||||
});
|
||||
}
|
||||
|
||||
stream_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY;
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -369,6 +369,20 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%);
|
|||
## there (and thus in ``conn.log``).
|
||||
event protocol_confirmation%(c: connection, atype: Analyzer::Tag, aid: count%);
|
||||
|
||||
## Generated if a DPD signature matched but the DPD buffer is already exhausted
|
||||
## and thus the analyzer could not be attached. While this does not confirm
|
||||
## that a protocol is actually used, it allows to retain that information.
|
||||
##
|
||||
## c: The connection.
|
||||
##
|
||||
## atype: The type of the analyzer confirming that its protocol is in
|
||||
## use. The value is one of the ``Analyzer::ANALYZER_*`` constants. For example,
|
||||
## ``Analyzer::ANALYZER_HTTP`` means the HTTP analyzer determined that it's indeed
|
||||
## parsing an HTTP connection.
|
||||
##
|
||||
## .. bro:see:: dpd_buffer_size
|
||||
event protocol_late_match%(c: connection, atype: Analyzer::Tag%);
|
||||
|
||||
## Generated when a protocol analyzer determines that a connection it is parsing
|
||||
## is not conforming to the protocol it expects. Zeek's dynamic protocol
|
||||
## detection heuristically activates analyzers as soon as it believes a
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path conn
|
||||
#open 2019-08-30-13-12-19
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service
|
||||
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string
|
||||
1567010592.624680 CHhAvVGS1DHFjwGM9 127.0.0.1 37526 127.0.0.1 80 tcp http 0.008395 61907 60478 SF - - 0 ShADadfF 10 62435 9 60954 - http
|
||||
1567010639.143657 ClEkJM2Vm5giqnMf4h 127.0.0.1 60644 127.0.0.1 5000 tcp - 0.015853 61917 60478 SF - - 0 ShADadfF 10 62445 9 60954 - http
|
||||
#close 2019-08-30-13-12-19
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path conn
|
||||
#open 2019-09-17-17-56-40
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service
|
||||
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string
|
||||
1300475167.096535 CHhAvVGS1DHFjwGM9 141.142.220.202 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 73 0 0 - -
|
||||
1300475167.097012 ClEkJM2Vm5giqnMf4h fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp dns - - - S0 - - 0 D 1 199 0 0 - -
|
||||
1300475167.099816 C4J4Th3PJpwUYZZ6gc 141.142.220.50 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 179 0 0 - -
|
||||
1300475168.853899 CmES5u32sYpV7JYN 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF - - 0 Dd 1 66 1 117 - -
|
||||
1300475168.854378 CP5puj4I8PtEU4qzYg 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF - - 0 Dd 1 80 1 127 - -
|
||||
1300475168.854837 C37jN32gN3y3AZzyf6 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF - - 0 Dd 1 66 1 211 - -
|
||||
1300475168.857956 C0LAHyvtKSQHyJxIl 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 38 89 SF - - 0 Dd 1 66 1 117 - -
|
||||
1300475168.858306 CFLRIC3zaTU1loLGxh 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 52 99 SF - - 0 Dd 1 80 1 127 - -
|
||||
1300475168.858713 C9rXSW3KSpTYvPrlI1 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 38 183 SF - - 0 Dd 1 66 1 211 - -
|
||||
1300475168.891644 C9mvWx3ezztgzcexV7 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 38 89 SF - - 0 Dd 1 66 1 117 - -
|
||||
1300475168.892037 CNnMIj2QSd84NKf7U3 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 52 99 SF - - 0 Dd 1 80 1 127 - -
|
||||
1300475168.892414 C7fIlMZDuRiqjpYbb 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 38 183 SF - - 0 Dd 1 66 1 211 - -
|
||||
1300475168.893988 CpmdRlaUoJLN3uIRa 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 38 89 SF - - 0 Dd 1 66 1 117 - -
|
||||
1300475168.894422 C1Xkzz2MaGtLrc1Tla 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 52 99 SF - - 0 Dd 1 80 1 127 - -
|
||||
1300475168.894787 CqlVyW1YwZ15RhTBc4 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 38 183 SF - - 0 Dd 1 66 1 211 - -
|
||||
1300475168.901749 CBA8792iHmnhPLksKa 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 36 131 SF - - 0 Dd 1 64 1 159 - -
|
||||
1300475168.902195 CGLPPc35OzDQij1XX8 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 36 198 SF - - 0 Dd 1 64 1 226 - -
|
||||
1300475169.899438 Cipfzj1BEnhejw8cGf 141.142.220.44 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 85 0 0 - -
|
||||
1300475170.862384 CV5WJ42jPYbNW9JNWf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 - - 0 D 7 546 0 0 - -
|
||||
1300475171.675372 CPhDKt12KQPUVbQz06 fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 - - 0 D 2 162 0 0 - -
|
||||
1300475171.677081 CAnFrb2Cvxr5T7quOc 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 - - 0 D 2 122 0 0 - -
|
||||
1300475173.116749 C8rquZ3DjgNW06JGLl fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 - - 0 D 2 162 0 0 - -
|
||||
1300475173.117362 CzrZOtXqhwwndQva3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 - - 0 D 2 122 0 0 - -
|
||||
1300475173.153679 CaGCc13FffXe6RkQl9 141.142.220.238 56641 141.142.220.255 137 udp dns - - - S0 - - 0 D 1 78 0 0 - -
|
||||
1300475168.652003 CtPZjS20MLrsMUOJi2 141.142.220.118 35634 208.80.152.2 80 tcp - 0.061329 463 350 OTH - - 0 DdA 2 567 1 402 - -
|
||||
1300475168.902635 CiyBAq1bBLNaTiTAc 141.142.220.118 35642 208.80.152.2 80 tcp http 0.120041 534 412 S1 - - 0 ShADad 4 750 3 576 - -
|
||||
1300475168.855305 C3eiCBGOLw3VtHfOj 141.142.220.118 49996 208.80.152.3 80 tcp http 0.218501 1171 733 S1 - - 0 ShADad 6 1491 4 949 - -
|
||||
1300475168.855330 CwjjYJ2WqgTbAqiHl6 141.142.220.118 49997 208.80.152.3 80 tcp http 0.219720 1125 734 S1 - - 0 ShADad 6 1445 4 950 - -
|
||||
1300475168.859163 Ck51lg1bScffFj34Ri 141.142.220.118 49998 208.80.152.3 80 tcp http 0.215893 1130 734 S1 - - 0 ShADad 6 1450 4 950 - -
|
||||
1300475168.892913 CykQaM33ztNt0csB9a 141.142.220.118 49999 208.80.152.3 80 tcp http 0.220961 1137 733 S1 - - 0 ShADad 6 1457 4 949 - -
|
||||
1300475168.892936 CtxTCR2Yer0FR1tIBg 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 - - 0 ShADad 6 1468 4 950 - -
|
||||
1300475168.895267 CLNN1k2QMum1aexUK7 141.142.220.118 50001 208.80.152.3 80 tcp http 0.227284 1178 734 S1 - - 0 ShADad 6 1498 4 950 - -
|
||||
1300475168.724007 CUM0KZ3MLUfNB0cl11 141.142.220.118 48649 208.80.152.118 80 tcp http 0.119905 525 232 S1 - - 0 ShADad 4 741 3 396 - -
|
||||
1300475169.780331 CFSwNi4CNGxcuffo49 141.142.220.235 6705 173.192.163.128 80 tcp - - - - OTH - - 0 ^h 0 0 1 48 - -
|
||||
#close 2019-09-17-17-56-40
|
||||
BIN
testing/btest/Traces/http/http-post-large.pcap
Normal file
BIN
testing/btest/Traces/http/http-post-large.pcap
Normal file
Binary file not shown.
|
|
@ -0,0 +1,11 @@
|
|||
# A basic test of the speculative service detection
|
||||
|
||||
# @TEST-EXEC: zeek -C -r $TRACES/http/http-post-large.pcap %INPUT
|
||||
# @TEST-EXEC: mv conn.log conn-post-large.log
|
||||
# @TEST-EXEC: btest-diff conn-post-large.log
|
||||
|
||||
# @TEST-EXEC: zeek -C -r $TRACES/wikipedia.trace %INPUT
|
||||
# @TEST-EXEC: mv conn.log conn-wiki.log
|
||||
# @TEST-EXEC: btest-diff conn-wiki.log
|
||||
|
||||
@load protocols/conn/speculative-service
|
||||
Loading…
Add table
Add a link
Reference in a new issue