diff --git a/CHANGES b/CHANGES index 714c2de12f..da889a0ef4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,17 @@ +3.1.0-dev.124 | 2019-09-17 11:16:47 -0700 + + * Add speculative service script. (Jan Grashoefer) + + The speculative service script handles dpd_late_match events to extend + conn.log with infos about potential protocol identifications. + + * Allow to handle late DPD matches. (Jan Grashoefer) + + If "dpd_match_only_beginning" is disabled, matches of protocol signatures + can be handeld using protocol_late_match. To prevent further matching in + this case, the new option "dpd_late_match_stop" may be activated. + 3.1.0-dev.118 | 2019-09-17 17:21:58 +0000 * GH-566: Fix cases where ssh_encrypted_packet event wasn't raised. diff --git a/NEWS b/NEWS index b94763dde6..ddc682850e 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,16 @@ Zeek 3.1.0 New Functionality ----------------- +- Add a new option, ``dpd_late_match_stop``, which can be used in conjuction + with the option ``dpd_match_only_beginning`` and the new event + ``protocol_late_match`` to help annotate the conn.log with a field + to speculate on the protocol/service in cases where the DPD buffer + was already exhausted and can't analyze the full connection anymore, + but where there was still a late signature match. A new script, + ``policy/protocols/conn/speculative-service.zeek``, was added as an + example of how to perform this tuning and add a "speculative_service" + field to conn.log, but it's not loaded by default. + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 00a1250b53..058db7a26b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.118 +3.1.0-dev.124 diff --git a/doc b/doc index a3dd09cd5a..218907b650 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit a3dd09cd5acfe6aa848a84eae5fdb18d64d590e1 +Subproject commit 218907b6500d627904f6ba0767f7ba362bf825f7 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 32ff925f13..82993bfb03 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4684,6 +4684,18 @@ const dpd_buffer_size = 1024 &redef; ## only signatures used for dynamic protocol detection. const dpd_match_only_beginning = T &redef; +## If true, stops signature matching after a late match. A late match may occur +## in case the DPD buffer is exhausted but a protocol signature matched. To +## allow late matching, :zeek:see:`dpd_match_only_beginning` must be disabled. +## +## .. zeek:see:: dpd_reassemble_first_packets dpd_buffer_size +## dpd_match_only_beginning +## +## .. note:: Despite the name, this option stops *all* signature matching, not +## only signatures used for dynamic protocol detection but is triggered by +## DPD signatures only. +const dpd_late_match_stop = F &redef; + ## If true, don't consider any ports for deciding which protocol analyzer to ## use. ## diff --git a/scripts/policy/protocols/conn/speculative-service.zeek b/scripts/policy/protocols/conn/speculative-service.zeek new file mode 100644 index 0000000000..ba1162d809 --- /dev/null +++ b/scripts/policy/protocols/conn/speculative-service.zeek @@ -0,0 +1,36 @@ +##! This script adds information about matched DPD signatures to the connection +##! log. + +@load base/protocols/conn + +module Conn; + +redef record Info += { + ## Protocol that was determined by a matching signature after the beginning + ## of a connection. In this situation no analyzer can be attached and hence + ## the data cannot be analyzed nor the protocol can be confirmed. + speculative_service: string &log &optional; +}; + +redef record connection += { + speculative_service: set[string] &default=string_set(); +}; + +redef dpd_match_only_beginning = F; +redef dpd_late_match_stop = T; + +event protocol_late_match(c: connection, atype: Analyzer::Tag) + { + local analyzer = Analyzer::name(atype); + add c$speculative_service[analyzer]; + } + +event connection_state_remove(c: connection) + { + local sp_service = ""; + for ( s in c$speculative_service ) + sp_service = sp_service == "" ? s : cat(sp_service, ",", s); + + if ( sp_service != "" ) + c$conn$speculative_service = to_lower(sp_service); + } diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 365dafcf71..8a2721dc14 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -67,6 +67,7 @@ @load protocols/conn/mac-logging.zeek @load protocols/conn/vlan-logging.zeek @load protocols/conn/weirds.zeek +#@load protocols/conn/speculative-service.zeek @load protocols/dhcp/msg-orig.zeek @load protocols/dhcp/software.zeek @load protocols/dhcp/sub-opts.zeek diff --git a/scripts/zeekygen/__load__.zeek b/scripts/zeekygen/__load__.zeek index 00555c57bd..64494037b8 100644 --- a/scripts/zeekygen/__load__.zeek +++ b/scripts/zeekygen/__load__.zeek @@ -6,6 +6,7 @@ @load frameworks/control/controller.zeek @load frameworks/files/extract-all-files.zeek @load policy/misc/dump-events.zeek +@load policy/protocols/conn/speculative-service.zeek @load ./example.zeek diff --git a/src/NetVar.cc b/src/NetVar.cc index 981ca005ff..1ab99170bb 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -164,6 +164,7 @@ RecordType* irc_join_info; int dpd_reassemble_first_packets; int dpd_buffer_size; int dpd_match_only_beginning; +int dpd_late_match_stop; int dpd_ignore_ports; TableVal* likely_server_ports; @@ -406,6 +407,7 @@ void init_net_var() opt_internal_int("dpd_reassemble_first_packets"); dpd_buffer_size = opt_internal_int("dpd_buffer_size"); dpd_match_only_beginning = opt_internal_int("dpd_match_only_beginning"); + dpd_late_match_stop = opt_internal_int("dpd_late_match_stop"); dpd_ignore_ports = opt_internal_int("dpd_ignore_ports"); likely_server_ports = internal_val("likely_server_ports")->AsTableVal(); diff --git a/src/NetVar.h b/src/NetVar.h index 632ea9ff42..95eb99d844 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -165,6 +165,7 @@ extern RecordType* irc_join_info; extern int dpd_reassemble_first_packets; extern int dpd_buffer_size; extern int dpd_match_only_beginning; +extern int dpd_late_match_stop; extern int dpd_ignore_ports; extern TableVal* likely_server_ports; diff --git a/src/analyzer/protocol/pia/PIA.cc b/src/analyzer/protocol/pia/PIA.cc index 7d5e5725d3..0d8f382dd9 100644 --- a/src/analyzer/protocol/pia/PIA.cc +++ b/src/analyzer/protocol/pia/PIA.cc @@ -1,5 +1,6 @@ #include "PIA.h" #include "RuleMatcher.h" +#include "Event.h" #include "analyzer/protocol/tcp/TCP_Flags.h" #include "analyzer/protocol/tcp/TCP_Reassembler.h" @@ -147,6 +148,20 @@ void PIA_UDP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule) DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded"); // FIXME: This is where to check whether an analyzer // supports partial connections once we get such. + + if ( protocol_late_match ) + { + // Queue late match event + EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal(); + Ref(tval); + + mgr.QueueEventFast(protocol_late_match, { + BuildConnVal(), + tval, + }); + } + + pkt_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY; return; } @@ -282,6 +297,20 @@ void PIA_TCP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule) DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded"); // FIXME: This is where to check whether an analyzer supports // partial connections once we get such. + + if ( protocol_late_match ) + { + // Queue late match event + EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal(); + Ref(tval); + + mgr.QueueEventFast(protocol_late_match, { + BuildConnVal(), + tval + }); + } + + stream_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY; return; } diff --git a/src/event.bif b/src/event.bif index 53814254c1..3b70b7d64c 100644 --- a/src/event.bif +++ b/src/event.bif @@ -369,6 +369,20 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); ## there (and thus in ``conn.log``). event protocol_confirmation%(c: connection, atype: Analyzer::Tag, aid: count%); +## Generated if a DPD signature matched but the DPD buffer is already exhausted +## and thus the analyzer could not be attached. While this does not confirm +## that a protocol is actually used, it allows to retain that information. +## +## c: The connection. +## +## atype: The type of the analyzer confirming that its protocol is in +## use. The value is one of the ``Analyzer::ANALYZER_*`` constants. For example, +## ``Analyzer::ANALYZER_HTTP`` means the HTTP analyzer determined that it's indeed +## parsing an HTTP connection. +## +## .. bro:see:: dpd_buffer_size +event protocol_late_match%(c: connection, atype: Analyzer::Tag%); + ## Generated when a protocol analyzer determines that a connection it is parsing ## is not conforming to the protocol it expects. Zeek's dynamic protocol ## detection heuristically activates analyzers as soon as it believes a diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log new file mode 100644 index 0000000000..0519bf5419 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2019-08-30-13-12-19 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string +1567010592.624680 CHhAvVGS1DHFjwGM9 127.0.0.1 37526 127.0.0.1 80 tcp http 0.008395 61907 60478 SF - - 0 ShADadfF 10 62435 9 60954 - http +1567010639.143657 ClEkJM2Vm5giqnMf4h 127.0.0.1 60644 127.0.0.1 5000 tcp - 0.015853 61917 60478 SF - - 0 ShADadfF 10 62445 9 60954 - http +#close 2019-08-30-13-12-19 diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log new file mode 100644 index 0000000000..4732b0b898 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log @@ -0,0 +1,43 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2019-09-17-17-56-40 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string +1300475167.096535 CHhAvVGS1DHFjwGM9 141.142.220.202 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 73 0 0 - - +1300475167.097012 ClEkJM2Vm5giqnMf4h fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp dns - - - S0 - - 0 D 1 199 0 0 - - +1300475167.099816 C4J4Th3PJpwUYZZ6gc 141.142.220.50 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 179 0 0 - - +1300475168.853899 CmES5u32sYpV7JYN 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.854378 CP5puj4I8PtEU4qzYg 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.854837 C37jN32gN3y3AZzyf6 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.857956 C0LAHyvtKSQHyJxIl 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.858306 CFLRIC3zaTU1loLGxh 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.858713 C9rXSW3KSpTYvPrlI1 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.891644 C9mvWx3ezztgzcexV7 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.892037 CNnMIj2QSd84NKf7U3 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.892414 C7fIlMZDuRiqjpYbb 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.893988 CpmdRlaUoJLN3uIRa 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.894422 C1Xkzz2MaGtLrc1Tla 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.894787 CqlVyW1YwZ15RhTBc4 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.901749 CBA8792iHmnhPLksKa 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 36 131 SF - - 0 Dd 1 64 1 159 - - +1300475168.902195 CGLPPc35OzDQij1XX8 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 36 198 SF - - 0 Dd 1 64 1 226 - - +1300475169.899438 Cipfzj1BEnhejw8cGf 141.142.220.44 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 85 0 0 - - +1300475170.862384 CV5WJ42jPYbNW9JNWf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 - - 0 D 7 546 0 0 - - +1300475171.675372 CPhDKt12KQPUVbQz06 fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 - - 0 D 2 162 0 0 - - +1300475171.677081 CAnFrb2Cvxr5T7quOc 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 - - 0 D 2 122 0 0 - - +1300475173.116749 C8rquZ3DjgNW06JGLl fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 - - 0 D 2 162 0 0 - - +1300475173.117362 CzrZOtXqhwwndQva3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 - - 0 D 2 122 0 0 - - +1300475173.153679 CaGCc13FffXe6RkQl9 141.142.220.238 56641 141.142.220.255 137 udp dns - - - S0 - - 0 D 1 78 0 0 - - +1300475168.652003 CtPZjS20MLrsMUOJi2 141.142.220.118 35634 208.80.152.2 80 tcp - 0.061329 463 350 OTH - - 0 DdA 2 567 1 402 - - +1300475168.902635 CiyBAq1bBLNaTiTAc 141.142.220.118 35642 208.80.152.2 80 tcp http 0.120041 534 412 S1 - - 0 ShADad 4 750 3 576 - - +1300475168.855305 C3eiCBGOLw3VtHfOj 141.142.220.118 49996 208.80.152.3 80 tcp http 0.218501 1171 733 S1 - - 0 ShADad 6 1491 4 949 - - +1300475168.855330 CwjjYJ2WqgTbAqiHl6 141.142.220.118 49997 208.80.152.3 80 tcp http 0.219720 1125 734 S1 - - 0 ShADad 6 1445 4 950 - - +1300475168.859163 Ck51lg1bScffFj34Ri 141.142.220.118 49998 208.80.152.3 80 tcp http 0.215893 1130 734 S1 - - 0 ShADad 6 1450 4 950 - - +1300475168.892913 CykQaM33ztNt0csB9a 141.142.220.118 49999 208.80.152.3 80 tcp http 0.220961 1137 733 S1 - - 0 ShADad 6 1457 4 949 - - +1300475168.892936 CtxTCR2Yer0FR1tIBg 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 - - 0 ShADad 6 1468 4 950 - - +1300475168.895267 CLNN1k2QMum1aexUK7 141.142.220.118 50001 208.80.152.3 80 tcp http 0.227284 1178 734 S1 - - 0 ShADad 6 1498 4 950 - - +1300475168.724007 CUM0KZ3MLUfNB0cl11 141.142.220.118 48649 208.80.152.118 80 tcp http 0.119905 525 232 S1 - - 0 ShADad 4 741 3 396 - - +1300475169.780331 CFSwNi4CNGxcuffo49 141.142.220.235 6705 173.192.163.128 80 tcp - - - - OTH - - 0 ^h 0 0 1 48 - - +#close 2019-09-17-17-56-40 diff --git a/testing/btest/Traces/http/http-post-large.pcap b/testing/btest/Traces/http/http-post-large.pcap new file mode 100644 index 0000000000..8fd0e8df2d Binary files /dev/null and b/testing/btest/Traces/http/http-post-large.pcap differ diff --git a/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek new file mode 100644 index 0000000000..b95f0f337c --- /dev/null +++ b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek @@ -0,0 +1,11 @@ +# A basic test of the speculative service detection + +# @TEST-EXEC: zeek -C -r $TRACES/http/http-post-large.pcap %INPUT +# @TEST-EXEC: mv conn.log conn-post-large.log +# @TEST-EXEC: btest-diff conn-post-large.log + +# @TEST-EXEC: zeek -C -r $TRACES/wikipedia.trace %INPUT +# @TEST-EXEC: mv conn.log conn-wiki.log +# @TEST-EXEC: btest-diff conn-wiki.log + +@load protocols/conn/speculative-service