From 0af57d12b24cb34801f4f70dcad50f0b05656dc4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Sep 2020 07:33:36 +0000 Subject: [PATCH] Change HTTP's DPD signatures so that each side can trigger the analyzer on its own. This is to avoid missing large sessions where a single side exceeds the DPD buffer size. It comes with the trade-off that now the analyzer can be triggered by anybody controlling one of the endpoints (instead of both). Test suite changes are minor, and nothing in "external". Closes #343. --- scripts/base/protocols/http/dpd.sig | 7 ++++++- .../http.log | 11 +++++++++-- .../output | 1 + .../conn-post-large.log | 6 +++--- .../btest/Traces/http/http_large_req_8001.pcap | Bin 0 -> 3229 bytes .../base/protocols/http/http-dpd-large-req.zeek | 13 +++++++++++++ 6 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.http.http-dpd-large-req/output create mode 100644 testing/btest/Traces/http/http_large_req_8001.pcap create mode 100644 testing/btest/scripts/base/protocols/http/http-dpd-large-req.zeek diff --git a/scripts/base/protocols/http/dpd.sig b/scripts/base/protocols/http/dpd.sig index d15436a679..8412f6c1f8 100644 --- a/scripts/base/protocols/http/dpd.sig +++ b/scripts/base/protocols/http/dpd.sig @@ -1,15 +1,20 @@ # List of HTTP headers pulled from: # http://annevankesteren.nl/2007/10/http-methods +# +# We match each side of the connection independently to avoid missing +# large HTTP sessions where one side exceeds the DPD buffer size on +# its own already. See https://github.com/zeek/zeek/issues/343. + signature dpd_http_client { ip-proto == tcp payload /^[[:space:]]*(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH|BCOPY|BDELETE|BMOVE|BPROPFIND|BPROPPATCH|NOTIFY|POLL|SUBSCRIBE|UNSUBSCRIBE|X-MS-ENUMATTS|RPC_OUT_DATA|RPC_IN_DATA)[[:space:]]*/ tcp-state originator + enable "http" } signature dpd_http_server { ip-proto == tcp payload /^HTTP\/[0-9]/ tcp-state responder - requires-reverse-signature dpd_http_client enable "http" } diff --git a/testing/btest/Baseline/scripts.base.frameworks.dpd.max_violations/http.log b/testing/btest/Baseline/scripts.base.frameworks.dpd.max_violations/http.log index 8cd4455b52..3486a0088b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.dpd.max_violations/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.dpd.max_violations/http.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path http -#open 2020-04-30-00-46-48 +#open 2020-09-07-08-39-48 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] 1354328870.191989 CHhAvVGS1DHFjwGM9 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - 1.1 - - 0 962 405 Method Not Allowed - - (empty) - - - - - - FNHaqE4UoY2x5hHRul - text/html @@ -16,8 +16,15 @@ 1354328882.928027 C37jN32gN3y3AZzyf6 128.2.6.136 46569 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - Fv4CUw2OVTa5d90Fh5 - text/html 1354328882.968948 C3eiCBGOLw3VtHfOj 128.2.6.136 46570 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FpKdCS1VswPP57cOE9 - text/html 1354328882.990373 CwjjYJ2WqgTbAqiHl6 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - 1.1 - - 0 43913 200 OK - - (empty) - - - - - - FKce9H2mSI6H6yHKzg - text/html +1354328887.114613 C0LAHyvtKSQHyJxIl 128.2.6.136 46572 173.194.75.103 80 1 - - - - 1.1 - - 0 961 405 Method Not Allowed - - (empty) - - - - - - FnwThTkiAjzMOp15d - text/html +1354328891.161077 CFLRIC3zaTU1loLGxh 128.2.6.136 46573 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FNmezC4DkmM3sleGfk - text/html +1354328891.204740 C9rXSW3KSpTYvPrlI1 128.2.6.136 46574 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FTBROX2JadJfHpHwyf - text/html +1354328891.245592 Ck51lg1bScffFj34Ri 128.2.6.136 46575 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FGHZXz1oh7AvmEq9i4 - text/html +1354328891.287655 C9mvWx3ezztgzcexV7 128.2.6.136 46576 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - Fgqofp246KRqF7D9sc - text/html 1354328891.309065 CNnMIj2QSd84NKf7U3 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - 1.1 - - 0 963 405 Method Not Allowed - - (empty) - - - - - - FsrHvh4vRpg5AYSB8 - text/html 1354328895.355012 C7fIlMZDuRiqjpYbb 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FTq0Uy1Ug7VB8q6CY7 - text/html +1354328895.416133 CykQaM33ztNt0csB9a 128.2.6.136 46579 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FukPcH2neOquJJLf8g - text/html +1354328895.459490 CtxTCR2Yer0FR1tIBg 128.2.6.136 46580 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FOo9cxBIsa3iJ5qN4 - text/html 1354328895.480865 CpmdRlaUoJLN3uIRa 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - 1.1 - - 0 963 405 Method Not Allowed - - (empty) - - - - - - FnYYzruLUTCbaQpR9 - text/html 1354328899.526682 C1Xkzz2MaGtLrc1Tla 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - 1.1 - - 0 925 400 Bad Request - - (empty) - - - - - - FG8LG51VfiVSWb3jJ4 - text/html 1354328903.572533 CqlVyW1YwZ15RhTBc4 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FmY2JP1uFMzpih2T5k - text/html @@ -48,4 +55,4 @@ 1354328932.692706 CudMuD3jKHCaCU5CE 128.2.6.136 46608 173.194.75.103 80 1 HEAD www.google.com /HTTP/1.1 - 1.0 - - 0 0 400 Bad Request - - (empty) - - - - - - - - - 1354328932.754657 CRJ9x54IaE7bkVEpad 128.2.6.136 46609 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - FXonC02oI6E6ZT4pi4 - text/html 1354328932.796568 CAvUKGaEgLlR4i6t2 128.2.6.136 46610 173.194.75.103 80 1 - - - - 1.0 - - 0 925 400 Bad Request - - (empty) - - - - - - F8h50j2nZJ3Oloni53 - text/html -#close 2020-04-30-00-46-49 +#close 2020-09-07-08-39-48 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-dpd-large-req/output b/testing/btest/Baseline/scripts.base.protocols.http.http-dpd-large-req/output new file mode 100644 index 0000000000..23e546483f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-dpd-large-req/output @@ -0,0 +1 @@ +http_request, 1.1, GET, / diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log index 0519bf5419..020d008924 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path conn -#open 2019-08-30-13-12-19 +#open 2020-09-07-08-40-00 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string 1567010592.624680 CHhAvVGS1DHFjwGM9 127.0.0.1 37526 127.0.0.1 80 tcp http 0.008395 61907 60478 SF - - 0 ShADadfF 10 62435 9 60954 - http -1567010639.143657 ClEkJM2Vm5giqnMf4h 127.0.0.1 60644 127.0.0.1 5000 tcp - 0.015853 61917 60478 SF - - 0 ShADadfF 10 62445 9 60954 - http -#close 2019-08-30-13-12-19 +1567010639.143657 ClEkJM2Vm5giqnMf4h 127.0.0.1 60644 127.0.0.1 5000 tcp http 0.015853 61917 60478 SF - - 0 ShADadfF 10 62445 9 60954 - http +#close 2020-09-07-08-40-00 diff --git a/testing/btest/Traces/http/http_large_req_8001.pcap b/testing/btest/Traces/http/http_large_req_8001.pcap new file mode 100644 index 0000000000000000000000000000000000000000..5af299e85a2f1b5f5fe3d8024697e51e50ae6a8b GIT binary patch literal 3229 zcmbW3UuauZ9LI0cw1!(UJO4m8w&g5Hh1T42&%HOvO_p}enzHG#Ep8cgFu|CdHo-KB zHz%!q3rvKzJ`DnbAP&UUmpQc?;>(6U)PLxU_%x7iFozSFOnvYr{?5JGpC-0&7V=AS zbI$#o-}iUV`Tc%Bef8-%x8#xTy;?0va>2zfyRV%7=Pi$P6n=9%dwxA_zH(>qopXsn zuQVx19w8y#7O^{>o8fm=x((x-jO7?r+3xW)6?g2d1W#l{g97& z+zx&Hyhl0?*W4VgvxZHNH`h4bzk`$c3jV~}&e+nQhhc>q??2HPy1k#1yz;((|G2=( z?Ui~TUpXDBJacOo_kaJ2M>+s>Ze9KxZU1j|_UT! z)+^c8N$kDm5I-K@jCkhfjl_eK8HCZHOeT$qLVWU}N{z)3iE0W7n~J7rBxb6r>XT2_ zs1;1i&@v3?>Q)Iyl#rpoKq8l;Rrqrl5BubT8Y|Fjo?2)wTdGsU3R#Bo6G-^U~6HKi0G;zz}5{Bm3mY}rJ;+cG)xhd zAx%VOC@i8fOhi0exK3^azG znsTB8RrV98<#wfYUEY%TjZO4-6PKnq@yX?G;)_>35|n7$x_tbh`7I~zdGmjXgN>mp zf8Qsrav$4OTSc0xqGN}B@(F6qQ7eW{6lbd?Y7eVQ7^R`&R?1jc3`O(F$tke6cHl^?@$OGJd*Lr6O}T9QXgasRhZ1^8w?lNY-u0L6|z>1ve7!54w?_< zrD%DE6=Fz>MtrjKJWheu*I-+6>(uIcx%nN}y8EvVtv5&dA$mI+L(rJnoew+@TWfr8 z^4LV?RC*G6l_M#XK6xPZ`~(UFF`m*VFivKY&H%I|1mnr00Td`OR*hjiKR>U`>q^C% z!I|T>3Io@eRm?FZ&+>sWInI^wFKArgYnHMs;`avW>-FN?Xy9HfG7yUZ$~lo44Qv(^ zPGm(^qGQRTMRTlTEud1d#-Qgx(-jNhICh5RIBqZbY9+tG9Uzoutput +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Tests our DPD signatures with a session where one side exceeds the DPD buffer size. + +@load base/protocols/conn +@load base/protocols/http +@load base/frameworks/dpd + +event http_request(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string) + { + print "http_request", version, method, original_URI; + }