From 5901b6d33caf717c8208d621c68a0fa2bb123502 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Wed, 21 Aug 2019 18:38:09 +0200 Subject: [PATCH 1/5] Allow to handle late DPD matches. If dpd_match_only_beginning is disabled, matches of protocol signatures can be handeld using protocol_late_match. To prevent further matching in this case, dpd_late_match_stop may be activated. --- scripts/base/init-bare.zeek | 12 ++++++++++++ src/NetVar.cc | 2 ++ src/NetVar.h | 1 + src/analyzer/protocol/pia/PIA.cc | 29 +++++++++++++++++++++++++++++ src/event.bif | 14 ++++++++++++++ 5 files changed, 58 insertions(+) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 32ff925f13..82993bfb03 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4684,6 +4684,18 @@ const dpd_buffer_size = 1024 &redef; ## only signatures used for dynamic protocol detection. const dpd_match_only_beginning = T &redef; +## If true, stops signature matching after a late match. A late match may occur +## in case the DPD buffer is exhausted but a protocol signature matched. To +## allow late matching, :zeek:see:`dpd_match_only_beginning` must be disabled. +## +## .. zeek:see:: dpd_reassemble_first_packets dpd_buffer_size +## dpd_match_only_beginning +## +## .. note:: Despite the name, this option stops *all* signature matching, not +## only signatures used for dynamic protocol detection but is triggered by +## DPD signatures only. +const dpd_late_match_stop = F &redef; + ## If true, don't consider any ports for deciding which protocol analyzer to ## use. ## diff --git a/src/NetVar.cc b/src/NetVar.cc index 981ca005ff..1ab99170bb 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -164,6 +164,7 @@ RecordType* irc_join_info; int dpd_reassemble_first_packets; int dpd_buffer_size; int dpd_match_only_beginning; +int dpd_late_match_stop; int dpd_ignore_ports; TableVal* likely_server_ports; @@ -406,6 +407,7 @@ void init_net_var() opt_internal_int("dpd_reassemble_first_packets"); dpd_buffer_size = opt_internal_int("dpd_buffer_size"); dpd_match_only_beginning = opt_internal_int("dpd_match_only_beginning"); + dpd_late_match_stop = opt_internal_int("dpd_late_match_stop"); dpd_ignore_ports = opt_internal_int("dpd_ignore_ports"); likely_server_ports = internal_val("likely_server_ports")->AsTableVal(); diff --git a/src/NetVar.h b/src/NetVar.h index 41b2028064..7c5f218cb6 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -166,6 +166,7 @@ extern RecordType* irc_join_info; extern int dpd_reassemble_first_packets; extern int dpd_buffer_size; extern int dpd_match_only_beginning; +extern int dpd_late_match_stop; extern int dpd_ignore_ports; extern TableVal* likely_server_ports; diff --git a/src/analyzer/protocol/pia/PIA.cc b/src/analyzer/protocol/pia/PIA.cc index bf9f27be7c..f035f942be 100644 --- a/src/analyzer/protocol/pia/PIA.cc +++ b/src/analyzer/protocol/pia/PIA.cc @@ -1,5 +1,6 @@ #include "PIA.h" #include "RuleMatcher.h" +#include "Event.h" #include "analyzer/protocol/tcp/TCP_Flags.h" #include "analyzer/protocol/tcp/TCP_Reassembler.h" @@ -144,6 +145,20 @@ void PIA_UDP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule) DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded"); // FIXME: This is where to check whether an analyzer // supports partial connections once we get such. + + if ( protocol_late_match ) + { + // Queue late match event + EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal(); + Ref(tval); + + val_list *vl = new val_list; + vl->append(BuildConnVal()); + vl->append(tval); + mgr.QueueEvent(protocol_late_match, vl); + } + + pkt_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY; return; } @@ -279,6 +294,20 @@ void PIA_TCP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule) DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded"); // FIXME: This is where to check whether an analyzer supports // partial connections once we get such. + + if ( protocol_late_match ) + { + // Queue late match event + EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal(); + Ref(tval); + + val_list *vl = new val_list; + vl->append(BuildConnVal()); + vl->append(tval); + mgr.QueueEvent(protocol_late_match, vl); + } + + stream_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY; return; } diff --git a/src/event.bif b/src/event.bif index 5222545ae5..62ab89f1c1 100644 --- a/src/event.bif +++ b/src/event.bif @@ -369,6 +369,20 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); ## there (and thus in ``conn.log``). event protocol_confirmation%(c: connection, atype: Analyzer::Tag, aid: count%); +## Generated if a DPD signature matched but the DPD buffer is already exhausted +## and thus the analyzer could not be attached. While this does not confirm +## that a protocol is actually used, it allows to retain that information. +## +## c: The connection. +## +## atype: The type of the analyzer confirming that its protocol is in +## use. The value is one of the ``Analyzer::ANALYZER_*`` constants. For example, +## ``Analyzer::ANALYZER_HTTP`` means the HTTP analyzer determined that it's indeed +## parsing an HTTP connection. +## +## .. bro:see:: dpd_buffer_size +event protocol_late_match%(c: connection, atype: Analyzer::Tag%); + ## Generated when a protocol analyzer determines that a connection it is parsing ## is not conforming to the protocol it expects. Zeek's dynamic protocol ## detection heuristically activates analyzers as soon as it believes a From 788b56a6528a06d5d63e860bb91c530bd07eb89d Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Thu, 29 Aug 2019 11:47:04 +0200 Subject: [PATCH 2/5] Add speculative service script. The speculative service script handles dpd_late_match events to extend conn.log with infos about potential protocol identifications. --- .../protocols/conn/speculative-service.zeek | 34 ++++++++++++++++++ .../conn.log | 11 ++++++ .../btest/Traces/http/http-post-large.pcap | Bin 0 -> 247952 bytes .../protocols/conn/speculative-service.zeek | 6 ++++ 4 files changed, 51 insertions(+) create mode 100644 scripts/policy/protocols/conn/speculative-service.zeek create mode 100644 testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log create mode 100644 testing/btest/Traces/http/http-post-large.pcap create mode 100644 testing/btest/scripts/policy/protocols/conn/speculative-service.zeek diff --git a/scripts/policy/protocols/conn/speculative-service.zeek b/scripts/policy/protocols/conn/speculative-service.zeek new file mode 100644 index 0000000000..2fb9b1cdaa --- /dev/null +++ b/scripts/policy/protocols/conn/speculative-service.zeek @@ -0,0 +1,34 @@ +##! This script adds information about matched DPD signatures to the connection +##! log. + +@load base/protocols/conn + +module Conn; + +redef record Info += { + ## Protocol that was determined by a matching signature after the beginning + ## of a connection. In this situation no analyzer can be attached and hence + ## the data cannot be analyzed nor the protocol can be confirmed. + speculative_service: string &log &optional; +}; + +redef record connection += { + speculative_service: set[string] &default=string_set(); +}; + +redef dpd_match_only_beginning = F; +redef dpd_late_match_stop = T; + +event protocol_late_match(c: connection, atype: Analyzer::Tag) + { + local analyzer = Analyzer::name(atype); + add c$speculative_service[analyzer]; + } + +event connection_state_remove(c: connection) + { + local sp_service = ""; + for ( s in c$speculative_service ) + sp_service = sp_service == "" ? s : cat(sp_service, ",", s); + c$conn$speculative_service = to_lower(sp_service); + } diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log new file mode 100644 index 0000000000..ba0ed8ed25 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2019-08-29-09-45-13 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string +1567010592.624680 CHhAvVGS1DHFjwGM9 127.0.0.1 37526 127.0.0.1 80 tcp http 0.008395 61907 60478 SF - - 0 ShADadfF 10 62435 9 60954 - http +1567010639.143657 ClEkJM2Vm5giqnMf4h 127.0.0.1 60644 127.0.0.1 5000 tcp - 0.015853 61917 60478 SF - - 0 ShADadfF 10 62445 9 60954 - http +#close 2019-08-29-09-45-13 diff --git a/testing/btest/Traces/http/http-post-large.pcap b/testing/btest/Traces/http/http-post-large.pcap new file mode 100644 index 0000000000000000000000000000000000000000..8fd0e8df2db616f71bbdf6542c89726f9df3c0b7 GIT binary patch literal 247952 zcmeHwTZ~=TdETCpEK>taAct*X*AHEh;Y9M9IcJ6%Qq+)Qco8p&DUlUNwcSE?_nf`v z%o6vF`!bwiYHI)oHDjYlEEod%6bFc4#c?a42;$q<8YM@BGp0o;M=L8*h4-Up$-tWBKHFM{mD(Ys&LR zAHDtFqoW7L&cA!^*W~q)M;`gONIto)WX}`H$N#XF>|K82&CiVd?tg^zPk;C4Z~u6$ zWRYG=z68mqaV9zXAFjIt9-#pxyQcrzxdit0;XM!a2kEx3`E|=Bv(Ia0uGILC2OS4Cg zHm06d=S}Ns(mXUZJ%#_%>YUBYLbBYHH^;|LwOS^H)WR$m&BY=LC;T)GZObog5;vFS z6}`On)hIl6?fI$3a}%Fye$G8RS__)I=*R8SZ<|?V;>lO8Oqkex;BfZ?XX929*tk8b z+Ba-Ep#rlI`h^)EJCno(E^u;gCB?S~S}}S3`t`|$B#S1?EHom(1idd@G;zCFoK;7r zXO7Oemt)hC=Fh6D#-x*eXqO=WWwT&1lg+BdqDW_(&2INL(Z6Xs^8+*aD{*dGWo9N{ zHrGm%7db{@d~AFy_g9|3V1Kbl@KfuHNoFE#)4Ysykc3I5b6aRXGR0>PozblXLJCuq zc-ar|sTQtQo6r`Syq%js2iA{toWy0cqK(a>B+!M4QV6o~k`2nZ&}E@Re;y~&rl?M0 zbmX^VU*mGFl`9ZnGT)uBD{(0a%d{x{hCWs3C6guEB)NRwZ?(!?zESAc%DhN)P~wo5 z8~48Pb!O(vsG+Z*;kt#k`ueJ$2fAI(o2;FgSehOuXuYos|Ei6Ad?NSbR#8GiUaZq6 zjdk8*f89BU*k7BSUmA}WN}FrH#4^In54FM4w3)?H?yOK7>qQd>*Z{Cp@yAjbre%SR zM?Q|1OrB$bB_ZarHPsrvR#xjvO5r$&$D*lO}3D+6toP1ds5 za%4Z((%Dff$x<{rFVW%@7ak_qCJXtF%|qEY-3Dn>d{<+)Y9$exob-mY!MU<_uE?;w zgwYBucKiBP)WZDBW#oK}#yt8Lv(PSJ4e2Cxqk$Wp%*WU+VvgDfgA!Ob4=|!_Z3B!X zZfuc%1H+q!eoO9GvQ%)2G;x#P&$0at=~UhQX)2>K9NSv8aH&*?I@Y(pJv>SnhQ8Jz zyXvP?Vyhoclk6&#qFi>gTBm<@)KqK@9hFxU){ef57xBNV^;X(xNAc~`MX`_LV*|>l zQ%|eGfL_{BUu%t2)KqttT9v?U!70Q&6?qRSt<~}aJ|eE@!&)16sT}G9wu(L-%k*eh|#lGf&U_ z14MBU%-?BuBA6$bhd-C$krFLM&wQUtc*ny@3FZmr86myp5#eTOtW6^Fy-(auFi$W~Fi$W~Fi$W~Fi$W~ zFwYV>4?{^Idgkewr)U15w&yX-o~7vw3R02%pX~qJ$^Jh)E-bRylg)k)mN8_QJ;Ur7 zX5X!s)~%GroO$NV-&s4Do_Tua>6srk&%AnP;nknqeB3+jdG5fk?&E-W*7Ii4D<{2^ zkNs8ry626^;az^?&CiVd?hju)`%`cAiIK0~dI~R&92h(Q?zvx+LnQtWNPHD?^)d>c=D;cYYjgcYFi$W~Fi$W~Fi$W~Fi$W~Fi$W~Fi$W~Fi$YA zyT1NiDlkVdPcTn#I&9#S%nF-hRv=S=Ep%Llq2Dcty&-yz7GmE@lWE~IvAsaSJL3em ztYo>2)b5upf1$Lw_RGRVsO1`JQ%2fmw#-b$>!WR~7fl>wCPTmCkEJq9%fc^=d>k*C zJU2Qbm?xMgm?xMgm?xMgm?xMgm@nf(?;&y(dgkewr}1Ohj2{H^1oMFBvCTudnYfZs zHq~ZXW4CH0QIrG;e#g1FR{EeJC754h_>l$l5}5BoI}h6qj2tN_t6Q*f=|Z{sBU;hZ`{b>^$g$>jp2OvOYwvT5+vTypAh-IV9K^n&pbWz z4=`8gnP)*h7UW|=K4e{GI=@R5X?o`AnWtyI*196U{qP0nbQuBYncuFe6~hp&K;nhX z5ijiL{&E#5|INBJF6Ug0%cw3A*{yMTF8Ev2xcp{cjmxjrH7@`4_a67Ipx&e${SQ~; z@@)EwllU+0Dp6`&KG!Wf`E`+aVQ%hnbGk954oyv|OP?PflScnGPI2$ToNqtk%4q&c zeF|4b6AeaX%O9;ifm8GfbIK{cE)xGAB>n`gm81XR8vUo~KNE>R{$I7kPB;AMMd=1< z^dAf~`s-(sxG-@sIk%FUSyh1WQ`1LgXP%jzKBUgSI5$34_1DPT>-N{9uD|}czPet2y)}nHzJ0g;`m{(~MQKzV z{SWD{)tU4QBJtvDwZy8w-kNj$g%SAL(}VqW{1;xhbY|{Lm(MEcrW0cdkK_GMjUQC6 z;`PWBezzl^bUpcG_szlQ%eVbjtIXx|g>sG-6_iOwq&$l24-?IR@r=s6dXPDsqg@O0==+l4D>+sV+H&d>lP4SurF4DTGeKRy$F}qX-?l()Y*xY>_0_{d?y(q^-7_x zqn{Ma)e%mtkfVbqOZd}|CZsQNY$m4PzLCTm%2Et2J*5KuFAMlRmRo*?*cnLMR$98 z$`TodZLFFZR2X2!c~}%H40f#%c2!BIkXApEP?XMUo5$S;s$xe&b=ss;IdL24D6gW- zb~IPKD0jYi$p#4J=xCZ96}C^W#BPm`^(mH4(X4JK&@nrTXsr#1%IVHgtDV^Tood@t zcK3(^S}iodIDk4L0|PCA2#N^G0-$dYX;uq}lZKOK6O)GKi7hhY42!->*j+2Cgx!0< zG$YG~u)Aj&BJ8fLhlJe%GW(2(``@2I*bO`Gj{C<5y9b?G}dP2)hZp;dQ+;c89Q=-fl$1 zWql}POTtnKy9v7)FZ=*;)r8%I-3-Rxt)vCQZo+QDZo=*d%!458ChR8cW{~-g#e5TX z6Lu4J6Lz~C9_Ds42flmn-4k2^gx!SQgx!SQgxy<0QBk3R9gG!Zw$&D8TYcm#M+TWQ z$egTC<^B(A*Z~7-K12jNQju*&D?IV<&jctGmL-3Vm&16W^Ok&a8jym zH}^5U-Sl?T+f8pbz1?JdmpVLqjKFnR^i@)-)l-MCo3NX(dy_=_!?KMLb`y3JcJDyR zA;NCLZo+QDZo+QDZo+PBdR3=bK#T>%P^WEQ7MT;6M<#PJnXk!E$qBm&y9v7qyIEQo zt6*0cx|wyev80N&DT=2J_8U5_s>JuYhz4a7c962y{{@rt5mqR3; zgT%kN>bavU4o^hl)xS=^C=&nWk7|kD{IZE=)0 zr)j{%0c(p(Pwy~$cuVQ$zYRwZh-N3A4FQ9nH|^OjAK%^D{2^vG|~O^C|h zHdl{j{yFo{nSajw^LWVyWn3t*`Gh|1Uq%fz4<=0m%s*%Txr9Hn$XS*s@CG{aM8DN4 zBb5G=Zn0%vBq|APg3P{lZ1WtE!q-Zx=0iUYaMw>W-{b~+URb8k=gUalxRKdV1tv6` zru4e6qT0fAEddk!03ZQKA#N*!LN0{eagdn|!(n5!REB9u*lp|rVYe(*!tPz#;P1Pe z$z}$VhiqoOy==>iQuS?)`^-^C*iG0?*iG0?*qsEbFj1OhPyqLq4T7F#8u=A#ivwyC zb|+z3q<&GEC3m}u)C+=~{1V%$M7Apf%ZANt37c+}2M46R2!Ki0Eu-ecwL6dgg3*Vt zJDMQuZk73N*;NU<>FtJ5rRs#-_%0-?@UPlPyghKuElGi#o+-rSg}V2gX{__!M3n=# zGiIFXu6QQdv8rnO*KRZzqqm}r%_1VG{ceY(x0~K>db@W^E8C(#cm|o%+fCRFTSQLS zjVKTm*%$#SO9(ab3A@FDNZ8%g?t2h#UlfC*x0~K>db{cE-eV44!fwKD!fwKD7CB>) zGnWxnl5L%A>&)$D*w>DRecju`dmk?@VK-qnVK-qnVK+l0_Iro~gYmaWE2Fo24Po*I z!swXW%^-8Kt^2T(b@L#0I|pJ|wz0~GurIv0UGH?9#3kq<8GCy79C*TR!fwKD!fwKD z!fwKDIL?rMn&t@n9IV24ALI2yrfm$1zRKF-tS#>9L;}&%+f8pbz1>@6SN+Id+=uK& z_>udGu$!=(u$!=(u$!=(uzN4U?&Hn$#Fzl#uEg|9BJr;lYl&46vETZ8S48X+qu$*l{&Zc5Z-3mi z@egZ>eQo^N>Y&75uM3^sURPrJ&s`hawZu*ve}*>x-_<7u+9(o#{Sx}|>FH7LS5YcW zj{b)$#&-PRA(8m>&069I?;pJR*6;t@-;+;%cl7pqx28OA^wHbzJvu4{**-iZua7+P z$j3$UFRv@v^F;FT?$sZ>`N8|Azg+$l(!c-RpTGU%wUR}8E%_x#zIOq+gU`4&|K^%D ze;?BSZm<_VJS0-3t^WxUe`(cohg?JbkX~5*)WI2%_IB=e7{i#9Ho)$-9JUpX>!Ol3>6M~*h8o>u2g>uS|%R$4<3cCWX|(EEjIwO!#RU+Lo(T)m)a9q?e;`@YN_hcJ2A8#&Z*&Yktl> zJ6a2xyy(a6(r=qtW#Y+Ku1uKNec*8S183tF`1RxVtZLt|=>(Yb7eW+G8y^GLPk{@Z zoLfop?SWQIUcY{Qav{m0$ua}aJ_MMc_l1ilZWoKQ>d5rW(HZx0Y+6!kYgSz~CY|&{ zy9D_!n*{^>bY6zOcUIoQWdbaL9x{J>293OI*aV7{At*<33@my}~X#>d9Ta)0Id z3-%X_1V6QQm_s`wrFKS2o!dhDktsfV=!|Z4iHmgazM_S^Ka;nC&vbwsZe6)Wb>$Y- z6{)GNNlkTG=&G=aHbr$3qa(i^`x@7DP1H7RGT)uBE3umftWOnMxSuo>T={+{U0R;L zCS?vZXec{E-TTJZnVB!6hQ5LXCf!0?eSH;3R=3M}leIGwOVi^7t@m}!Fonx=)KeNA zwN9Tj)_ITp1w;3mImG_zFqmz4erY^}#8NvFOU(RGgTzu-B$l1twXyE9X<(JI2Um)9;0)-KNDwjG0A!?nc6 z1q`-iLzAczho*ZqM%*@a=Pnhk=-AFL;=*kX{xDtGxaI0URofJlM_gW~Nqw7CXOGg| zFdrEx{Ys&)qpviWEFzp*J7Norf)HECsilwuwi;JIQCIa7vE|67uKVgI>bicSuIneF ziFqguavP*g@m-DGs+B}&a?%^p2ItD!xgx{z5=JYu*zN0E5lZ9eihBj40m@U%jf(PA zC#f3^+~{OJ#&*$SfkY^38CW+DFrsa31B@i*O61?bIH#fClKYh`6`UeXq`Z#yb8LS@ zI#qXnn#!mQ$F|mxrckK}vC%!8N*MIXT8HeapH7LbemDd~84&b?Rw#vw>dPQD19~RMb>=mRgm-ZNVwT zJr#KmDXrD=>k*<7R|wYH(0#)(*0R6YUw6T+)v_X(cXZJk23zbe_SbeAi$v2L4y~79 zzF*lTnBOCYA6Y;I^I~i0*_{aHdp1gfc|gfKjg=3NM}=TM$#&0UM=*ak$0@;lZK50c#d^LQksk!}^vsJlpPu>fqeXEL%-?Bu zBA6$bhd-C$k)n4K%oEI)euKd|4{vZzH~xFyM3fWE6U+m#q7$~$)67VQ%|$X0%oEHL z%s)U%2EjbR{CYto^vu&U&oKL4%cZ4fo}PK8J8yY6xcz{-Msap=VyGrSg{W0g(XjZq_vk;UZz+Ac-P#fi{~9 zL@-Y85fA`i8KJ}ZcPmFx^)>C+Kpb3K}44N=#!iaXOa0{Jl1J1SWEu=)I z05S!TDS%7?7IAyz@F!D%zFUCMj>l=q3A+RqyHwS2d+)p5tJ^n=?!NssJdQi<*X2R8 zBo{STyf_i6pUQJv3Jqxz*hB}mjq-81jr6tB>iN)*10?&WnQw9vqY6}+7JgAi`T?@P zH=yjc3R9FO)52$BdxRzfYsAf>E#p#?k=lJewHHe83ixGVBGhsXwLwSQ%$Av{?$+AI zdeOu|W-<&4{#Yu*w8VfI`8ZxOd2Vz>Fi$W~Fi$Y=7XusSzEmm*<_YEr<_YEr=1KdP zWYC)l<^k$jMG26X7y8~4CUu(NGI<|{8)HX^@7FZXA9SP5^4PdBB0v-7`V6#d? zzhyF*x)$WOP7`Oz@^j?jZfL*TdArl@+nk1HTVry8o7WF-WHG@!!92k{!92k{!92ly zwjh$&g_om?xOO7ZnJ>Ji$D{ zJi$D{Ji$D3=G)AfXP7-b^V?+n7# z1oL+Wk?ei|$?$Az1oH&*1oH&*1oH&*1oH&*%$aAH{dPq%43E}JFwZc1ZNm`h=0W0a z{HDB@yK{)<3c)l>(c`OUs+mos&>%Wp1@dRI_+QjY$It9E(z_BAK*m+vZ3s$D+UEkJov zBwm=CyWE^^OsPXtQ|i*^$H%16{~M>ccZtq}&$@z|->Oeh71aFJH%@0_%;;1sV~C{z4ZfhgEaaNx{W?163--Y0iyfK zxs}w+s=_Q6&BY=LkEzz8pXH`_zAP3dkB*PkpSx({cCk3Cj!ZpsXnK6?ipiEtHmkm1 zva2^t*=|lXrVlp`tII3JViGr>X*}DQ86SHAjP3H7U_u=_s!o+{bkX#4YWm3R%rmox zpHt^woEsmj`s)I&Zk_%*>iX-O_0{$D*S9gqxBL1_uI`ld*G-YQiXy2v`XAC?t24KA zk@&|i))G7Y^=;Q*7=f=nJGD}6-E?A1;c>kGsqusARlFXV!tZwE zlddP9?7mspA~Yw?XT}sDc01mz{;mWz7rSDP&nLl3_e<%C3sxyPT!e&j?4Qa3dX=2$-!SU=b4i^e+dv7azKI&+5ow9)w_ z3tmESlq*|-c^)d1tx#3jiq1;3vFetsz=~4cvK8`i^thC*P<7dgYPCvi&xDiaYj^|PZV55v>PL$xF#5%=!s)dUE!L`K4n}aRcz|`sFU8!Dq6So=l?H!%sCMve` zi|9|AyDiDaRi)BuGs0ekOY1bJZ!7BTM5@b6Aw#5IDb#iJlfvahII#kICY~(ePd}QF zzR0nqj52ZP76SOJ5tFS9| z!nArr+29=MpRElyB*?f@jtjNE*?xKLG>3l(!f zxKNc1Fpju&IWAOHbD_!-!O&G)sH);Zz5iWcr>E#{Pfr<;Vc5p1nL&jCgisHQVuiu3 zHNvhc=@ioHhtmr?+dS?*P!&5Gs?#Q&%8A=RM|l-xwxhY?MY;3U`fH_Wc2w9ty%M`M zKGvsLIz_X(oj}LzD5AACAS$OjN3C{Z>vyVcPublg3TU;^0OJ7ahzv}z0kwtw1T@Kh z>g)y80wU}t>;^L@%@aFp#u*lUm9V>3R0+HHfN4gS4PkfBGDO&2Sq}-j#k0|8MBM-W z48m@)YwxswjIevqIY~HKo0j&7-Kft=)3bT)puLu`8)Y%*?PgQ}tkydpJwez_Z#TW& zn~sPe?1rDNFCc@ko3I=AuRCLR2)iYXgWm2wl(8jYsf68x-HaE0fVgVHZo+N`8IuX zx3BH5bY{YC!fwKD!fxhvSJ{vZGG~xE()O9#y&38OF@p?)Wr(nwLFU-NncKbHBuv#p zV{SLujB1(!2AOY)MU1eUu$!=(u$!=(uv>KYJwdxiZ#TW&^mdc07M#BHcGKHUZ#TW& zHTfY@_VxmKH(@tnH(@tnH(@tnH(@uK%s&#*KE2(jK9T02(r3!&4ySw)b`y3Jb`y3J zcC&!kcE|YiF=Xrt4ib@h{@p8d-~ ziL5PdQA>UB<`J#1wm3?-@5}o0Gz~Bio`LWTgl8apPcht4C7GsyL4?90XUsom{<)bi zqlUV|0Po!j@V>8G6+^QFhT}dcHk^?QTVxs_>?Z79G;xrb3}KNrR!e1=mW5B)y?|a( zNm|$>R;?s%mj=d|MH?y`r6vokD-9Kit=k6bTFF3o2EvCHRZp*OQOFfO;TZ_eKzIhiGZ3DE@CeAedrIDp8f6K)Z5Y;Bk$Z&F>%NLA1L48VhQ*cHvM!q| zNSm-5osuUZtPLto<`KAWLSqTLCtytw=JX^lvNBL+*<>w?*db?$D8tZ?T1f^Igv!gj zWmCd#!fy3inHPx)N*qauTRGNJq09r6(k1K`+e3~nl#r~Vrl!-m`uoY!&7JYO@o!)MGyBP@o02^YpfbOg7(mNTMf}tb~CD~Lc$$jsl^meaFA>F_d zBkU&Z#=2qtIrGogY#z)%7mTbTza9G(iRJz;IZSUiz1{S7)7wpN_eVFP4ZU}g=80{N z8QWT;?9k6A!tOPiGhw%nVCV>rh(v;-b91fqVeZ=kj)MJ^h#<>`u$!=Z&E`SaP1r4l z2G$ne?r=+byXozwx0~MXVULA+$lxT+6B{#65O%N8oC&)LyQ{!fg~ap)Q?`A0b7BdG zRttX1hBmjA&PUiy*u7?A5Ox!G6L#+eX%1mGVK-qnVK-s7Lb#Pn6k^ED-U+!O>|Uce z6Lu4J6Lu>keTF^)h8^SES}Y|znPf62lR25p$z)C@bEL*oD>5|=K;BKEJYu2+`qb}15{zgSCjMZ_S@6%qTysCPGs|0r2c;=yCCjo+vx_Oq2M8)|GgB$+hwGwZu*vpFtaMu0Ao)Mv-{>{{x4x89o32 literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek new file mode 100644 index 0000000000..558be50008 --- /dev/null +++ b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek @@ -0,0 +1,6 @@ +# A basic test of the speculative service detection + +# @TEST-EXEC: zeek -C -r $TRACES/http/http-post-large.pcap %INPUT +# @TEST-EXEC: btest-diff conn.log + +@load protocols/conn/speculative-service From a810365f0e6e55ae423697b682e68683bec362e6 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Fri, 30 Aug 2019 11:30:33 +0200 Subject: [PATCH 3/5] Update test-all-policy script. --- scripts/test-all-policy.zeek | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 365dafcf71..67f24fa3a9 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -67,6 +67,7 @@ @load protocols/conn/mac-logging.zeek @load protocols/conn/vlan-logging.zeek @load protocols/conn/weirds.zeek +@load protocols/conn/speculative-service.zeek @load protocols/dhcp/msg-orig.zeek @load protocols/dhcp/software.zeek @load protocols/dhcp/sub-opts.zeek From 81b2b2121192a50c4d03170a30665a1c294c158d Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Fri, 30 Aug 2019 15:16:37 +0200 Subject: [PATCH 4/5] Improve logging of speculative service. --- .../protocols/conn/speculative-service.zeek | 4 +- .../{conn.log => conn-post-large.log} | 4 +- .../conn-wiki.log | 43 +++++++++++++++++++ .../protocols/conn/speculative-service.zeek | 7 ++- 4 files changed, 54 insertions(+), 4 deletions(-) rename testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/{conn.log => conn-post-large.log} (93%) create mode 100644 testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log diff --git a/scripts/policy/protocols/conn/speculative-service.zeek b/scripts/policy/protocols/conn/speculative-service.zeek index 2fb9b1cdaa..ba1162d809 100644 --- a/scripts/policy/protocols/conn/speculative-service.zeek +++ b/scripts/policy/protocols/conn/speculative-service.zeek @@ -30,5 +30,7 @@ event connection_state_remove(c: connection) local sp_service = ""; for ( s in c$speculative_service ) sp_service = sp_service == "" ? s : cat(sp_service, ",", s); - c$conn$speculative_service = to_lower(sp_service); + + if ( sp_service != "" ) + c$conn$speculative_service = to_lower(sp_service); } diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log similarity index 93% rename from testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log rename to testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log index ba0ed8ed25..0519bf5419 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path conn -#open 2019-08-29-09-45-13 +#open 2019-08-30-13-12-19 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string 1567010592.624680 CHhAvVGS1DHFjwGM9 127.0.0.1 37526 127.0.0.1 80 tcp http 0.008395 61907 60478 SF - - 0 ShADadfF 10 62435 9 60954 - http 1567010639.143657 ClEkJM2Vm5giqnMf4h 127.0.0.1 60644 127.0.0.1 5000 tcp - 0.015853 61917 60478 SF - - 0 ShADadfF 10 62445 9 60954 - http -#close 2019-08-29-09-45-13 +#close 2019-08-30-13-12-19 diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log new file mode 100644 index 0000000000..cf12fbe5f1 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log @@ -0,0 +1,43 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2019-08-30-13-12-19 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string +1300475167.096535 CHhAvVGS1DHFjwGM9 141.142.220.202 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 73 0 0 - - +1300475167.097012 ClEkJM2Vm5giqnMf4h fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp dns - - - S0 - - 0 D 1 199 0 0 - - +1300475167.099816 C4J4Th3PJpwUYZZ6gc 141.142.220.50 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 179 0 0 - - +1300475168.853899 CmES5u32sYpV7JYN 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.854378 CP5puj4I8PtEU4qzYg 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.854837 C37jN32gN3y3AZzyf6 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.857956 C0LAHyvtKSQHyJxIl 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.858306 CFLRIC3zaTU1loLGxh 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.858713 C9rXSW3KSpTYvPrlI1 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.891644 C9mvWx3ezztgzcexV7 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.892037 CNnMIj2QSd84NKf7U3 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.892414 C7fIlMZDuRiqjpYbb 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.893988 CpmdRlaUoJLN3uIRa 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 38 89 SF - - 0 Dd 1 66 1 117 - - +1300475168.894422 C1Xkzz2MaGtLrc1Tla 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 52 99 SF - - 0 Dd 1 80 1 127 - - +1300475168.894787 CqlVyW1YwZ15RhTBc4 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 38 183 SF - - 0 Dd 1 66 1 211 - - +1300475168.901749 CBA8792iHmnhPLksKa 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 36 131 SF - - 0 Dd 1 64 1 159 - - +1300475168.902195 CGLPPc35OzDQij1XX8 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 36 198 SF - - 0 Dd 1 64 1 226 - - +1300475169.899438 Cipfzj1BEnhejw8cGf 141.142.220.44 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 85 0 0 - - +1300475170.862384 CV5WJ42jPYbNW9JNWf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 - - 0 D 7 546 0 0 - - +1300475171.675372 CPhDKt12KQPUVbQz06 fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 - - 0 D 2 162 0 0 - - +1300475171.677081 CAnFrb2Cvxr5T7quOc 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 - - 0 D 2 122 0 0 - - +1300475173.116749 C8rquZ3DjgNW06JGLl fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 - - 0 D 2 162 0 0 - - +1300475173.117362 CzrZOtXqhwwndQva3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 - - 0 D 2 122 0 0 - - +1300475173.153679 CaGCc13FffXe6RkQl9 141.142.220.238 56641 141.142.220.255 137 udp dns - - - S0 - - 0 D 1 78 0 0 - - +1300475169.780331 CFSwNi4CNGxcuffo49 141.142.220.235 6705 173.192.163.128 80 tcp - - - - OTH - - 0 ^h 0 0 1 48 - - +1300475168.892913 CykQaM33ztNt0csB9a 141.142.220.118 49999 208.80.152.3 80 tcp http 0.220961 1137 733 S1 - - 0 ShADad 6 1457 4 949 - - +1300475168.724007 CUM0KZ3MLUfNB0cl11 141.142.220.118 48649 208.80.152.118 80 tcp http 0.119905 525 232 S1 - - 0 ShADad 4 741 3 396 - - +1300475168.855330 CwjjYJ2WqgTbAqiHl6 141.142.220.118 49997 208.80.152.3 80 tcp http 0.219720 1125 734 S1 - - 0 ShADad 6 1445 4 950 - - +1300475168.855305 C3eiCBGOLw3VtHfOj 141.142.220.118 49996 208.80.152.3 80 tcp http 0.218501 1171 733 S1 - - 0 ShADad 6 1491 4 949 - - +1300475168.652003 CtPZjS20MLrsMUOJi2 141.142.220.118 35634 208.80.152.2 80 tcp - 0.061329 463 350 OTH - - 0 DdA 2 567 1 402 - - +1300475168.902635 CiyBAq1bBLNaTiTAc 141.142.220.118 35642 208.80.152.2 80 tcp http 0.120041 534 412 S1 - - 0 ShADad 4 750 3 576 - - +1300475168.859163 Ck51lg1bScffFj34Ri 141.142.220.118 49998 208.80.152.3 80 tcp http 0.215893 1130 734 S1 - - 0 ShADad 6 1450 4 950 - - +1300475168.892936 CtxTCR2Yer0FR1tIBg 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 - - 0 ShADad 6 1468 4 950 - - +1300475168.895267 CLNN1k2QMum1aexUK7 141.142.220.118 50001 208.80.152.3 80 tcp http 0.227284 1178 734 S1 - - 0 ShADad 6 1498 4 950 - - +#close 2019-08-30-13-12-19 diff --git a/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek index 558be50008..b95f0f337c 100644 --- a/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek +++ b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek @@ -1,6 +1,11 @@ # A basic test of the speculative service detection # @TEST-EXEC: zeek -C -r $TRACES/http/http-post-large.pcap %INPUT -# @TEST-EXEC: btest-diff conn.log +# @TEST-EXEC: mv conn.log conn-post-large.log +# @TEST-EXEC: btest-diff conn-post-large.log + +# @TEST-EXEC: zeek -C -r $TRACES/wikipedia.trace %INPUT +# @TEST-EXEC: mv conn.log conn-wiki.log +# @TEST-EXEC: btest-diff conn-wiki.log @load protocols/conn/speculative-service From b216e9cbc957ecabfafcf05b115d9c14de87960a Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Fri, 30 Aug 2019 20:19:24 +0200 Subject: [PATCH 5/5] Improve dpd_late_match event generation. --- src/analyzer/protocol/pia/PIA.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/analyzer/protocol/pia/PIA.cc b/src/analyzer/protocol/pia/PIA.cc index f035f942be..e19e4abf80 100644 --- a/src/analyzer/protocol/pia/PIA.cc +++ b/src/analyzer/protocol/pia/PIA.cc @@ -152,10 +152,10 @@ void PIA_UDP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule) EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal(); Ref(tval); - val_list *vl = new val_list; - vl->append(BuildConnVal()); - vl->append(tval); - mgr.QueueEvent(protocol_late_match, vl); + mgr.QueueEventFast(protocol_late_match, { + BuildConnVal(), + tval, + }); } pkt_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY; @@ -301,10 +301,10 @@ void PIA_TCP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule) EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal(); Ref(tval); - val_list *vl = new val_list; - vl->append(BuildConnVal()); - vl->append(tval); - mgr.QueueEvent(protocol_late_match, vl); + mgr.QueueEventFast(protocol_late_match, { + BuildConnVal(), + tval + }); } stream_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY;