From 788b56a6528a06d5d63e860bb91c530bd07eb89d Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Thu, 29 Aug 2019 11:47:04 +0200 Subject: [PATCH] Add speculative service script. The speculative service script handles dpd_late_match events to extend conn.log with infos about potential protocol identifications. --- .../protocols/conn/speculative-service.zeek | 34 ++++++++++++++++++ .../conn.log | 11 ++++++ .../btest/Traces/http/http-post-large.pcap | Bin 0 -> 247952 bytes .../protocols/conn/speculative-service.zeek | 6 ++++ 4 files changed, 51 insertions(+) create mode 100644 scripts/policy/protocols/conn/speculative-service.zeek create mode 100644 testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log create mode 100644 testing/btest/Traces/http/http-post-large.pcap create mode 100644 testing/btest/scripts/policy/protocols/conn/speculative-service.zeek diff --git a/scripts/policy/protocols/conn/speculative-service.zeek b/scripts/policy/protocols/conn/speculative-service.zeek new file mode 100644 index 0000000000..2fb9b1cdaa --- /dev/null +++ b/scripts/policy/protocols/conn/speculative-service.zeek @@ -0,0 +1,34 @@ +##! This script adds information about matched DPD signatures to the connection +##! log. + +@load base/protocols/conn + +module Conn; + +redef record Info += { + ## Protocol that was determined by a matching signature after the beginning + ## of a connection. In this situation no analyzer can be attached and hence + ## the data cannot be analyzed nor the protocol can be confirmed. + speculative_service: string &log &optional; +}; + +redef record connection += { + speculative_service: set[string] &default=string_set(); +}; + +redef dpd_match_only_beginning = F; +redef dpd_late_match_stop = T; + +event protocol_late_match(c: connection, atype: Analyzer::Tag) + { + local analyzer = Analyzer::name(atype); + add c$speculative_service[analyzer]; + } + +event connection_state_remove(c: connection) + { + local sp_service = ""; + for ( s in c$speculative_service ) + sp_service = sp_service == "" ? s : cat(sp_service, ",", s); + c$conn$speculative_service = to_lower(sp_service); + } diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log new file mode 100644 index 0000000000..ba0ed8ed25 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2019-08-29-09-45-13 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents speculative_service +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] string +1567010592.624680 CHhAvVGS1DHFjwGM9 127.0.0.1 37526 127.0.0.1 80 tcp http 0.008395 61907 60478 SF - - 0 ShADadfF 10 62435 9 60954 - http +1567010639.143657 ClEkJM2Vm5giqnMf4h 127.0.0.1 60644 127.0.0.1 5000 tcp - 0.015853 61917 60478 SF - - 0 ShADadfF 10 62445 9 60954 - http +#close 2019-08-29-09-45-13 diff --git a/testing/btest/Traces/http/http-post-large.pcap b/testing/btest/Traces/http/http-post-large.pcap new file mode 100644 index 0000000000000000000000000000000000000000..8fd0e8df2db616f71bbdf6542c89726f9df3c0b7 GIT binary patch literal 247952 zcmeHwTZ~=TdETCpEK>taAct*X*AHEh;Y9M9IcJ6%Qq+)Qco8p&DUlUNwcSE?_nf`v z%o6vF`!bwiYHI)oHDjYlEEod%6bFc4#c?a42;$q<8YM@BGp0o;M=L8*h4-Up$-tWBKHFM{mD(Ys&LR zAHDtFqoW7L&cA!^*W~q)M;`gONIto)WX}`H$N#XF>|K82&CiVd?tg^zPk;C4Z~u6$ zWRYG=z68mqaV9zXAFjIt9-#pxyQcrzxdit0;XM!a2kEx3`E|=Bv(Ia0uGILC2OS4Cg zHm06d=S}Ns(mXUZJ%#_%>YUBYLbBYHH^;|LwOS^H)WR$m&BY=LC;T)GZObog5;vFS z6}`On)hIl6?fI$3a}%Fye$G8RS__)I=*R8SZ<|?V;>lO8Oqkex;BfZ?XX929*tk8b z+Ba-Ep#rlI`h^)EJCno(E^u;gCB?S~S}}S3`t`|$B#S1?EHom(1idd@G;zCFoK;7r zXO7Oemt)hC=Fh6D#-x*eXqO=WWwT&1lg+BdqDW_(&2INL(Z6Xs^8+*aD{*dGWo9N{ zHrGm%7db{@d~AFy_g9|3V1Kbl@KfuHNoFE#)4Ysykc3I5b6aRXGR0>PozblXLJCuq zc-ar|sTQtQo6r`Syq%js2iA{toWy0cqK(a>B+!M4QV6o~k`2nZ&}E@Re;y~&rl?M0 zbmX^VU*mGFl`9ZnGT)uBD{(0a%d{x{hCWs3C6guEB)NRwZ?(!?zESAc%DhN)P~wo5 z8~48Pb!O(vsG+Z*;kt#k`ueJ$2fAI(o2;FgSehOuXuYos|Ei6Ad?NSbR#8GiUaZq6 zjdk8*f89BU*k7BSUmA}WN}FrH#4^In54FM4w3)?H?yOK7>qQd>*Z{Cp@yAjbre%SR zM?Q|1OrB$bB_ZarHPsrvR#xjvO5r$&$D*lO}3D+6toP1ds5 za%4Z((%Dff$x<{rFVW%@7ak_qCJXtF%|qEY-3Dn>d{<+)Y9$exob-mY!MU<_uE?;w zgwYBucKiBP)WZDBW#oK}#yt8Lv(PSJ4e2Cxqk$Wp%*WU+VvgDfgA!Ob4=|!_Z3B!X zZfuc%1H+q!eoO9GvQ%)2G;x#P&$0at=~UhQX)2>K9NSv8aH&*?I@Y(pJv>SnhQ8Jz zyXvP?Vyhoclk6&#qFi>gTBm<@)KqK@9hFxU){ef57xBNV^;X(xNAc~`MX`_LV*|>l zQ%|eGfL_{BUu%t2)KqttT9v?U!70Q&6?qRSt<~}aJ|eE@!&)16sT}G9wu(L-%k*eh|#lGf&U_ z14MBU%-?BuBA6$bhd-C$krFLM&wQUtc*ny@3FZmr86myp5#eTOtW6^Fy-(auFi$W~Fi$W~Fi$W~Fi$W~ zFwYV>4?{^Idgkewr)U15w&yX-o~7vw3R02%pX~qJ$^Jh)E-bRylg)k)mN8_QJ;Ur7 zX5X!s)~%GroO$NV-&s4Do_Tua>6srk&%AnP;nknqeB3+jdG5fk?&E-W*7Ii4D<{2^ zkNs8ry626^;az^?&CiVd?hju)`%`cAiIK0~dI~R&92h(Q?zvx+LnQtWNPHD?^)d>c=D;cYYjgcYFi$W~Fi$W~Fi$W~Fi$W~Fi$W~Fi$W~Fi$YA zyT1NiDlkVdPcTn#I&9#S%nF-hRv=S=Ep%Llq2Dcty&-yz7GmE@lWE~IvAsaSJL3em ztYo>2)b5upf1$Lw_RGRVsO1`JQ%2fmw#-b$>!WR~7fl>wCPTmCkEJq9%fc^=d>k*C zJU2Qbm?xMgm?xMgm?xMgm?xMgm@nf(?;&y(dgkewr}1Ohj2{H^1oMFBvCTudnYfZs zHq~ZXW4CH0QIrG;e#g1FR{EeJC754h_>l$l5}5BoI}h6qj2tN_t6Q*f=|Z{sBU;hZ`{b>^$g$>jp2OvOYwvT5+vTypAh-IV9K^n&pbWz z4=`8gnP)*h7UW|=K4e{GI=@R5X?o`AnWtyI*196U{qP0nbQuBYncuFe6~hp&K;nhX z5ijiL{&E#5|INBJF6Ug0%cw3A*{yMTF8Ev2xcp{cjmxjrH7@`4_a67Ipx&e${SQ~; z@@)EwllU+0Dp6`&KG!Wf`E`+aVQ%hnbGk954oyv|OP?PflScnGPI2$ToNqtk%4q&c zeF|4b6AeaX%O9;ifm8GfbIK{cE)xGAB>n`gm81XR8vUo~KNE>R{$I7kPB;AMMd=1< z^dAf~`s-(sxG-@sIk%FUSyh1WQ`1LgXP%jzKBUgSI5$34_1DPT>-N{9uD|}czPet2y)}nHzJ0g;`m{(~MQKzV z{SWD{)tU4QBJtvDwZy8w-kNj$g%SAL(}VqW{1;xhbY|{Lm(MEcrW0cdkK_GMjUQC6 z;`PWBezzl^bUpcG_szlQ%eVbjtIXx|g>sG-6_iOwq&$l24-?IR@r=s6dXPDsqg@O0==+l4D>+sV+H&d>lP4SurF4DTGeKRy$F}qX-?l()Y*xY>_0_{d?y(q^-7_x zqn{Ma)e%mtkfVbqOZd}|CZsQNY$m4PzLCTm%2Et2J*5KuFAMlRmRo*?*cnLMR$98 z$`TodZLFFZR2X2!c~}%H40f#%c2!BIkXApEP?XMUo5$S;s$xe&b=ss;IdL24D6gW- zb~IPKD0jYi$p#4J=xCZ96}C^W#BPm`^(mH4(X4JK&@nrTXsr#1%IVHgtDV^Tood@t zcK3(^S}iodIDk4L0|PCA2#N^G0-$dYX;uq}lZKOK6O)GKi7hhY42!->*j+2Cgx!0< zG$YG~u)Aj&BJ8fLhlJe%GW(2(``@2I*bO`Gj{C<5y9b?G}dP2)hZp;dQ+;c89Q=-fl$1 zWql}POTtnKy9v7)FZ=*;)r8%I-3-Rxt)vCQZo+QDZo=*d%!458ChR8cW{~-g#e5TX z6Lu4J6Lz~C9_Ds42flmn-4k2^gx!SQgx!SQgxy<0QBk3R9gG!Zw$&D8TYcm#M+TWQ z$egTC<^B(A*Z~7-K12jNQju*&D?IV<&jctGmL-3Vm&16W^Ok&a8jym zH}^5U-Sl?T+f8pbz1?JdmpVLqjKFnR^i@)-)l-MCo3NX(dy_=_!?KMLb`y3JcJDyR zA;NCLZo+QDZo+QDZo+PBdR3=bK#T>%P^WEQ7MT;6M<#PJnXk!E$qBm&y9v7qyIEQo zt6*0cx|wyev80N&DT=2J_8U5_s>JuYhz4a7c962y{{@rt5mqR3; zgT%kN>bavU4o^hl)xS=^C=&nWk7|kD{IZE=)0 zr)j{%0c(p(Pwy~$cuVQ$zYRwZh-N3A4FQ9nH|^OjAK%^D{2^vG|~O^C|h zHdl{j{yFo{nSajw^LWVyWn3t*`Gh|1Uq%fz4<=0m%s*%Txr9Hn$XS*s@CG{aM8DN4 zBb5G=Zn0%vBq|APg3P{lZ1WtE!q-Zx=0iUYaMw>W-{b~+URb8k=gUalxRKdV1tv6` zru4e6qT0fAEddk!03ZQKA#N*!LN0{eagdn|!(n5!REB9u*lp|rVYe(*!tPz#;P1Pe z$z}$VhiqoOy==>iQuS?)`^-^C*iG0?*iG0?*qsEbFj1OhPyqLq4T7F#8u=A#ivwyC zb|+z3q<&GEC3m}u)C+=~{1V%$M7Apf%ZANt37c+}2M46R2!Ki0Eu-ecwL6dgg3*Vt zJDMQuZk73N*;NU<>FtJ5rRs#-_%0-?@UPlPyghKuElGi#o+-rSg}V2gX{__!M3n=# zGiIFXu6QQdv8rnO*KRZzqqm}r%_1VG{ceY(x0~K>db@W^E8C(#cm|o%+fCRFTSQLS zjVKTm*%$#SO9(ab3A@FDNZ8%g?t2h#UlfC*x0~K>db{cE-eV44!fwKD!fwKD7CB>) zGnWxnl5L%A>&)$D*w>DRecju`dmk?@VK-qnVK-qnVK+l0_Iro~gYmaWE2Fo24Po*I z!swXW%^-8Kt^2T(b@L#0I|pJ|wz0~GurIv0UGH?9#3kq<8GCy79C*TR!fwKD!fwKD z!fwKDIL?rMn&t@n9IV24ALI2yrfm$1zRKF-tS#>9L;}&%+f8pbz1>@6SN+Id+=uK& z_>udGu$!=(u$!=(u$!=(uzN4U?&Hn$#Fzl#uEg|9BJr;lYl&46vETZ8S48X+qu$*l{&Zc5Z-3mi z@egZ>eQo^N>Y&75uM3^sURPrJ&s`hawZu*ve}*>x-_<7u+9(o#{Sx}|>FH7LS5YcW zj{b)$#&-PRA(8m>&069I?;pJR*6;t@-;+;%cl7pqx28OA^wHbzJvu4{**-iZua7+P z$j3$UFRv@v^F;FT?$sZ>`N8|Azg+$l(!c-RpTGU%wUR}8E%_x#zIOq+gU`4&|K^%D ze;?BSZm<_VJS0-3t^WxUe`(cohg?JbkX~5*)WI2%_IB=e7{i#9Ho)$-9JUpX>!Ol3>6M~*h8o>u2g>uS|%R$4<3cCWX|(EEjIwO!#RU+Lo(T)m)a9q?e;`@YN_hcJ2A8#&Z*&Yktl> zJ6a2xyy(a6(r=qtW#Y+Ku1uKNec*8S183tF`1RxVtZLt|=>(Yb7eW+G8y^GLPk{@Z zoLfop?SWQIUcY{Qav{m0$ua}aJ_MMc_l1ilZWoKQ>d5rW(HZx0Y+6!kYgSz~CY|&{ zy9D_!n*{^>bY6zOcUIoQWdbaL9x{J>293OI*aV7{At*<33@my}~X#>d9Ta)0Id z3-%X_1V6QQm_s`wrFKS2o!dhDktsfV=!|Z4iHmgazM_S^Ka;nC&vbwsZe6)Wb>$Y- z6{)GNNlkTG=&G=aHbr$3qa(i^`x@7DP1H7RGT)uBE3umftWOnMxSuo>T={+{U0R;L zCS?vZXec{E-TTJZnVB!6hQ5LXCf!0?eSH;3R=3M}leIGwOVi^7t@m}!Fonx=)KeNA zwN9Tj)_ITp1w;3mImG_zFqmz4erY^}#8NvFOU(RGgTzu-B$l1twXyE9X<(JI2Um)9;0)-KNDwjG0A!?nc6 z1q`-iLzAczho*ZqM%*@a=Pnhk=-AFL;=*kX{xDtGxaI0URofJlM_gW~Nqw7CXOGg| zFdrEx{Ys&)qpviWEFzp*J7Norf)HECsilwuwi;JIQCIa7vE|67uKVgI>bicSuIneF ziFqguavP*g@m-DGs+B}&a?%^p2ItD!xgx{z5=JYu*zN0E5lZ9eihBj40m@U%jf(PA zC#f3^+~{OJ#&*$SfkY^38CW+DFrsa31B@i*O61?bIH#fClKYh`6`UeXq`Z#yb8LS@ zI#qXnn#!mQ$F|mxrckK}vC%!8N*MIXT8HeapH7LbemDd~84&b?Rw#vw>dPQD19~RMb>=mRgm-ZNVwT zJr#KmDXrD=>k*<7R|wYH(0#)(*0R6YUw6T+)v_X(cXZJk23zbe_SbeAi$v2L4y~79 zzF*lTnBOCYA6Y;I^I~i0*_{aHdp1gfc|gfKjg=3NM}=TM$#&0UM=*ak$0@;lZK50c#d^LQksk!}^vsJlpPu>fqeXEL%-?Bu zBA6$bhd-C$k)n4K%oEI)euKd|4{vZzH~xFyM3fWE6U+m#q7$~$)67VQ%|$X0%oEHL z%s)U%2EjbR{CYto^vu&U&oKL4%cZ4fo}PK8J8yY6xcz{-Msap=VyGrSg{W0g(XjZq_vk;UZz+Ac-P#fi{~9 zL@-Y85fA`i8KJ}ZcPmFx^)>C+Kpb3K}44N=#!iaXOa0{Jl1J1SWEu=)I z05S!TDS%7?7IAyz@F!D%zFUCMj>l=q3A+RqyHwS2d+)p5tJ^n=?!NssJdQi<*X2R8 zBo{STyf_i6pUQJv3Jqxz*hB}mjq-81jr6tB>iN)*10?&WnQw9vqY6}+7JgAi`T?@P zH=yjc3R9FO)52$BdxRzfYsAf>E#p#?k=lJewHHe83ixGVBGhsXwLwSQ%$Av{?$+AI zdeOu|W-<&4{#Yu*w8VfI`8ZxOd2Vz>Fi$W~Fi$Y=7XusSzEmm*<_YEr<_YEr=1KdP zWYC)l<^k$jMG26X7y8~4CUu(NGI<|{8)HX^@7FZXA9SP5^4PdBB0v-7`V6#d? zzhyF*x)$WOP7`Oz@^j?jZfL*TdArl@+nk1HTVry8o7WF-WHG@!!92k{!92k{!92ly zwjh$&g_om?xOO7ZnJ>Ji$D{ zJi$D{Ji$D3=G)AfXP7-b^V?+n7# z1oL+Wk?ei|$?$Az1oH&*1oH&*1oH&*1oH&*%$aAH{dPq%43E}JFwZc1ZNm`h=0W0a z{HDB@yK{)<3c)l>(c`OUs+mos&>%Wp1@dRI_+QjY$It9E(z_BAK*m+vZ3s$D+UEkJov zBwm=CyWE^^OsPXtQ|i*^$H%16{~M>ccZtq}&$@z|->Oeh71aFJH%@0_%;;1sV~C{z4ZfhgEaaNx{W?163--Y0iyfK zxs}w+s=_Q6&BY=LkEzz8pXH`_zAP3dkB*PkpSx({cCk3Cj!ZpsXnK6?ipiEtHmkm1 zva2^t*=|lXrVlp`tII3JViGr>X*}DQ86SHAjP3H7U_u=_s!o+{bkX#4YWm3R%rmox zpHt^woEsmj`s)I&Zk_%*>iX-O_0{$D*S9gqxBL1_uI`ld*G-YQiXy2v`XAC?t24KA zk@&|i))G7Y^=;Q*7=f=nJGD}6-E?A1;c>kGsqusARlFXV!tZwE zlddP9?7mspA~Yw?XT}sDc01mz{;mWz7rSDP&nLl3_e<%C3sxyPT!e&j?4Qa3dX=2$-!SU=b4i^e+dv7azKI&+5ow9)w_ z3tmESlq*|-c^)d1tx#3jiq1;3vFetsz=~4cvK8`i^thC*P<7dgYPCvi&xDiaYj^|PZV55v>PL$xF#5%=!s)dUE!L`K4n}aRcz|`sFU8!Dq6So=l?H!%sCMve` zi|9|AyDiDaRi)BuGs0ekOY1bJZ!7BTM5@b6Aw#5IDb#iJlfvahII#kICY~(ePd}QF zzR0nqj52ZP76SOJ5tFS9| z!nArr+29=MpRElyB*?f@jtjNE*?xKLG>3l(!f zxKNc1Fpju&IWAOHbD_!-!O&G)sH);Zz5iWcr>E#{Pfr<;Vc5p1nL&jCgisHQVuiu3 zHNvhc=@ioHhtmr?+dS?*P!&5Gs?#Q&%8A=RM|l-xwxhY?MY;3U`fH_Wc2w9ty%M`M zKGvsLIz_X(oj}LzD5AACAS$OjN3C{Z>vyVcPublg3TU;^0OJ7ahzv}z0kwtw1T@Kh z>g)y80wU}t>;^L@%@aFp#u*lUm9V>3R0+HHfN4gS4PkfBGDO&2Sq}-j#k0|8MBM-W z48m@)YwxswjIevqIY~HKo0j&7-Kft=)3bT)puLu`8)Y%*?PgQ}tkydpJwez_Z#TW& zn~sPe?1rDNFCc@ko3I=AuRCLR2)iYXgWm2wl(8jYsf68x-HaE0fVgVHZo+N`8IuX zx3BH5bY{YC!fwKD!fxhvSJ{vZGG~xE()O9#y&38OF@p?)Wr(nwLFU-NncKbHBuv#p zV{SLujB1(!2AOY)MU1eUu$!=(u$!=(uv>KYJwdxiZ#TW&^mdc07M#BHcGKHUZ#TW& zHTfY@_VxmKH(@tnH(@tnH(@tnH(@uK%s&#*KE2(jK9T02(r3!&4ySw)b`y3Jb`y3J zcC&!kcE|YiF=Xrt4ib@h{@p8d-~ ziL5PdQA>UB<`J#1wm3?-@5}o0Gz~Bio`LWTgl8apPcht4C7GsyL4?90XUsom{<)bi zqlUV|0Po!j@V>8G6+^QFhT}dcHk^?QTVxs_>?Z79G;xrb3}KNrR!e1=mW5B)y?|a( zNm|$>R;?s%mj=d|MH?y`r6vokD-9Kit=k6bTFF3o2EvCHRZp*OQOFfO;TZ_eKzIhiGZ3DE@CeAedrIDp8f6K)Z5Y;Bk$Z&F>%NLA1L48VhQ*cHvM!q| zNSm-5osuUZtPLto<`KAWLSqTLCtytw=JX^lvNBL+*<>w?*db?$D8tZ?T1f^Igv!gj zWmCd#!fy3inHPx)N*qauTRGNJq09r6(k1K`+e3~nl#r~Vrl!-m`uoY!&7JYO@o!)MGyBP@o02^YpfbOg7(mNTMf}tb~CD~Lc$$jsl^meaFA>F_d zBkU&Z#=2qtIrGogY#z)%7mTbTza9G(iRJz;IZSUiz1{S7)7wpN_eVFP4ZU}g=80{N z8QWT;?9k6A!tOPiGhw%nVCV>rh(v;-b91fqVeZ=kj)MJ^h#<>`u$!=Z&E`SaP1r4l z2G$ne?r=+byXozwx0~MXVULA+$lxT+6B{#65O%N8oC&)LyQ{!fg~ap)Q?`A0b7BdG zRttX1hBmjA&PUiy*u7?A5Ox!G6L#+eX%1mGVK-qnVK-s7Lb#Pn6k^ED-U+!O>|Uce z6Lu4J6Lu>keTF^)h8^SES}Y|znPf62lR25p$z)C@bEL*oD>5|=K;BKEJYu2+`qb}15{zgSCjMZ_S@6%qTysCPGs|0r2c;=yCCjo+vx_Oq2M8)|GgB$+hwGwZu*vpFtaMu0Ao)Mv-{>{{x4x89o32 literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek new file mode 100644 index 0000000000..558be50008 --- /dev/null +++ b/testing/btest/scripts/policy/protocols/conn/speculative-service.zeek @@ -0,0 +1,6 @@ +# A basic test of the speculative service detection + +# @TEST-EXEC: zeek -C -r $TRACES/http/http-post-large.pcap %INPUT +# @TEST-EXEC: btest-diff conn.log + +@load protocols/conn/speculative-service