From ea2ce67c5f2e7d80097b6264bb2c4db8faf9d2b5 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 18 May 2015 14:30:32 -0400 Subject: [PATCH] Fixes an issue with missing zlib headers on deflated HTTP content. - Includes a test. --- src/analyzer/protocol/zip/ZIP.cc | 82 ++++++++++++------ .../http.log | 11 +++ .../Traces/http/missing-zlib-header.pcap | Bin 0 -> 14844 bytes .../protocols/http/missing-zlib-header.bro | 6 ++ 4 files changed, 72 insertions(+), 27 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log create mode 100644 testing/btest/Traces/http/missing-zlib-header.pcap create mode 100644 testing/btest/scripts/base/protocols/http/missing-zlib-header.bro diff --git a/src/analyzer/protocol/zip/ZIP.cc b/src/analyzer/protocol/zip/ZIP.cc index 132515f29a..e71ba438b7 100644 --- a/src/analyzer/protocol/zip/ZIP.cc +++ b/src/analyzer/protocol/zip/ZIP.cc @@ -22,10 +22,9 @@ ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method) zip->next_in = 0; zip->avail_in = 0; - // "15" here means maximum compression. "32" is a gross overload - // hack that means "check it for whether it's a gzip file". Sheesh. - zip_status = inflateInit2(zip, 15 + 32); - if ( zip_status != Z_OK ) + // "32" is a gross overload hack that means "check it + // for whether it's a gzip file". Sheesh. + if ( inflateInit2(zip, MAX_WBITS+32) != Z_OK ) { Weird("inflate_init_failed"); delete zip; @@ -54,40 +53,69 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) return; static unsigned int unzip_size = 4096; - Bytef unzipbuf[unzip_size]; + int allow_restart = 1; + u_char *unzipbuf = new u_char[unzip_size]; + if ( unzipbuf == NULL ) + { + Weird("failed_to_allocate_deflate_buffer"); + return; + } zip->next_in = (Bytef*) data; zip->avail_in = len; - do + Bytef *orig_in = zip->next_in; + size_t nread = zip->avail_in; + + for(;;) { - zip->next_out = unzipbuf; + zip->next_out = (Bytef *)unzipbuf; zip->avail_out = unzip_size; zip_status = inflate(zip, Z_SYNC_FLUSH); + if ( zip_status == Z_STREAM_END || + zip_status == Z_OK ) + { + allow_restart = 0; - if ( zip_status != Z_STREAM_END && - zip_status != Z_OK && - zip_status != Z_BUF_ERROR ) + int have = unzip_size - zip->avail_out; + if ( have ) + ForwardStream(have, unzipbuf, IsOrig()); + + if ( zip_status == Z_STREAM_END ) + { + inflateEnd(zip); + delete unzipbuf; + return; + } + + if ( zip->avail_in == 0 ) + { + delete unzipbuf; + return; + } + } + else if ( allow_restart && zip_status == Z_DATA_ERROR ) + { + // Some servers seem to not generate zlib headers, + // so this is an attempt to fix and continue anyway. + inflateEnd(zip); + if ( inflateInit2(zip, -MAX_WBITS) != Z_OK ) + { + delete unzipbuf; + return; + } + + zip->next_in = orig_in; + zip->avail_in = nread; + allow_restart = 0; + continue; + } + else { Weird("inflate_failed"); - inflateEnd(zip); - break; + delete unzipbuf; + return; } - - int have = unzip_size - zip->avail_out; - if ( have ) - ForwardStream(have, unzipbuf, IsOrig()); - - if ( zip_status == Z_STREAM_END ) - { - inflateEnd(zip); - delete zip; - zip = 0; - break; - } - - zip_status = Z_OK; } - while ( zip->avail_out == 0 ); } diff --git a/testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log b/testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log new file mode 100644 index 0000000000..c4c96b7fb9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open 2015-05-12-16-26-53 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] +1232039472.314927 CXWv6p3arKYeMETxOg 237.244.174.255 1905 79.218.110.244 80 1 GET ads1.msn.com /library/dap.js http://zone.msn.com/en/root/default.htm Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727) 0 13249 200 OK - - - (empty) - - - - - FBcNS3RwceOxW15xg text/plain +1232039472.446194 CXWv6p3arKYeMETxOg 237.244.174.255 1905 79.218.110.244 80 2 GET ads1.msn.com /library/dap.js http://zone.msn.com/en/root/default.htm Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727) 0 13249 200 OK - - - (empty) - - - - - FDWU85N0DpedJPh93 text/plain +#close 2015-05-12-16-26-53 diff --git a/testing/btest/Traces/http/missing-zlib-header.pcap b/testing/btest/Traces/http/missing-zlib-header.pcap new file mode 100644 index 0000000000000000000000000000000000000000..66406a9a4022f112bf4873b462bb30d4fe4f862f GIT binary patch literal 14844 zcmeHNc{r49-=0KSQnqX@rjVq;jD0MFv9FW0Ni_yz3!|}R86=TZ6GDg)g`#9B6{4c- zV@Vz@^w>t?QDew3`0knK?S1Qcp3i^ZcO36=xR1FUX6Cxi>v!Jw?>evLoHs8kZ*qV* zK|ijwH4tbM@J2ZRZ$3ca1nB|)vkLp-1nm^#i^6h)jDZTJ@%kX*mW9|E`u#QAcLDUR zr$MN%lpA8Knt>epAiHW1h=Y@>debIO4i?(E+V#-Ttss!fmR=ne5(^Q8+Io78hA;#W z#RG*zyLIw@MGLFv1nB@hu?qY8IktnP=h?9({v|eX01=52FbdRu@uL|lGNmp~kPQcj zUH*#2p`-%AX8%pu#%sT8v#oQSMtCHD*6V2%^Vz1N>D{8Kd*ti zJ1#I74nCxGh@aOE=ZPcW2yn1>aBv`8Nhty!fK&7j3Q%;%`zzrBln8iyu#yMP6C2_e ztmqx=|D!p?3LD@Rg7w0|!MFfONDx1-qX7hoa)g7=XsZ9al5v1L-oq!r3l8>*@CiH& z27vsq!8o9D5RL#b@WKTEaaiLceEj^dN~(&=;DbQ7f!JUlH$R*<*xKIQ7_6bFtPOVd z3Gl#&1%YiGz-o$6ZLp#Zzz}I=2ZkytE2?NHtNrp3R8dt01_RpTJbbWVB+v-)^8CDJ_#hw8fg|slZfW$}kT#4OI;l6-^CIbu|sFvYM7ER11gm&;}zN ztsETfjCGA|AddDxY^+vVDwjo1-fb0mu1fABERWk%$KrsSd{Ji|UEQ;GO>r!lI zQ#AYsif7pr@BCoH!ioa0rno@t@>hynAJ82vtd7h78x}MZ1o|=Xz%XTHu&pJKKv;Kg z90W+OU;^F`4)({MfdJ!37pei%R95EaH39|*9PHp7au^I%16yDNSPiwnPz^W~3WvhL zrq&MZX5FQBNU;_jFeB6N%g%40V1M$U%VuRcXK7qmfyjIwt zV2Cv^JUxAI9zQzN0$YTzJJbYvg{x`7VJd%i_@`(r6;f`gGLT|-BEVCdR`k+wE?u!*@1*xbno54N?j0^0(gqRh}>djo5{gRvDJ{2yxk zyvAn&eF&@@^|$8;gThr|K(_z-9IbExUcugQuo_fFixrcz9mLw!$lSynY2aXPYXgMx zuc!KF#b4Fw6ic13f2tEd?@awx*WKnBvIms>+ahBu<}C(31>fY3brqKR?W(@O^z#dy z6RFdCd`zX~OAnoG!#Uwjc#7(G%;eU6ABvo<|MSmN?OYqV@55_|y%|@m*otA$gr%+KJ5!C$x3dqmqo# z@c!y{zZBB@sqm2W0XcLU{-cCgJ9_nx_PKkc*SOPJo0+e}q6a9C!e(U}u0P+#KTtgs zYvol}6JDtD$)%1!{hImP6M1KH*7quO)v3>pqG|Bba)jrIM_;R0Gp)8Nban8LV(p2- z(ZRH`)djD}$cb~s&h6Byff+9N#zun5QVdch-yA=_N&sUznLcRNhs?Jw3ZL`1(}#n>n{Eie8b;-BvIE`mUam0FmyV z?L-42ap9X1XAzmmY!!`mJ461eq$*S}tz=G+)IBEv_e(e{1v)EsNA~KJYg6nUWOX%SVMJUyg! z!?ACk19PI^I0sK;KK`Wc16`gjBq6YxopMq!*VO0fRisMuhXGR7ZGMb}Li*jtG5f+f z1=Z`+=H|Nb2I^wYbE~@%83SZuH4;=5;+WA)8>u2?^qZQJW-Uxjn<7NFuHItg(U(># zR_cz8x8EbZW1ff%8&Yf|nMn0B?sgP0Ol!^zZrg_6tD8q1oSr=GQjLoa)~W6I^2nP| z9U0}U7Zv54ZFP4rgv5)B9#X7aDmY5J{?2!N6&_)QWG<$Wk{xs6F6#-$$G~elCi#}= zh0+gPDxxetTy3aRueN-xO_QPx&>o5wr``farAsicM$5`1NFsm111by>dB4us_GNNsIxfh(Zq z+BDx#Ys8t#3u>1pkAghaXS8+iHoeXe>N{Dca&=Oh{`nN+T13Gtmq6HeOCih6BbP^* z*$2Kj4v8RIYAz0Hrz#7z46EhjIYwI0W>13FFZn}cj-EB0~*BX1_x^_xc!i0a53ZTFP^IR!aL zh0~!I)S!bUbESp3m_7{fwo8 z-`^l7T6ZPG5#>W!p5mC4hO%r*rRDp{QDghDCbFs$bB_kjQx$bbda1E~dfxTqM3JW- z%;VbZ86uo_Reng|Nin3v({&6&}-SCZ~oBR zqZPWmyM{?>4+CenTH|APsjseRUxktqTBNXDeaoZEZBDsj_$ja5bh=Mj#h&NI27SB0 zk(bM9=8+N?Ibv>!zYiUYj^0vKd2jL@>5tt+lUFZQ_a!5(1oaP<@V{hak$Il|wi0Yt zy_CLLzgO7JP|91n<;KAD6=l5@gA-AnXwtKNd*`$B$Z481DJr$8^qCGkX{+Co;fD(8 zfu*#~bCt4HJGc1`AN@9>KC#oRRiN>N_vzL59nGBrpZCO$*`6_~-8XuJEKF&|Z0Rhd zghtv&OWA|TgHDlGiK~}>XRc0*-bmTUEX&Q=@kC>@^>9=a1*}TR9$u3`3s(A0+s!31 zH>pWe^49WFOkw_vzQ*4K(ibDA5{9yh-r)1CgHIJ(Az3^58Px-Tz`(pH))Bf(ze)Hn-fA-Oka#~jXq&cH>({U{ zkxX@w!cqNCB3fI8SG+w08-mQvNsGm!RxU+Llcys$Ee&sV{a(-XT$w-;0+J3`>Fk%E zR%n^hb9iANN$h*-b)ZlrzU36Hv3sUb0h>eV{g`_;!M2oMJ6F)5jUx5l6nl2Os5s=n@{;|F8d)-~9&6se%{JNfVk)z4N43?|f!AO0KlloRDs5X<_0buQ{7!Mc*GLnQszlQcm=c=qioid%!F7bfiMv zp*i|!6pbe&gsW(%xBQuNRLqA5qQmbbW`-EkRU~^4$cGy86O1(y7T^c1!<$(@xGcm8 zEwa@sn(p@U@K3``U*$uVxX_d4x3m!7z`US$O;2wriNvk0)9v7rTNhpb{?vT-F^OuurHiWq!3QU` z&;#)f%h@6Kn|W1Cznbx-W{i1qp8hy-?P)>iu7(e7rtKl2>zDCipMgnzL&(i&EBR9`fzy{lj$Sy8DMN_AHY8>nvhX z`ePO;(E@=OMVZ=odpF81!DkaGHd9pfq4XwEJ)E$qQ+nWdL%ow?UiO>*ooMm7%f?}g z{4gR@P;J~q1d@01(Xi-5@eb(k2WY1>1$|dXG_N*^LuAHQ{LC?yc@BC-8Q1cgEcY+b zE22E)NE{vL(@7;FYo;Yyhpx!S6*atLO!~t2M?F(Tt{QiGHIxW`Y*Zq?T5H?t5hyRT zcB!#t*!jYBh6XJ3F6!dM^@#cogu<2a&Gb2m)fB{gEga{Bqt^1Jv1^DY+0mhD>6o)E zW5tMhjSRTrT4Ufmg+eb5 ze%nDRPNSyTX^&94$@zgWy~=YewH=J_BebLo0Ffx@@PLF1W z97!QQxzf;GM)OHYzZN@z_`-BMA*9RCIa05r82`22`QwKmguca+7z(p{ZRlG2>Ix=m zzS*l*d(7hZD)jOp#A}yr-{iyjtFP<7AazJs%h!a5EKSqrYhu>=sIQeUf!9&_eU)*_enZnt#+E4`o*LpA{>qbj(Yq?#>XC1idmDJTP+1*Oq%X zrR!bPX_MLdv{$>!P|4C4lUnX8mzzgh7`3g%V^FO$gx6Ub!u1J^H{nxznD4}16-Dve zsLM~eG6plP6tXXjW~C9$!(+aEliS2CF?4}}tFE_PiUq&(k({{%YR%kLv7qBrSO}g& z<-DnHT+L&4E4Dm{_DReiE_>S$m`jyV5`JA;!;iUEl^GN*zNUcHx_{GQrC)lex4 zL<(oy!fYE9N>{x@KOB|VVGr;ici!u z1>(YV|2MH>sU9~seKnzmVJVK5&YjJ(7fGiW-S{;;+laa1TYU`=E^4YJMt~b&K z65QTdJTr*%%!;9_AV!HWCC{oGmC0AWWX(j5f{Ira~J$b?48+u(ZS@Z~2rjG^Qf5jF~$kvJ@Ccd!3`dJJZg4Rx$Br(2*SF zELrmDk*Or>Gp&hZZBEYkDZfF~1QcUaE_*?y&F+o)Ce!8k)B?48RYyhz_2wnW+`I2z z@7t;eyKB>Q)NsnI^y$tJbUGq>DXsF6&8s)x?79UI-P?nBFWO)8gDBkNjD8XzFC%UB zrp7}$I{GQI}vw<&K;%Na`S`W^#2fllOtAPUq2m=agopzw(<}Bs{AMn zTlvK$z;{PLpii-D(EUPIfxZ>TC)8jSZ_>$!R~LoDceMBF%sy*6k_UQWayhER*}Js& zEyMRpx%i^1;vhv*%yf}NQ2KE8_c(X3IUj*WY{KCPMhFyKB!vsswW$H`;)0>_m3AT%0JlS+kA}vC8 zwc#@NnzHP}#B7M)2L*=}y+|tR|{cBT5X+7zMFsJK@lYPI1KMT=6Ay|?i z!=(~RKa=NGRruP!Cxg-R?UIP2i)zPi(rjBIu(1?AgVe1!G&dKT(qBz`KBDh)a#kty zyxy|6VNz4_(_>0r@B8kSO_|<%XeeH|X9VPbB^Fuj#Bux!dZ+k;|(p^8$`+wNDS z!AeCXGBPYi#XC!j&P%MG$lYzWYd^Px1J!;nZc&gw6JAm0DpB?m0xtt>d$|@5+1atEcAL-LHfl>#n7!Y1nDZKTe;hI+p69 zQX6oL=3f-#W)X3IwFKSko4O_Y!S_uu9tmb0pYppEZq@Wcgk&W-_vJ5LIBqk=RD;_h zmJhFRKlz;cmQ(L(nk0wdxh)wE^3BQqGI?k1xLY~LOyw(KS3kRoggt*xKPNjfiadR| zFJZFP&6o)%M)Ds2cxgQIhO1-`B~#5>yQt3fHb(WDJ%`Hb{R!<=d2{(RWgYvRM`?f7Z2eYT4ef5Ijx^ zcGcSfO3+K%Vb=itd?XIOnA@AFdVml1-FSwJ51F%xcjO-OCCF!ZL_qpVi^!X@K-Y1o zwD|o#4o*vSu64OVpM0>S9@e;__se2uoVFoTf9i^`p-1Dv8MCtmI~s%c|Mqn`CrWP{ zV&N&V{$4pg-k>c6UOb+Iu^NllBHF%Z_Q(x!BL?f|T1`F5LT;Q0verHNuJ6`fio>Rk zE>maeF(0t0)#*Sthv+)NT@m&|Jae|g$8`^sU_{zyQ=QGoj}z!nO|#z71V}DdzHP-j z!*b>-U7=Mi&XbT%_DZYcL3UJu+|-+dOgUoCm_>Ky!Qh1(#|8^HY()flU|bY6qu~iYpR)ST2En z(dqPwC$z6!d>zb{c&fLo>fPW1gcn3r#$89z>_8_eW0&&Jm{F_tsNhvIHaU+ zeu$MYlS|TXpQ(PkeK>r#Mls#k+yt%D{57bu;6^NPiWU`|*HnA(`XacWy1FM*{^abAdCw~{gyc!xmzGijoLkCZl}rc9%3Y&;mwFJYDX0Y zX`;m$5|DmtIWl!-a2j2&i3|!)lAGC96nAWjd86}!mp7I)-J5mdbz`D4nV9+6EdSXp zr5WmCB&{!T(ST;eEojT#&yYUGp^a%PC&w&2VaDyO8lWlcTas9jx)6KmV;1g^xI82S z&*QQ`?*8emZ}Jm<=e{R@ooMpKrEbA?)9Bk`Y2kx&lgSd}4tH+r-+IM);()&1bHipO zMhx3qtbMxfl0S>R_M3h{hX;u&$h1T-~G)c z8tu#f}DQS4z;{EIcurZ~l>c=>0F-2m1){#`G76_)r9w~LJ>61m~u z0lyXax6QcW-+?#$yAA(t!@v74@bCK8SN=Y>@+bVN{4No|M9EhEO7Mn%x8dJy_;(xr z-G+bn-{#+Gaj(1Pf5TS(^S>%TR`G}OC-2$t+%`P74bN@EbKCIT{@Xma??UUH9F=}! zALH)&%YoL^8V#$+I>z1k!>MB*3D1bFi$!B&N&X$nbA*Mpx|?%!k&@D_nG z{93?|KpJ?qtb>Jq%M$)xjbDY|&zvp%f_qRnC$4e=8!P)Cu>=8*wm)NK(AL9353o6^|AnIk!0`*qu7Ps@!1^D??_Qw* literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro b/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro new file mode 100644 index 0000000000..25923f70da --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro @@ -0,0 +1,6 @@ +# This tests an issue where some web servers don't +# include an appropriate ZLIB header on deflated +# content. +# +# @TEST-EXEC: bro -r $TRACES/http/missing-zlib-header.pcap %INPUT +# @TEST-EXEC: btest-diff http.log