From bf08770764f00d9d15246338464d8e3a0990c8d7 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 4 Mar 2015 13:31:13 -0600 Subject: [PATCH 01/52] Correct a minor typo in the docs --- src/strings.bif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/strings.bif b/src/strings.bif index b8d21cb04a..1f09667064 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -698,7 +698,7 @@ function split_n%(str: string, re: pattern, ## ## Returns: An array of strings where, if *incl_sep* is true, each two ## successive elements correspond to a substring in *str* of the part -## not matching *re* (event-indexed) and the part that matches *re* +## not matching *re* (even-indexed) and the part that matches *re* ## (odd-indexed). ## ## .. bro:see:: split_string split_string1 split_string_all str_split From 8841d0ae771b3d201ded29e921d8ef3fbe02fbaa Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 19 Mar 2015 16:01:28 -0500 Subject: [PATCH 02/52] Minor improvements to logging framework documentation --- scripts/base/frameworks/logging/main.bro | 51 ++++++++++--------- .../base/frameworks/logging/writers/ascii.bro | 14 ++--- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index d4d5c0244e..aecc552146 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -6,9 +6,10 @@ module Log; export { - ## Type that defines an ID unique to each log stream. Scripts creating new log - ## streams need to redef this enum to add their own specific log ID. The log ID - ## implicitly determines the default name of the generated log file. + ## Type that defines an ID unique to each log stream. Scripts creating new + ## log streams need to redef this enum to add their own specific log ID. + ## The log ID implicitly determines the default name of the generated log + ## file. type Log::ID: enum { ## Dummy place-holder. UNKNOWN @@ -20,25 +21,24 @@ export { ## If true, remote logging is by default enabled for all filters. const enable_remote_logging = T &redef; - ## Default writer to use if a filter does not specify - ## anything else. + ## Default writer to use if a filter does not specify anything else. const default_writer = WRITER_ASCII &redef; - ## Default separator between fields for logwriters. - ## Can be overwritten by individual writers. + ## Default separator to use between fields. + ## Individual writers can use a different value. const separator = "\t" &redef; - ## Separator between set elements. - ## Can be overwritten by individual writers. + ## Default separator to use between elements of a set. + ## Individual writers can use a different value. const set_separator = "," &redef; - ## String to use for empty fields. This should be different from - ## *unset_field* to make the output unambiguous. - ## Can be overwritten by individual writers. + ## Default string to use for empty fields. This should be different + ## from *unset_field* to make the output unambiguous. + ## Individual writers can use a different value. const empty_field = "(empty)" &redef; - ## String to use for an unset &optional field. - ## Can be overwritten by individual writers. + ## Default string to use for an unset &optional field. + ## Individual writers can use a different value. const unset_field = "-" &redef; ## Type defining the content of a logging stream. @@ -63,7 +63,7 @@ export { ## If no ``path`` is defined for the filter, then the first call ## to the function will contain an empty string. ## - ## rec: An instance of the streams's ``columns`` type with its + ## rec: An instance of the stream's ``columns`` type with its ## fields set to the values to be logged. ## ## Returns: The path to be used for the filter. @@ -81,7 +81,8 @@ export { terminating: bool; ##< True if rotation occured due to Bro shutting down. }; - ## Default rotation interval. Zero disables rotation. + ## Default rotation interval to use for filters that do not specify + ## an interval. Zero disables rotation. ## ## Note that this is overridden by the BroControl LogRotationInterval ## option. @@ -116,8 +117,8 @@ export { ## Indicates whether a log entry should be recorded. ## If not given, all entries are recorded. ## - ## rec: An instance of the streams's ``columns`` type with its - ## fields set to the values to logged. + ## rec: An instance of the stream's ``columns`` type with its + ## fields set to the values to be logged. ## ## Returns: True if the entry is to be recorded. pred: function(rec: any): bool &optional; @@ -125,10 +126,10 @@ export { ## Output path for recording entries matching this ## filter. ## - ## The specific interpretation of the string is up to - ## the used writer, and may for example be the destination + ## The specific interpretation of the string is up to the + ## logging writer, and may for example be the destination ## file name. Generally, filenames are expected to be given - ## without any extensions; writers will add appropiate + ## without any extensions; writers will add appropriate ## extensions automatically. ## ## If this path is found to conflict with another filter's @@ -153,7 +154,7 @@ export { ## then the first call to the function will contain an ## empty string. ## - ## rec: An instance of the streams's ``columns`` type with its + ## rec: An instance of the stream's ``columns`` type with its ## fields set to the values to be logged. ## ## Returns: The path to be used for the filter, which will be @@ -177,7 +178,7 @@ export { ## If true, entries are passed on to remote peers. log_remote: bool &default=enable_remote_logging; - ## Rotation interval. + ## Rotation interval. Zero disables rotation. interv: interval &default=default_rotation_interval; ## Callback function to trigger for rotated files. If not set, the @@ -207,9 +208,9 @@ export { ## Removes a logging stream completely, stopping all the threads. ## - ## id: The ID enum to be associated with the new logging stream. + ## id: The ID associated with the logging stream. ## - ## Returns: True if a new stream was successfully removed. + ## Returns: True if the stream was successfully removed. ## ## .. bro:see:: Log::create_stream global remove_stream: function(id: ID) : bool; diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 5f59229f7f..c10c86145e 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -1,15 +1,15 @@ ##! Interface for the ASCII log writer. Redefinable options are available ##! to tweak the output format of ASCII logs. ##! -##! The ASCII writer supports currently one writer-specific filter option via -##! ``config``: setting ``tsv`` to the string ``T`` turns the output into +##! The ASCII writer currently supports one writer-specific per-filter config +##! option: setting ``tsv`` to the string ``T`` turns the output into ##! "tab-separated-value" mode where only a single header row with the column ##! names is printed out as meta information, with no "# fields" prepended; no -##! other meta data gets included in that mode. +##! other meta data gets included in that mode. Example filter using this:: ##! -##! Example filter using this:: -##! -##! local my_filter: Log::Filter = [$name = "my-filter", $writer = Log::WRITER_ASCII, $config = table(["tsv"] = "T")]; +##! local f: Log::Filter = [$name = "my-filter", +##! $writer = Log::WRITER_ASCII, +##! $config = table(["tsv"] = "T")]; ##! module LogAscii; @@ -29,6 +29,8 @@ export { ## Format of timestamps when writing out JSON. By default, the JSON ## formatter will use double values for timestamps which represent the ## number of seconds from the UNIX epoch. + ## + ## This option is also available as a per-filter ``$config`` option. const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef; ## If true, include lines with log meta information such as column names From ea2ce67c5f2e7d80097b6264bb2c4db8faf9d2b5 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 18 May 2015 14:30:32 -0400 Subject: [PATCH 03/52] Fixes an issue with missing zlib headers on deflated HTTP content. - Includes a test. --- src/analyzer/protocol/zip/ZIP.cc | 82 ++++++++++++------ .../http.log | 11 +++ .../Traces/http/missing-zlib-header.pcap | Bin 0 -> 14844 bytes .../protocols/http/missing-zlib-header.bro | 6 ++ 4 files changed, 72 insertions(+), 27 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log create mode 100644 testing/btest/Traces/http/missing-zlib-header.pcap create mode 100644 testing/btest/scripts/base/protocols/http/missing-zlib-header.bro diff --git a/src/analyzer/protocol/zip/ZIP.cc b/src/analyzer/protocol/zip/ZIP.cc index 132515f29a..e71ba438b7 100644 --- a/src/analyzer/protocol/zip/ZIP.cc +++ b/src/analyzer/protocol/zip/ZIP.cc @@ -22,10 +22,9 @@ ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method) zip->next_in = 0; zip->avail_in = 0; - // "15" here means maximum compression. "32" is a gross overload - // hack that means "check it for whether it's a gzip file". Sheesh. - zip_status = inflateInit2(zip, 15 + 32); - if ( zip_status != Z_OK ) + // "32" is a gross overload hack that means "check it + // for whether it's a gzip file". Sheesh. + if ( inflateInit2(zip, MAX_WBITS+32) != Z_OK ) { Weird("inflate_init_failed"); delete zip; @@ -54,40 +53,69 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) return; static unsigned int unzip_size = 4096; - Bytef unzipbuf[unzip_size]; + int allow_restart = 1; + u_char *unzipbuf = new u_char[unzip_size]; + if ( unzipbuf == NULL ) + { + Weird("failed_to_allocate_deflate_buffer"); + return; + } zip->next_in = (Bytef*) data; zip->avail_in = len; - do + Bytef *orig_in = zip->next_in; + size_t nread = zip->avail_in; + + for(;;) { - zip->next_out = unzipbuf; + zip->next_out = (Bytef *)unzipbuf; zip->avail_out = unzip_size; zip_status = inflate(zip, Z_SYNC_FLUSH); + if ( zip_status == Z_STREAM_END || + zip_status == Z_OK ) + { + allow_restart = 0; - if ( zip_status != Z_STREAM_END && - zip_status != Z_OK && - zip_status != Z_BUF_ERROR ) + int have = unzip_size - zip->avail_out; + if ( have ) + ForwardStream(have, unzipbuf, IsOrig()); + + if ( zip_status == Z_STREAM_END ) + { + inflateEnd(zip); + delete unzipbuf; + return; + } + + if ( zip->avail_in == 0 ) + { + delete unzipbuf; + return; + } + } + else if ( allow_restart && zip_status == Z_DATA_ERROR ) + { + // Some servers seem to not generate zlib headers, + // so this is an attempt to fix and continue anyway. + inflateEnd(zip); + if ( inflateInit2(zip, -MAX_WBITS) != Z_OK ) + { + delete unzipbuf; + return; + } + + zip->next_in = orig_in; + zip->avail_in = nread; + allow_restart = 0; + continue; + } + else { Weird("inflate_failed"); - inflateEnd(zip); - break; + delete unzipbuf; + return; } - - int have = unzip_size - zip->avail_out; - if ( have ) - ForwardStream(have, unzipbuf, IsOrig()); - - if ( zip_status == Z_STREAM_END ) - { - inflateEnd(zip); - delete zip; - zip = 0; - break; - } - - zip_status = Z_OK; } - while ( zip->avail_out == 0 ); } diff --git a/testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log b/testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log new file mode 100644 index 0000000000..c4c96b7fb9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open 2015-05-12-16-26-53 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] +1232039472.314927 CXWv6p3arKYeMETxOg 237.244.174.255 1905 79.218.110.244 80 1 GET ads1.msn.com /library/dap.js http://zone.msn.com/en/root/default.htm Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727) 0 13249 200 OK - - - (empty) - - - - - FBcNS3RwceOxW15xg text/plain +1232039472.446194 CXWv6p3arKYeMETxOg 237.244.174.255 1905 79.218.110.244 80 2 GET ads1.msn.com /library/dap.js http://zone.msn.com/en/root/default.htm Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727) 0 13249 200 OK - - - (empty) - - - - - FDWU85N0DpedJPh93 text/plain +#close 2015-05-12-16-26-53 diff --git a/testing/btest/Traces/http/missing-zlib-header.pcap b/testing/btest/Traces/http/missing-zlib-header.pcap new file mode 100644 index 0000000000000000000000000000000000000000..66406a9a4022f112bf4873b462bb30d4fe4f862f GIT binary patch literal 14844 zcmeHNc{r49-=0KSQnqX@rjVq;jD0MFv9FW0Ni_yz3!|}R86=TZ6GDg)g`#9B6{4c- zV@Vz@^w>t?QDew3`0knK?S1Qcp3i^ZcO36=xR1FUX6Cxi>v!Jw?>evLoHs8kZ*qV* zK|ijwH4tbM@J2ZRZ$3ca1nB|)vkLp-1nm^#i^6h)jDZTJ@%kX*mW9|E`u#QAcLDUR zr$MN%lpA8Knt>epAiHW1h=Y@>debIO4i?(E+V#-Ttss!fmR=ne5(^Q8+Io78hA;#W z#RG*zyLIw@MGLFv1nB@hu?qY8IktnP=h?9({v|eX01=52FbdRu@uL|lGNmp~kPQcj zUH*#2p`-%AX8%pu#%sT8v#oQSMtCHD*6V2%^Vz1N>D{8Kd*ti zJ1#I74nCxGh@aOE=ZPcW2yn1>aBv`8Nhty!fK&7j3Q%;%`zzrBln8iyu#yMP6C2_e ztmqx=|D!p?3LD@Rg7w0|!MFfONDx1-qX7hoa)g7=XsZ9al5v1L-oq!r3l8>*@CiH& z27vsq!8o9D5RL#b@WKTEaaiLceEj^dN~(&=;DbQ7f!JUlH$R*<*xKIQ7_6bFtPOVd z3Gl#&1%YiGz-o$6ZLp#Zzz}I=2ZkytE2?NHtNrp3R8dt01_RpTJbbWVB+v-)^8CDJ_#hw8fg|slZfW$}kT#4OI;l6-^CIbu|sFvYM7ER11gm&;}zN ztsETfjCGA|AddDxY^+vVDwjo1-fb0mu1fABERWk%$KrsSd{Ji|UEQ;GO>r!lI zQ#AYsif7pr@BCoH!ioa0rno@t@>hynAJ82vtd7h78x}MZ1o|=Xz%XTHu&pJKKv;Kg z90W+OU;^F`4)({MfdJ!37pei%R95EaH39|*9PHp7au^I%16yDNSPiwnPz^W~3WvhL zrq&MZX5FQBNU;_jFeB6N%g%40V1M$U%VuRcXK7qmfyjIwt zV2Cv^JUxAI9zQzN0$YTzJJbYvg{x`7VJd%i_@`(r6;f`gGLT|-BEVCdR`k+wE?u!*@1*xbno54N?j0^0(gqRh}>djo5{gRvDJ{2yxk zyvAn&eF&@@^|$8;gThr|K(_z-9IbExUcugQuo_fFixrcz9mLw!$lSynY2aXPYXgMx zuc!KF#b4Fw6ic13f2tEd?@awx*WKnBvIms>+ahBu<}C(31>fY3brqKR?W(@O^z#dy z6RFdCd`zX~OAnoG!#Uwjc#7(G%;eU6ABvo<|MSmN?OYqV@55_|y%|@m*otA$gr%+KJ5!C$x3dqmqo# z@c!y{zZBB@sqm2W0XcLU{-cCgJ9_nx_PKkc*SOPJo0+e}q6a9C!e(U}u0P+#KTtgs zYvol}6JDtD$)%1!{hImP6M1KH*7quO)v3>pqG|Bba)jrIM_;R0Gp)8Nban8LV(p2- z(ZRH`)djD}$cb~s&h6Byff+9N#zun5QVdch-yA=_N&sUznLcRNhs?Jw3ZL`1(}#n>n{Eie8b;-BvIE`mUam0FmyV z?L-42ap9X1XAzmmY!!`mJ461eq$*S}tz=G+)IBEv_e(e{1v)EsNA~KJYg6nUWOX%SVMJUyg! z!?ACk19PI^I0sK;KK`Wc16`gjBq6YxopMq!*VO0fRisMuhXGR7ZGMb}Li*jtG5f+f z1=Z`+=H|Nb2I^wYbE~@%83SZuH4;=5;+WA)8>u2?^qZQJW-Uxjn<7NFuHItg(U(># zR_cz8x8EbZW1ff%8&Yf|nMn0B?sgP0Ol!^zZrg_6tD8q1oSr=GQjLoa)~W6I^2nP| z9U0}U7Zv54ZFP4rgv5)B9#X7aDmY5J{?2!N6&_)QWG<$Wk{xs6F6#-$$G~elCi#}= zh0+gPDxxetTy3aRueN-xO_QPx&>o5wr``farAsicM$5`1NFsm111by>dB4us_GNNsIxfh(Zq z+BDx#Ys8t#3u>1pkAghaXS8+iHoeXe>N{Dca&=Oh{`nN+T13Gtmq6HeOCih6BbP^* z*$2Kj4v8RIYAz0Hrz#7z46EhjIYwI0W>13FFZn}cj-EB0~*BX1_x^_xc!i0a53ZTFP^IR!aL zh0~!I)S!bUbESp3m_7{fwo8 z-`^l7T6ZPG5#>W!p5mC4hO%r*rRDp{QDghDCbFs$bB_kjQx$bbda1E~dfxTqM3JW- z%;VbZ86uo_Reng|Nin3v({&6&}-SCZ~oBR zqZPWmyM{?>4+CenTH|APsjseRUxktqTBNXDeaoZEZBDsj_$ja5bh=Mj#h&NI27SB0 zk(bM9=8+N?Ibv>!zYiUYj^0vKd2jL@>5tt+lUFZQ_a!5(1oaP<@V{hak$Il|wi0Yt zy_CLLzgO7JP|91n<;KAD6=l5@gA-AnXwtKNd*`$B$Z481DJr$8^qCGkX{+Co;fD(8 zfu*#~bCt4HJGc1`AN@9>KC#oRRiN>N_vzL59nGBrpZCO$*`6_~-8XuJEKF&|Z0Rhd zghtv&OWA|TgHDlGiK~}>XRc0*-bmTUEX&Q=@kC>@^>9=a1*}TR9$u3`3s(A0+s!31 zH>pWe^49WFOkw_vzQ*4K(ibDA5{9yh-r)1CgHIJ(Az3^58Px-Tz`(pH))Bf(ze)Hn-fA-Oka#~jXq&cH>({U{ zkxX@w!cqNCB3fI8SG+w08-mQvNsGm!RxU+Llcys$Ee&sV{a(-XT$w-;0+J3`>Fk%E zR%n^hb9iANN$h*-b)ZlrzU36Hv3sUb0h>eV{g`_;!M2oMJ6F)5jUx5l6nl2Os5s=n@{;|F8d)-~9&6se%{JNfVk)z4N43?|f!AO0KlloRDs5X<_0buQ{7!Mc*GLnQszlQcm=c=qioid%!F7bfiMv zp*i|!6pbe&gsW(%xBQuNRLqA5qQmbbW`-EkRU~^4$cGy86O1(y7T^c1!<$(@xGcm8 zEwa@sn(p@U@K3``U*$uVxX_d4x3m!7z`US$O;2wriNvk0)9v7rTNhpb{?vT-F^OuurHiWq!3QU` z&;#)f%h@6Kn|W1Cznbx-W{i1qp8hy-?P)>iu7(e7rtKl2>zDCipMgnzL&(i&EBR9`fzy{lj$Sy8DMN_AHY8>nvhX z`ePO;(E@=OMVZ=odpF81!DkaGHd9pfq4XwEJ)E$qQ+nWdL%ow?UiO>*ooMm7%f?}g z{4gR@P;J~q1d@01(Xi-5@eb(k2WY1>1$|dXG_N*^LuAHQ{LC?yc@BC-8Q1cgEcY+b zE22E)NE{vL(@7;FYo;Yyhpx!S6*atLO!~t2M?F(Tt{QiGHIxW`Y*Zq?T5H?t5hyRT zcB!#t*!jYBh6XJ3F6!dM^@#cogu<2a&Gb2m)fB{gEga{Bqt^1Jv1^DY+0mhD>6o)E zW5tMhjSRTrT4Ufmg+eb5 ze%nDRPNSyTX^&94$@zgWy~=YewH=J_BebLo0Ffx@@PLF1W z97!QQxzf;GM)OHYzZN@z_`-BMA*9RCIa05r82`22`QwKmguca+7z(p{ZRlG2>Ix=m zzS*l*d(7hZD)jOp#A}yr-{iyjtFP<7AazJs%h!a5EKSqrYhu>=sIQeUf!9&_eU)*_enZnt#+E4`o*LpA{>qbj(Yq?#>XC1idmDJTP+1*Oq%X zrR!bPX_MLdv{$>!P|4C4lUnX8mzzgh7`3g%V^FO$gx6Ub!u1J^H{nxznD4}16-Dve zsLM~eG6plP6tXXjW~C9$!(+aEliS2CF?4}}tFE_PiUq&(k({{%YR%kLv7qBrSO}g& z<-DnHT+L&4E4Dm{_DReiE_>S$m`jyV5`JA;!;iUEl^GN*zNUcHx_{GQrC)lex4 zL<(oy!fYE9N>{x@KOB|VVGr;ici!u z1>(YV|2MH>sU9~seKnzmVJVK5&YjJ(7fGiW-S{;;+laa1TYU`=E^4YJMt~b&K z65QTdJTr*%%!;9_AV!HWCC{oGmC0AWWX(j5f{Ira~J$b?48+u(ZS@Z~2rjG^Qf5jF~$kvJ@Ccd!3`dJJZg4Rx$Br(2*SF zELrmDk*Or>Gp&hZZBEYkDZfF~1QcUaE_*?y&F+o)Ce!8k)B?48RYyhz_2wnW+`I2z z@7t;eyKB>Q)NsnI^y$tJbUGq>DXsF6&8s)x?79UI-P?nBFWO)8gDBkNjD8XzFC%UB zrp7}$I{GQI}vw<&K;%Na`S`W^#2fllOtAPUq2m=agopzw(<}Bs{AMn zTlvK$z;{PLpii-D(EUPIfxZ>TC)8jSZ_>$!R~LoDceMBF%sy*6k_UQWayhER*}Js& zEyMRpx%i^1;vhv*%yf}NQ2KE8_c(X3IUj*WY{KCPMhFyKB!vsswW$H`;)0>_m3AT%0JlS+kA}vC8 zwc#@NnzHP}#B7M)2L*=}y+|tR|{cBT5X+7zMFsJK@lYPI1KMT=6Ay|?i z!=(~RKa=NGRruP!Cxg-R?UIP2i)zPi(rjBIu(1?AgVe1!G&dKT(qBz`KBDh)a#kty zyxy|6VNz4_(_>0r@B8kSO_|<%XeeH|X9VPbB^Fuj#Bux!dZ+k;|(p^8$`+wNDS z!AeCXGBPYi#XC!j&P%MG$lYzWYd^Px1J!;nZc&gw6JAm0DpB?m0xtt>d$|@5+1atEcAL-LHfl>#n7!Y1nDZKTe;hI+p69 zQX6oL=3f-#W)X3IwFKSko4O_Y!S_uu9tmb0pYppEZq@Wcgk&W-_vJ5LIBqk=RD;_h zmJhFRKlz;cmQ(L(nk0wdxh)wE^3BQqGI?k1xLY~LOyw(KS3kRoggt*xKPNjfiadR| zFJZFP&6o)%M)Ds2cxgQIhO1-`B~#5>yQt3fHb(WDJ%`Hb{R!<=d2{(RWgYvRM`?f7Z2eYT4ef5Ijx^ zcGcSfO3+K%Vb=itd?XIOnA@AFdVml1-FSwJ51F%xcjO-OCCF!ZL_qpVi^!X@K-Y1o zwD|o#4o*vSu64OVpM0>S9@e;__se2uoVFoTf9i^`p-1Dv8MCtmI~s%c|Mqn`CrWP{ zV&N&V{$4pg-k>c6UOb+Iu^NllBHF%Z_Q(x!BL?f|T1`F5LT;Q0verHNuJ6`fio>Rk zE>maeF(0t0)#*Sthv+)NT@m&|Jae|g$8`^sU_{zyQ=QGoj}z!nO|#z71V}DdzHP-j z!*b>-U7=Mi&XbT%_DZYcL3UJu+|-+dOgUoCm_>Ky!Qh1(#|8^HY()flU|bY6qu~iYpR)ST2En z(dqPwC$z6!d>zb{c&fLo>fPW1gcn3r#$89z>_8_eW0&&Jm{F_tsNhvIHaU+ zeu$MYlS|TXpQ(PkeK>r#Mls#k+yt%D{57bu;6^NPiWU`|*HnA(`XacWy1FM*{^abAdCw~{gyc!xmzGijoLkCZl}rc9%3Y&;mwFJYDX0Y zX`;m$5|DmtIWl!-a2j2&i3|!)lAGC96nAWjd86}!mp7I)-J5mdbz`D4nV9+6EdSXp zr5WmCB&{!T(ST;eEojT#&yYUGp^a%PC&w&2VaDyO8lWlcTas9jx)6KmV;1g^xI82S z&*QQ`?*8emZ}Jm<=e{R@ooMpKrEbA?)9Bk`Y2kx&lgSd}4tH+r-+IM);()&1bHipO zMhx3qtbMxfl0S>R_M3h{hX;u&$h1T-~G)c z8tu#f}DQS4z;{EIcurZ~l>c=>0F-2m1){#`G76_)r9w~LJ>61m~u z0lyXax6QcW-+?#$yAA(t!@v74@bCK8SN=Y>@+bVN{4No|M9EhEO7Mn%x8dJy_;(xr z-G+bn-{#+Gaj(1Pf5TS(^S>%TR`G}OC-2$t+%`P74bN@EbKCIT{@Xma??UUH9F=}! zALH)&%YoL^8V#$+I>z1k!>MB*3D1bFi$!B&N&X$nbA*Mpx|?%!k&@D_nG z{93?|KpJ?qtb>Jq%M$)xjbDY|&zvp%f_qRnC$4e=8!P)Cu>=8*wm)NK(AL9353o6^|AnIk!0`*qu7Ps@!1^D??_Qw* literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro b/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro new file mode 100644 index 0000000000..25923f70da --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro @@ -0,0 +1,6 @@ +# This tests an issue where some web servers don't +# include an appropriate ZLIB header on deflated +# content. +# +# @TEST-EXEC: bro -r $TRACES/http/missing-zlib-header.pcap %INPUT +# @TEST-EXEC: btest-diff http.log From c870fefbef72eeec1c673c7d1fd4ed24fb03676a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 May 2015 13:00:58 -0500 Subject: [PATCH 04/52] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index b02fefd5cf..8fc6938017 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit b02fefd5cf78c1576e59c106f5211ce5ae47cfdd +Subproject commit 8fc6938017dc15acfb26fa29e6ad0933019781c5 From 08822e0dd43df88fcbe719c0092a2787cfc1e339 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 22 May 2015 11:46:50 -0500 Subject: [PATCH 05/52] Allow '<' and '>' in MIME multipart boundaries. The spec doesn't actually seem to permit these, but Seth had a (private) pcap showing them used in the wild (and the HTTP/MIME analyzer failed to parse content as a result). --- src/analyzer/protocol/http/HTTP.cc | 7 +++- src/analyzer/protocol/mime/MIME.cc | 66 +++++++++++++++--------------- src/analyzer/protocol/mime/MIME.h | 7 ++-- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index f60d1372ac..ff72c6f350 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -1025,8 +1025,11 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) } else { - ProtocolViolation("not a http reply line"); - reply_state = EXPECT_REPLY_NOTHING; + if ( line != end_of_line ) + { + ProtocolViolation("not a http reply line"); + reply_state = EXPECT_REPLY_NOTHING; + } } break; diff --git a/src/analyzer/protocol/mime/MIME.cc b/src/analyzer/protocol/mime/MIME.cc index a1759d97d0..17630f0501 100644 --- a/src/analyzer/protocol/mime/MIME.cc +++ b/src/analyzer/protocol/mime/MIME.cc @@ -246,11 +246,16 @@ int MIME_get_field_name(int len, const char* data, data_chunk_t* name) } // See RFC 2045, page 12. -int MIME_is_tspecial (char ch) +int MIME_is_tspecial (char ch, bool is_boundary = false) { - return ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == '@' || - ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' || - ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '='; + if ( is_boundary ) + return ch == '(' || ch == ')' || ch == '@' || + ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' || + ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '='; + else + return ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == '@' || + ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' || + ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '='; } int MIME_is_field_name_char (char ch) @@ -258,26 +263,27 @@ int MIME_is_field_name_char (char ch) return ch >= 33 && ch <= 126 && ch != ':'; } -int MIME_is_token_char (char ch) +int MIME_is_token_char (char ch, bool is_boundary = false) { - return ch >= 33 && ch <= 126 && ! MIME_is_tspecial(ch); + return ch >= 33 && ch <= 126 && ! MIME_is_tspecial(ch, is_boundary); } // See RFC 2045, page 12. // A token is composed of characters that are not SPACE, CTLs or tspecials -int MIME_get_token(int len, const char* data, data_chunk_t* token) +int MIME_get_token(int len, const char* data, data_chunk_t* token, + bool is_boundary) { int i = MIME_skip_lws_comments(len, data); while ( i < len ) { int j; - if ( MIME_is_token_char(data[i]) ) + if ( MIME_is_token_char(data[i], is_boundary) ) { token->data = (data + i); for ( j = i; j < len; ++j ) { - if ( ! MIME_is_token_char(data[j]) ) + if ( ! MIME_is_token_char(data[j], is_boundary) ) break; } @@ -359,7 +365,7 @@ int MIME_get_quoted_string(int len, const char* data, data_chunk_t* str) return -1; } -int MIME_get_value(int len, const char* data, BroString*& buf) +int MIME_get_value(int len, const char* data, BroString*& buf, bool is_boundary) { int offset = MIME_skip_lws_comments(len, data); @@ -380,7 +386,7 @@ int MIME_get_value(int len, const char* data, BroString*& buf) else { data_chunk_t str; - int end = MIME_get_token(len, data, &str); + int end = MIME_get_token(len, data, &str, is_boundary); if ( end < 0 ) return -1; @@ -863,8 +869,22 @@ int MIME_Entity::ParseFieldParameters(int len, const char* data) len -= offset; BroString* val = 0; - // token or quoted-string - offset = MIME_get_value(len, data, val); + + if ( current_field_type == MIME_CONTENT_TYPE && + content_type == CONTENT_TYPE_MULTIPART && + strcasecmp_n(attr, "boundary") == 0 ) + { + // token or quoted-string (and some lenience for characters + // not explicitly allowed by the RFC, but encountered in the wild) + offset = MIME_get_value(len, data, val, true); + data_chunk_t vd = get_data_chunk(val); + multipart_boundary = new BroString((const u_char*)vd.data, + vd.length, 1); + } + else + // token or quoted-string + offset = MIME_get_value(len, data, val); + if ( offset < 0 ) { IllegalFormat("value not found in parameter specification"); @@ -874,8 +894,6 @@ int MIME_Entity::ParseFieldParameters(int len, const char* data) data += offset; len -= offset; - - ParseParameter(attr, get_data_chunk(val)); delete val; } @@ -920,24 +938,6 @@ void MIME_Entity::ParseContentEncoding(data_chunk_t encoding_mechanism) content_encoding = i; } -void MIME_Entity::ParseParameter(data_chunk_t attr, data_chunk_t val) - { - switch ( current_field_type ) { - case MIME_CONTENT_TYPE: - if ( content_type == CONTENT_TYPE_MULTIPART && - strcasecmp_n(attr, "boundary") == 0 ) - multipart_boundary = new BroString((const u_char*)val.data, val.length, 1); - break; - - case MIME_CONTENT_TRANSFER_ENCODING: - break; - - default: - break; - } - } - - int MIME_Entity::CheckBoundaryDelimiter(int len, const char* data) { if ( ! multipart_boundary ) diff --git a/src/analyzer/protocol/mime/MIME.h b/src/analyzer/protocol/mime/MIME.h index a3ee45d071..46c96658fb 100644 --- a/src/analyzer/protocol/mime/MIME.h +++ b/src/analyzer/protocol/mime/MIME.h @@ -117,7 +117,6 @@ protected: void ParseContentType(data_chunk_t type, data_chunk_t sub_type); void ParseContentEncoding(data_chunk_t encoding_mechanism); - void ParseParameter(data_chunk_t attr, data_chunk_t val); void BeginBody(); void NewDataLine(int len, const char* data, int trailing_CRLF); @@ -275,9 +274,11 @@ extern int MIME_count_leading_lws(int len, const char* data); extern int MIME_count_trailing_lws(int len, const char* data); extern int MIME_skip_comments(int len, const char* data); extern int MIME_skip_lws_comments(int len, const char* data); -extern int MIME_get_token(int len, const char* data, data_chunk_t* token); +extern int MIME_get_token(int len, const char* data, data_chunk_t* token, + bool is_boundary = false); extern int MIME_get_slash_token_pair(int len, const char* data, data_chunk_t* first, data_chunk_t* second); -extern int MIME_get_value(int len, const char* data, BroString*& buf); +extern int MIME_get_value(int len, const char* data, BroString*& buf, + bool is_boundary = false); extern int MIME_get_field_name(int len, const char* data, data_chunk_t* name); extern BroString* MIME_decode_quoted_pairs(data_chunk_t buf); From fcaf1d9c95e8ee0c89003cf380a3f3f5e2deaa1a Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 25 May 2015 13:08:03 -0500 Subject: [PATCH 06/52] Update install documentation and fix some typos --- doc/install/guidelines.rst | 10 ++++++---- doc/install/install.rst | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/install/guidelines.rst b/doc/install/guidelines.rst index af33b8fee1..d1e1777165 100644 --- a/doc/install/guidelines.rst +++ b/doc/install/guidelines.rst @@ -8,10 +8,12 @@ How to Upgrade If you're doing an upgrade install (rather than a fresh install), there's two suggested approaches: either install Bro using the same installation prefix directory as before, or pick a new prefix and copy -local customizations over. Regardless of which approach you choose, -if you are using BroControl, then after upgrading Bro you will need to -run "broctl check" (to verify that your new configuration is OK) -and "broctl install" to complete the upgrade process. +local customizations over. + +Regardless of which approach you choose, if you are using BroControl, then +before doing the upgrade you should stop all running Bro processes with the +"broctl stop" command. After the upgrade is complete then you will need +to run "broctl deploy". In the following we summarize general guidelines for upgrading, see the :ref:`release-notes` for version-specific information. diff --git a/doc/install/install.rst b/doc/install/install.rst index 18bacd7758..ab3f6cafc8 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -46,8 +46,7 @@ To build Bro from source, the following additional dependencies are required: * zlib headers * Perl -To install the required dependencies, you can use (when done, make sure -that ``bash`` and ``python`` are in your ``PATH``): +To install the required dependencies, you can use: * RPM/RedHat-based Linux: @@ -68,13 +67,17 @@ that ``bash`` and ``python`` are in your ``PATH``): .. console:: - sudo pkg_add -r bash cmake swig bison python perl py27-sqlite3 + sudo pkg install bash cmake swig bison python perl py27-sqlite3 + + Note that in older versions of FreeBSD, you might have to use the + "pkg_add -r" command instead of "pkg install". * Mac OS X: - Compiling source code on Macs requires first downloading Xcode_, - then going through its "Preferences..." -> "Downloads" menus to - install the "Command Line Tools" component. + Compiling source code on Macs requires first installing Xcode_ (in older + versions of Xcode, you would then need to go through its + "Preferences..." -> "Downloads" menus to install the "Command Line Tools" + component). OS X comes with all required dependencies except for CMake_ and SWIG_. Distributions of these dependencies can likely be obtained from your @@ -94,7 +97,6 @@ build time: * curl (used by a Bro script that implements active HTTP) * gperftools (tcmalloc is used to improve memory and CPU usage) * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump) - * Ruby executable, library, and headers (for Broccoli Ruby bindings) LibGeoIP is probably the most interesting and can be installed on most platforms by following the instructions for :ref:`installing @@ -119,8 +121,8 @@ platforms for binary releases and for installation instructions. Linux based binary installations are usually performed by adding information about the Bro packages to the respective system packaging - tool. Theen the usual system utilities such as ``apt``, ``yum`` - or ``zyppper`` are used to perforn the installation. By default, + tool. Then the usual system utilities such as ``apt``, ``yum`` + or ``zypper`` are used to perform the installation. By default, installations of binary packages will go into ``/opt/bro``. * MacOS Disk Image with Installer @@ -131,7 +133,7 @@ platforms for binary releases and for installation instructions. The primary install prefix for binary packages is ``/opt/bro``. Installing from Source -========================== +====================== Bro releases are bundled into source packages for convenience and are available on the `bro downloads page`_. Alternatively, the latest From 5147b0bb02588f223cf04fac2ac3c3d9a7640217 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 27 May 2015 12:22:18 -0700 Subject: [PATCH 07/52] set fedora 21 specific environment variable to not make it complain about md5 signed certs. Addresses BIT-1402 --- testing/external/subdir-btest.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index f79c432445..b2b36bf84f 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -19,3 +19,5 @@ DIST=%(testbase)s/../../.. BUILD=%(testbase)s/../../../build BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX BRO_DNS_FAKE=1 +# For fedora 21 - they disable MD5 for certificate verification and need setting an environment variable to permit it. +OPENSSL_ENABLE_MD5_VERIFY=1 From e02ad1711c1b76bb02c1cc7051e16e8128e2c344 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 27 May 2015 16:23:02 -0500 Subject: [PATCH 08/52] Add link to broctl doc from the quickstart doc --- doc/quickstart/index.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/quickstart/index.rst b/doc/quickstart/index.rst index 616c94c261..ba9896e19d 100644 --- a/doc/quickstart/index.rst +++ b/doc/quickstart/index.rst @@ -24,9 +24,10 @@ Managing Bro with BroControl BroControl is an interactive shell for easily operating/managing Bro installations on a single system or even across multiple systems in a traffic-monitoring cluster. This section explains how to use BroControl -to manage a stand-alone Bro installation. For instructions on how to -configure a Bro cluster, see the :doc:`Cluster Configuration -<../configuration/index>` documentation. +to manage a stand-alone Bro installation. For a complete reference on +BroControl, see the :doc:`BroControl <../components/broctl/README>` +documentation. For instructions on how to configure a Bro cluster, +see the :doc:`Cluster Configuration <../configuration/index>` documentation. A Minimal Starting Configuration -------------------------------- From b386b2ba5122ac4d51f586f15a1793621fa1a5ad Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Thu, 28 May 2015 12:11:06 +0200 Subject: [PATCH 09/52] BIT-1314: Add detection for Quantum Insert attacks TCP_Reassembler can now keep a history of old TCP segments using the `tcp_max_old_segments` option. A value of zero will disable it. An overlapping segment with different data can indicate a possible TCP injection attack. The rexmit_inconsistency event will fire if this is the case. --- src/NetVar.cc | 2 + src/NetVar.h | 1 + src/Reassem.cc | 70 +++++++++++++++++++- src/Reassem.h | 12 ++++ src/analyzer/protocol/tcp/TCP_Reassembler.cc | 3 + 5 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/NetVar.cc b/src/NetVar.cc index 123e947701..dc049005bb 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -49,6 +49,7 @@ double tcp_partial_close_delay; int tcp_max_initial_window; int tcp_max_above_hole_without_any_acks; int tcp_excessive_data_without_further_acks; +int tcp_max_old_segments; RecordType* socks_address; @@ -354,6 +355,7 @@ void init_net_var() opt_internal_int("tcp_max_above_hole_without_any_acks"); tcp_excessive_data_without_further_acks = opt_internal_int("tcp_excessive_data_without_further_acks"); + tcp_max_old_segments = opt_internal_int("tcp_max_old_segments"); socks_address = internal_type("SOCKS::Address")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index bf2d9a5712..efadaaad6d 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -52,6 +52,7 @@ extern double tcp_reset_delay; extern int tcp_max_initial_window; extern int tcp_max_above_hole_without_any_acks; extern int tcp_excessive_data_without_further_acks; +extern int tcp_max_old_segments; extern RecordType* socks_address; diff --git a/src/Reassem.cc b/src/Reassem.cc index 8bf965427b..bf2ed5df02 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -34,12 +34,38 @@ uint64 Reassembler::total_size = 0; Reassembler::Reassembler(uint64 init_seq) { blocks = last_block = 0; + old_blocks = last_old_block = 0; + total_old_blocks = max_old_blocks = 0; trim_seq = last_reassem_seq = init_seq; } Reassembler::~Reassembler() { ClearBlocks(); + ClearOldBlocks(); + } + +void Reassembler::CheckOverlap(DataBlock *head, DataBlock *tail, + uint64 seq, uint64 len, const u_char* data) + { + if ( ! head || ! tail ) + return; + + if ( seq_between(seq, head->seq, tail->upper) ) + { + for ( DataBlock* b = head; b; b = b->next ) + { + if ( seq_between(seq, b->seq, b->upper) ) + { + uint64 overlap_start = seq; + uint64 overlap_offset = overlap_start - b->seq; + uint64 new_b_len = len; + uint64 b_len = b->upper - overlap_start; + uint64 overlap_len = min(new_b_len, b_len); + Overlap(&b->block[overlap_offset], data, overlap_len); + } + } + } } void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data) @@ -49,6 +75,9 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data) uint64 upper_seq = seq + len; + CheckOverlap( blocks, last_block, seq, len, data ); + CheckOverlap( old_blocks, last_old_block, seq, len, data ); + if ( upper_seq <= trim_seq ) // Old data, don't do any work for it. return; @@ -119,7 +148,33 @@ uint64 Reassembler::TrimToSeq(uint64 seq) num_missing += seq - blocks->upper; } - delete blocks; + if (max_old_blocks) + { + blocks->next = 0; + if (last_old_block) + { + blocks->prev = last_old_block; + last_old_block->next = blocks; + } + else + { + blocks->prev = 0; + old_blocks = blocks; + } + + last_old_block = blocks; + total_old_blocks++; + + while (old_blocks && total_old_blocks > max_old_blocks) + { + DataBlock* next = old_blocks->next; + delete old_blocks; + old_blocks = next; + total_old_blocks--; + } + } + else + delete blocks; blocks = b; } @@ -156,6 +211,19 @@ void Reassembler::ClearBlocks() last_block = 0; } +void Reassembler::ClearOldBlocks() + { + while ( old_blocks ) + { + DataBlock* b = old_blocks->next; + delete old_blocks; + old_blocks = b; + } + + last_old_block = 0; + } + + uint64 Reassembler::TotalSize() const { uint64 size = 0; diff --git a/src/Reassem.h b/src/Reassem.h index 39617f7816..e55c809990 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -36,6 +36,7 @@ public: // Delete all held blocks. void ClearBlocks(); + void ClearOldBlocks(); int HasBlocks() const { return blocks != 0; } uint64 LastReassemSeq() const { return last_reassem_seq; } @@ -50,6 +51,8 @@ public: // Sum over all data buffered in some reassembler. static uint64 TotalMemoryAllocation() { return total_size; } + void SetMaxOldBlocks(uint32 count) { max_old_blocks = count; } + protected: Reassembler() { } @@ -65,10 +68,19 @@ protected: DataBlock* AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, const u_char* data); + void CheckOverlap(DataBlock *head, DataBlock *tail, + uint64 seq, uint64 len, const u_char* data); + DataBlock* blocks; DataBlock* last_block; + + DataBlock* old_blocks; + DataBlock* last_old_block; + uint64 last_reassem_seq; uint64 trim_seq; // how far we've trimmed + uint32 max_old_blocks; + uint32 total_old_blocks; static uint64 total_size; }; diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 16bb9cc56d..bbcd9cb43a 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -42,6 +42,9 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, seq_to_skip = 0; in_delivery = false; + if ( tcp_max_old_segments ) + SetMaxOldBlocks(tcp_max_old_segments); + if ( tcp_contents ) { // Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT); From 2aa214d8358fe583bd56920545e2057565ab6500 Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Thu, 28 May 2015 12:12:22 +0200 Subject: [PATCH 10/52] BIT-1314: Added QI test for rexmit_inconsistency --- .../Baseline/core.tcp.quantum-insert/.stdout | 4 ++++ .../tcp/qi_internet_SYNACK_curl_jsonip.pcap | Bin 0 -> 1718 bytes testing/btest/core/tcp/quantum-insert.bro | 12 ++++++++++++ 3 files changed, 16 insertions(+) create mode 100644 testing/btest/Baseline/core.tcp.quantum-insert/.stdout create mode 100644 testing/btest/Traces/tcp/qi_internet_SYNACK_curl_jsonip.pcap create mode 100644 testing/btest/core/tcp/quantum-insert.bro diff --git a/testing/btest/Baseline/core.tcp.quantum-insert/.stdout b/testing/btest/Baseline/core.tcp.quantum-insert/.stdout new file mode 100644 index 0000000000..3f2c5fad1b --- /dev/null +++ b/testing/btest/Baseline/core.tcp.quantum-insert/.stdout @@ -0,0 +1,4 @@ +----- rexmit_inconsistency ----- +1429652006.683290 c: [orig_h=178.200.100.200, orig_p=39976/tcp, resp_h=96.126.98.124, resp_p=80/tcp] +1429652006.683290 t1: HTTP/1.1 200 OK\x0d\x0aContent-Length: 5\x0d\x0a\x0d\x0aBANG! +1429652006.683290 t2: HTTP/1.1 200 OK\x0d\x0aServer: nginx/1.4.4\x0d\x0aDate: diff --git a/testing/btest/Traces/tcp/qi_internet_SYNACK_curl_jsonip.pcap b/testing/btest/Traces/tcp/qi_internet_SYNACK_curl_jsonip.pcap new file mode 100644 index 0000000000000000000000000000000000000000..d906d9cac9dbccaface027bfe7feeeff22f9c742 GIT binary patch literal 1718 zcmah}O>7fK6rOc{u%}%JsA{7Us=-1KPZKP1q&)yPi}s2Xf`pI|qLq3Jszh&Xih7B9=mGkkS9a_*q$@pn)^GNE z-+SLT^Xr4Viy_iU>d&i-gb47mdwF>F$3lc0gEa=}S$P!;lg)46sfNR3jF9xw_i2*u zzH#O5qx_>!FU-wf>nE8#sdFp1xf)87WrL7VXV*`H&>0dsI*1^I>5JsR1m+R_=J^OY z0&5H(*RN4SKL7>DaQDeLB4Y)Txz>N_>!nLj;n~e?pRulYeDuyxnC`lvI*1HaHX-sq z-$uwNfMT@qR;OI#z3pLu_cK7hywH;-96qXZMPx+$?rDTzW3}djlX#Lb5OGWLwTQiG z^2w7{#PP8#Q`kf{n^EF&T%@xucchU5H$9Ez%1%j1$_Z7Dv3?_w8kWlDDbqS*N|w#d zIX6Ef5A9_oW3GAQG^A!`jsjXt9@NWc``p;e54EE8k~y^-p^^-DmO%P32xmyxG`5K?c94aQ<@dQzl^ zEYpMLq@&y{ctwp3VXe`TW8*uq;QPIF1GRp>jkEpcYU^z0%b-?uVQZS?_qWa#5%(T! zLaaHq$yw^!c63%{M|%tN<%J%y`a{M?1nsY*s}OPdcoSk!f7Od1$G?;}alAL#;CL9P zTI>2WCD&n?7=bYp;p$o+a`cv7%dwZNI(@Ai+eSXaoeFm}W)=+dbC6$^Rgr$Adt77L zG9O@xIGfTPP$)jc;;NQNYU(f>pUSR_l|5_2X5F?+Mo#w(%T&-f`&q82J1+M=EPE%U zlt{CVZn`JHzhh?3${S_@EL;T3@_Y@-2xH z=oZH+Nh76_bw-+Y3}^uy`iA@z_lj2D)fgNqw3V6^>A9$3N401?nUcYRavTDFqXSWW z&MJG5QT)X~G~-x1;l1d2wx%cr?yVn==y{RG>WAXtmz2PjRExG7rGIuD^`|d``WG)e z-<^T0b@B?Tjt?sD-f8kuyE}K(+_}9~`TNiJq~+W3nZ6kJv{zZ0u zlL2;>2D`6(Za%v|U()qEKgce<_Cbx^<^a3jkzEze&aUGPc8`JGVL|AkkSk*zdw=)6 Rw>Fq#bQGB$?D&7C{{eG_ Date: Tue, 26 May 2015 08:58:37 -0700 Subject: [PATCH 11/52] Fix segfault when DNS is not available. Based on patch by Frank Meier. BIT-1387 #merged --- CHANGES | 5 +++++ VERSION | 2 +- aux/plugins | 2 +- src/DNS_Mgr.cc | 9 +++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 6dbaea2555..d5fdcf2f87 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.4-beta-2 | 2015-05-26 08:58:37 -0700 + + * Fix segfault when DNS is not available. Addresses BIT-1387. (Frank + Meier and Robin Sommer) + 2.4-beta | 2015-05-07 21:55:31 -0700 * Release 2.4-beta. diff --git a/VERSION b/VERSION index 0a8901319c..a31f783894 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-beta +2.4-beta-2 diff --git a/aux/plugins b/aux/plugins index e1ea9f67cf..e78ad8574b 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit e1ea9f67cfe3d6a81e0c1479ced0b9aa73e77c87 +Subproject commit e78ad8574ba0e5280065b4edadecd8330601d9cb diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 11fd258d09..99947e3531 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1219,6 +1219,9 @@ void DNS_Mgr::IssueAsyncRequests() void DNS_Mgr::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, iosource::FD_Set* except) { + if ( ! nb_dns ) + return; + read->Insert(nb_dns_fd(nb_dns)); } @@ -1358,6 +1361,9 @@ void DNS_Mgr::Process() void DNS_Mgr::DoProcess(bool flush) { + if ( ! nb_dns ) + return; + while ( asyncs_timeouts.size() > 0 ) { AsyncRequest* req = asyncs_timeouts.top(); @@ -1422,6 +1428,9 @@ void DNS_Mgr::DoProcess(bool flush) int DNS_Mgr::AnswerAvailable(int timeout) { + if ( ! nb_dns ) + return -1; + int fd = nb_dns_fd(nb_dns); if ( fd < 0 ) { From d9ef8c36c92372af64e4c7bd08203d9f7ed71293 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 May 2015 12:02:26 -0700 Subject: [PATCH 12/52] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 97c17d2172..33f3d8d6ee 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 97c17d21725e42b36f4b49579077ecdc28ddb86a +Subproject commit 33f3d8d6ee171f40cfa43cd2c68b49d83f7897f2 From 0a9b768e4657e62cf4657a993b907bc067ecffb5 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 May 2015 12:15:48 -0700 Subject: [PATCH 13/52] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 33f3d8d6ee..f348334316 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 33f3d8d6ee171f40cfa43cd2c68b49d83f7897f2 +Subproject commit f348334316233f2ad1c8993f17b1f40cf0aacd0b From fbf40090a818e2d5b2efc9b1c854fc56d3472051 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 May 2015 13:20:44 -0700 Subject: [PATCH 14/52] Updating submodule(s). [nomail] --- aux/broctl | 2 +- aux/plugins | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broctl b/aux/broctl index f348334316..57af232945 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit f348334316233f2ad1c8993f17b1f40cf0aacd0b +Subproject commit 57af232945ef1270838e4d12d06e6d210d2e6935 diff --git a/aux/plugins b/aux/plugins index e78ad8574b..475beed1e9 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit e78ad8574ba0e5280065b4edadecd8330601d9cb +Subproject commit 475beed1e9688f572ee60c196e07c0fa72e1ed9f From 2b1cd66f17194a30b90490965cbdffdd71c18c09 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 May 2015 13:37:52 -0700 Subject: [PATCH 15/52] Updating CHANGES and VERSION. --- CHANGES | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d5fdcf2f87..e22814cb93 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.4-beta-6 | 2015-05-28 13:20:44 -0700 + + * Updating submodule(s). + 2.4-beta-2 | 2015-05-26 08:58:37 -0700 * Fix segfault when DNS is not available. Addresses BIT-1387. (Frank diff --git a/VERSION b/VERSION index a31f783894..19c129080a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-beta-2 +2.4-beta-6 From 7cf04c9f3a1d425a6bc5c8fa9895b65fc9ef21c3 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 28 May 2015 17:52:32 -0500 Subject: [PATCH 16/52] Improve logging framework doc Reorganized the content to be easier to follow, added a few more examples, fixed some ugly formatting (removed scrollbars that make the examples difficult to read). --- doc/frameworks/logging-input-sqlite.rst | 4 +- doc/frameworks/logging.rst | 618 ++++++++++++++---------- 2 files changed, 371 insertions(+), 251 deletions(-) diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 15df91b8c6..6f5e867686 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -67,8 +67,8 @@ that are present in the ASCII log files:: 'id.orig_p' integer, ... -Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a -log stream, you can remove the default filter: +Note that the ASCII ``conn.log`` will still be created. To prevent this file +from being created, you can remove the default filter: .. code:: bro diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index 765d7bed23..7cb21da36b 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -19,195 +19,128 @@ Terminology Bro's logging interface is built around three main abstractions: - Log streams - A stream corresponds to a single log. It defines the set of - fields that a log consists of with their names and fields. - Examples are the ``conn`` for recording connection summaries, + Streams + A log stream corresponds to a single log. It defines the set of + fields that a log consists of with their names and types. + Examples are the ``conn`` stream for recording connection summaries, and the ``http`` stream for recording HTTP activity. Filters Each stream has a set of filters attached to it that determine what information gets written out. By default, each stream has - one default filter that just logs everything directly to disk - with an automatically generated file name. However, further - filters can be added to record only a subset, split a stream - into different outputs, or to even duplicate the log to - multiple outputs. If all filters are removed from a stream, - all output is disabled. + one default filter that just logs everything directly to disk. + However, additional filters can be added to record only a subset + of the log records, write to different outputs, or set a custom + rotation interval. If all filters are removed from a stream, + then output is disabled for that stream. Writers - A writer defines the actual output format for the information - being logged. At the moment, Bro comes with only one type of - writer, which produces tab separated ASCII files. In the - future we will add further writers, like for binary output and - direct logging into a database. + Each filter has a writer. A writer defines the actual output + format for the information being logged. The default writer is + the ASCII writer, which produces tab-separated ASCII files. Other + writers are available, like for binary output or direct logging + into a database. -Basics -====== +There are several different ways to customize Bro's logging: you can create +a new log stream, you can extend an existing log with new fields, you +can apply filters to an existing log stream, or you can customize the output +format by setting log writer options. All of these approaches are +described in this document. -The data fields that a stream records are defined by a record type -specified when it is created. Let's look at the script generating Bro's -connection summaries as an example, -:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record -:bro:type:`Conn::Info` that lists all the fields that go into -``conn.log``, each marked with a ``&log`` attribute indicating that it -is part of the information written out. To write a log record, the -script then passes an instance of :bro:type:`Conn::Info` to the logging -framework's :bro:id:`Log::write` function. +Streams +======= -By default, each stream automatically gets a filter named ``default`` -that generates the normal output by recording all record fields into a -single output file. +In order to log data to a new log stream, all of the following needs to be +done: -In the following, we summarize ways in which the logging can be -customized. We continue using the connection summaries as our example -to work with. +- A :bro:type:`record` type must be defined which consists of all the + fields that will be logged (by convention, the name of this record type is + usually "Info"). +- A log stream ID (an :bro:type:`enum` with type name "Log::ID") must be + defined that uniquely identifies the new log stream. +- A log stream must be created using the :bro:id:`Log::create_stream` function. +- When the data to be logged becomes available, the :bro:id:`Log::write` + function must be called. -Filtering ---------- - -To create a new output file for an existing stream, you can add a -new filter. A filter can, e.g., restrict the set of fields being -logged: +In the following example, we create a new module "Foo" which creates +a new log stream. .. code:: bro - event bro_init() + module Foo; + + export { + # Create an ID for our new stream. By convention, this is + # called "LOG". + redef enum Log::ID += { LOG }; + + # Define the record type that will contain the data to log. + # By convention, the type is called "Info". + # All fields to log must have the &log attribute. + # If a field might not contain any data, use the + # &optional attribute. + type Info: record { + ts: time &log; + id: conn_id &log; + service: string &log &optional; + }; + } + + # Optionally, we can add a new field to the connection record so that + # the data we are logging (our "Info" record) will be easily + # accessible in a variety of event handlers. + # By convention, the name of this new field is the lowercase name of + # the module. + redef record connection += { + foo: Info &optional; + }; + + # This event is handled at a priority higher than zero so that if + # users modify this stream in another script, they can do so at the + # default priority of zero. + event bro_init() &priority=5 { - # Add a new filter to the Conn::LOG stream that logs only - # timestamp and originator address. - local filter: Log::Filter = [$name="orig-only", $path="origs", $include=set("ts", "id.orig_h")]; - Log::add_filter(Conn::LOG, filter); + # Create the stream. This adds a default filter automatically. + Log::create_stream(Foo::LOG, [$columns=Info, $path="foo"]); } -Note the fields that are set for the filter: - - ``name`` - A mandatory name for the filter that can later be used - to manipulate it further. - - ``path`` - The filename for the output file, without any extension (which - may be automatically added by the writer). Default path values - are generated by taking the stream's ID and munging it slightly. - :bro:enum:`Conn::LOG` is converted into ``conn``, - :bro:enum:`PacketFilter::LOG` is converted into - ``packet_filter``, and :bro:enum:`Known::CERTS_LOG` is - converted into ``known_certs``. - - ``include`` - A set limiting the fields to the ones given. The names - correspond to those in the :bro:type:`Conn::Info` record, with - sub-records unrolled by concatenating fields (separated with - dots). - -Using the code above, you will now get a new log file ``origs.log`` -that looks like this:: - - #separator \x09 - #path origs - #fields ts id.orig_h - #types time addr - 1128727430.350788 141.42.64.125 - 1128727435.450898 141.42.64.125 - -If you want to make this the only log file for the stream, you can -remove the default filter (which, conveniently, has the name -``default``): +At this point, the only thing missing is a call to the :bro:id:`Log::write` +function to send data to the logging framework. The actual event handler +where this should take place will depend on where your data becomes available. +In this example, the connection_established event provides our data, and we +also store a copy of the data being logged into the :bro:type:`connection` +record: .. code:: bro - event bro_init() + event connection_established(c: connection) { - # Remove the filter called "default". - Log::remove_filter(Conn::LOG, "default"); + local rec: Foo::Info = [$ts=network_time(), $id=c$id]; + + # Store a copy of the data in the connection record so other + # event handlers can access it. + c$foo = rec; + + Log::write(Foo::LOG, rec); } -An alternate approach to "turning off" a log is to completely disable -the stream: +When you are developing scripts that add data to the :bro:type:`connection` +record, care must be given to when and how long data is stored. +Normally data saved to the connection record will remain there for the +duration of the connection and from a practical perspective it's not +uncommon to need to delete that data before the end of the connection. -.. code:: bro - event bro_init() - { - Log::disable_stream(Conn::LOG); - } +Add Fields to a Log +------------------- -If you want to skip only some fields but keep the rest, there is a -corresponding ``exclude`` filter attribute that you can use instead of -``include`` to list only the ones you are not interested in. +You can add additional fields to a log by extending the record +type that defines its content, and setting a value for the new fields +before each log record is written. -A filter can also determine output paths *dynamically* based on the -record being logged. That allows, e.g., to record local and remote -connections into separate files. To do this, you define a function -that returns the desired path: - -.. code:: bro - - function split_log(id: Log::ID, path: string, rec: Conn::Info) : string - { - # Return "conn-local" if originator is a local IP, otherwise "conn-remote". - local lr = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; - return fmt("%s-%s", path, lr); - } - - event bro_init() - { - local filter: Log::Filter = [$name="conn-split", $path_func=split_log, $include=set("ts", "id.orig_h")]; - Log::add_filter(Conn::LOG, filter); - } - -Running this will now produce two files, ``local.log`` and -``remote.log``, with the corresponding entries. One could extend this -further for example to log information by subnets or even by IP -address. Be careful, however, as it is easy to create many files very -quickly ... - -.. sidebar:: A More Generic Path Function - - The ``split_log`` method has one draw-back: it can be used - only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded - into its argument list. However, Bro allows to do a more generic - variant: - - .. code:: bro - - function split_log(id: Log::ID, path: string, rec: record { id: conn_id; } ) : string - { - return Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; - } - - This function can be used with all log streams that have records - containing an ``id: conn_id`` field. - -While so far we have seen how to customize the columns being logged, -you can also control which records are written out by providing a -predicate that will be called for each log record: - -.. code:: bro - - function http_only(rec: Conn::Info) : bool - { - # Record only connections with successfully analyzed HTTP traffic - return rec$service == "http"; - } - - event bro_init() - { - local filter: Log::Filter = [$name="http-only", $path="conn-http", $pred=http_only]; - Log::add_filter(Conn::LOG, filter); - } - -This will result in a log file ``conn-http.log`` that contains only -traffic detected and analyzed as HTTP traffic. - -Extending ---------- - -You can add further fields to a log stream by extending the record -type that defines its content. Let's say we want to add a boolean -field ``is_private`` to :bro:type:`Conn::Info` that indicates whether the -originator IP address is part of the :rfc:`1918` space: +Let's say we want to add a boolean field ``is_private`` to +:bro:type:`Conn::Info` that indicates whether the originator IP address +is part of the :rfc:`1918` space: .. code:: bro @@ -218,9 +151,21 @@ originator IP address is part of the :rfc:`1918` space: is_private: bool &default=F &log; }; +As this example shows, when extending a log stream's "Info" record, each +new field must always be declared either with a ``&default`` value or +as ``&optional``. Furthermore, you need to add the ``&log`` attribute +or otherwise the field won't appear in the log file. -Now we need to set the field. A connection's summary is generated at -the time its state is removed from memory. We can add another handler +Now we need to set the field. Although the details vary depending on which +log is being extended, in general it is important to choose a suitable event +in which to set the additional fields because we need to make sure that +the fields are set before the log record is written. Sometimes the right +choice is the same event which writes the log record, but at a higher +priority (in order to ensure that the event handler that sets the additional +fields is executed before the event handler that writes the log record). + +In this example, since a connection's summary is generated at +the time its state is removed from memory, we can add another handler at that time that sets our field correctly: .. code:: bro @@ -232,31 +177,56 @@ at that time that sets our field correctly: } Now ``conn.log`` will show a new field ``is_private`` of type -``bool``. +``bool``. If you look at the Bro script which defines the connection +log stream :doc:`/scripts/base/protocols/conn/main.bro`, you will see +that the connection log record is written in an event handler for the +same event as used in this example to set the additional fields, but at a +lower priority than the one used in this example. -Notes: +For extending logs this way, one needs a bit of knowledge about how +the script that creates the log stream is organizing its state +keeping. Most of the standard Bro scripts attach their log state to +the :bro:type:`connection` record where it can then be accessed, just +like ``c$conn`` above. For example, the HTTP analysis adds a field +``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection` +record. -- For extending logs this way, one needs a bit of knowledge about how - the script that creates the log stream is organizing its state - keeping. Most of the standard Bro scripts attach their log state to - the :bro:type:`connection` record where it can then be accessed, just - as the ``c$conn`` above. For example, the HTTP analysis adds a field - ``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection` - record. See the script reference for more information. -- When extending records as shown above, the new fields must always be - declared either with a ``&default`` value or as ``&optional``. - Furthermore, you need to add the ``&log`` attribute or otherwise the - field won't appear in the output. - -Hooking into the Logging ------------------------- +Define a Logging Event +---------------------- Sometimes it is helpful to do additional analysis of the information being logged. For these cases, a stream can specify an event that will -be generated every time a log record is written to it. All of Bro's -default log streams define such an event. For example, the connection -log stream raises the event :bro:id:`Conn::log_conn`. You +be generated every time a log record is written to it. To do this, we +need to modify the example module shown above to look something like this: + +.. code:: bro + + module Foo; + + export { + redef enum Log::ID += { LOG }; + + type Info: record { + ts: time &log; + id: conn_id &log; + }; + + # Define a logging event. By convention, this is called + # "log_". + global log_foo: event(rec: Info); + } + + event bro_init() &priority=5 + { + # Here we specify both the "Info" record type and the "log_foo" + # event. + Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo, + $path="foo"]); + } + +All of Bro's default log streams define such an event. For example, the +connection log stream raises the event :bro:id:`Conn::log_conn`. You could use that for example for flagging when a connection to a specific destination exceeds a certain duration: @@ -281,15 +251,193 @@ externally with Perl scripts. Much of what such an external script would do later offline, one may instead do directly inside of Bro in real-time. -Rotation --------- +Disable a Stream +---------------- -By default, no log rotation occurs, but it's globally controllable for all -filters by redefining the :bro:id:`Log::default_rotation_interval` option: +One way to "turn off" a log is to completely disable the stream. For +example, the following example will prevent the conn.log from being written: .. code:: bro - redef Log::default_rotation_interval = 1 hr; + event bro_init() + { + Log::disable_stream(Conn::LOG); + } + +Note that this must run after the stream is created, so the priority +of this event handler must be lower than the priority of the event handler +where the stream was created. + + +Filters +======= + +A stream has one or more filters attached to it (a stream without any filters +will not produce any log output). When a stream is created, it automatically +gets a default filter attached to it. This default filter can be removed +or replaced, or other filters can be added to the stream. This is accomplished +by using either the :bro:id:`Log::add_filter` or :bro:id:`Log::remove_filter` +function. This section shows how to use filters to do such tasks as +rename a log file, split the output into multiple files, control which +records are written, and set a custom rotation interval. + +Rename Log File +--------------- + +Normally, the log filename for a given log stream is determined when the +stream is created, unless you explicitly specify a different one by adding +a filter. + +The easiest way to change a log filename is to simply replace the +default log filter with a new filter that specifies a value for the "path" +field. In this example, "conn.log" will be changed to "myconn.log": + +.. code:: bro + + event bro_init() + { + # Replace default filter for the Conn::LOG stream in order to + # change the log filename. + + local f = Log::get_filter(Conn::LOG, "default"); + f$path = "myconn"; + Log::add_filter(Conn::LOG, f); + } + +Keep in mind that the "path" field of a log filter never contains the +filename extension. The extension will be determined later by the log writer. + +Add a New Log File +------------------ + +Normally, a log stream writes to only one log file. However, you can +add filters so that the stream writes to multiple files. This is useful +if you want to restrict the set of fields being logged to the new file. + +In this example, a new filter is added to the Conn::LOG stream that writes +two fields to a new log file: + +.. code:: bro + + event bro_init() + { + # Add a new filter to the Conn::LOG stream that logs only + # timestamp and originator address. + + local filter: Log::Filter = [$name="orig-only", $path="origs", + $include=set("ts", "id.orig_h")]; + Log::add_filter(Conn::LOG, filter); + } + + +Notice how the "include" filter attribute specifies a set that limits the +fields to the ones given. The names correspond to those in the +:bro:type:`Conn::Info` record (however, because the "id" field is itself a +record, we can specify an individual field of "id" by the dot notation +shown in the example). + +Using the code above, in addition to the regular ``conn.log``, you will +now also get a new log file ``origs.log`` that looks like the regular +``conn.log``, but will have only the fields specified in the "include" +filter attribute. + +If you want to skip only some fields but keep the rest, there is a +corresponding ``exclude`` filter attribute that you can use instead of +``include`` to list only the ones you are not interested in. + +If you want to make this the only log file for the stream, you can +remove the default filter: + +.. code:: bro + + event bro_init() + { + # Remove the filter called "default". + Log::remove_filter(Conn::LOG, "default"); + } + +Determine Log Path Dynamically +------------------------------ + +Instead of using the "path" filter attribute, a filter can determine +output paths *dynamically* based on the record being logged. That +allows, e.g., to record local and remote connections into separate +files. To do this, you define a function that returns the desired path, +and use the "path_func" filter attribute: + +.. code:: bro + + function myfunc(id: Log::ID, path: string, rec: Conn::Info) : string + { + # Return "conn-local" if originator is a local IP, otherwise + # return "conn-remote". + + local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; + return fmt("%s-%s", path, r); + } + + event bro_init() + { + local filter: Log::Filter = [$name="conn-split", + $path_func=myfunc, $include=set("ts", "id.orig_h")]; + Log::add_filter(Conn::LOG, filter); + } + +Running this will now produce two new files, ``conn-local.log`` and +``conn-remote.log``, with the corresponding entries. One could extend this +further for example to log information by subnets or even by IP +address. Be careful, however, as it is easy to create many files very +quickly. + +The ``myfunc`` function has one drawback: it can be used +only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded +into its argument list. However, Bro allows to do a more generic +variant: + +.. code:: bro + + function myfunc(id: Log::ID, path: string, + rec: record { id: conn_id; } ) : string + { + local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; + return fmt("%s-%s", path, r); + } + +This function can be used with all log streams that have records +containing an ``id: conn_id`` field. + +Filter Log Records +------------------ + +We have seen how to customize the columns being logged, but +you can also control which records are written out by providing a +predicate that will be called for each log record: + +.. code:: bro + + function http_only(rec: Conn::Info) : bool + { + # Record only connections with successfully analyzed HTTP traffic + return rec$service == "http"; + } + + event bro_init() + { + local filter: Log::Filter = [$name="http-only", $path="conn-http", + $pred=http_only]; + Log::add_filter(Conn::LOG, filter); + } + +This will result in a new log file ``conn-http.log`` that contains only +traffic detected and analyzed as HTTP traffic. + +Rotation +-------- + +The log rotation interval is globally controllable for all +filters by redefining the :bro:id:`Log::default_rotation_interval` option +(note that when using BroControl, this option is set automatically via +the BroControl configuration). Or specifically for certain :bro:type:`Log::Filter` instances by setting their ``interv`` field. Here's an example of changing just the @@ -301,90 +449,62 @@ their ``interv`` field. Here's an example of changing just the { local f = Log::get_filter(Conn::LOG, "default"); f$interv = 1 min; - Log::remove_filter(Conn::LOG, "default"); Log::add_filter(Conn::LOG, f); } -ASCII Writer Configuration --------------------------- +Writers +======= -The ASCII writer has a number of options for customizing the format of -its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`. +Each filter has a writer. If you do not specify a writer when adding a +filter to a stream, then the ASCII writer is the default. -Adding Streams -============== +There are two ways to specify a non-default writer. To change the default +writer for all log filters, just redefine the :bro:id:`Log::default_writer` +option. Alternatively, you can specify the writer to use on a per-filter +basis by setting a value for the filter's "writer" field. Consult the +documentation of the writer to use to see if there are other options that are +needed. -It's easy to create a new log stream for custom scripts. Here's an -example for the ``Foo`` module: +ASCII Writer +------------ + +The ASCII writer has a number of options for customizing the format of its +output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`. +If you change the output format options, then be careful to check whether +your postprocessing scripts can still recognize your log files. + +Some writer options are global (i.e., they affect all log filters using +that log writer). For example, to change the output format of all ASCII +logs to JSON format: .. code:: bro - module Foo; + redef LogAscii::use_json = T; - export { - # Create an ID for our new stream. By convention, this is - # called "LOG". - redef enum Log::ID += { LOG }; +Some writer options are filter-specific (i.e., they affect only the filters +that explicitly specify the option). For example, to change the output +format of the ``conn.log`` only: - # Define the fields. By convention, the type is called "Info". - type Info: record { - ts: time &log; - id: conn_id &log; - }; +.. code:: bro - # Define a hook event. By convention, this is called - # "log_". - global log_foo: event(rec: Info); - - } - - # This event should be handled at a higher priority so that when - # users modify your stream later and they do it at priority 0, - # their code runs after this. - event bro_init() &priority=5 + event bro_init() { - # Create the stream. This also adds a default filter automatically. - Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo, $path="foo"]); + local f = Log::get_filter(Conn::LOG, "default"); + # Use tab-separated-value mode + f$config = table(["tsv"] = "T"); + Log::add_filter(Conn::LOG, f); } -You can also add the state to the :bro:type:`connection` record to make -it easily accessible across event handlers: - -.. code:: bro - - redef record connection += { - foo: Info &optional; - } - -Now you can use the :bro:id:`Log::write` method to output log records and -save the logged ``Foo::Info`` record into the connection record: - -.. code:: bro - - event connection_established(c: connection) - { - local rec: Foo::Info = [$ts=network_time(), $id=c$id]; - c$foo = rec; - Log::write(Foo::LOG, rec); - } - -See the existing scripts for how to work with such a new connection -field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`. - -When you are developing scripts that add data to the :bro:type:`connection` -record, care must be given to when and how long data is stored. -Normally data saved to the connection record will remain there for the -duration of the connection and from a practical perspective it's not -uncommon to need to delete that data before the end of the connection. Other Writers ------------- -Bro supports the following built-in output formats other than ASCII: +Bro supports the following additional built-in output formats: .. toctree:: :maxdepth: 1 logging-input-sqlite -Further formats are available as external plugins. +Additional writers are available as external plugins. + From 24701f26782aa8764edf43d82e0f906adef44363 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 29 May 2015 14:38:50 -0500 Subject: [PATCH 17/52] Fix a "make doc" warning Also fixed some indentation. --- scripts/base/frameworks/logging/main.bro | 2 +- scripts/base/frameworks/logging/writers/sqlite.bro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 1732cc5540..769136e6e6 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -152,7 +152,7 @@ export { ## easy to flood the disk by returning a new string for each ## connection. Upon adding a filter to a stream, if neither ## ``path`` nor ``path_func`` is explicitly set by them, then - ## :bro:see:`default_path_func` is used. + ## :bro:see:`Log::default_path_func` is used. ## ## id: The ID associated with the log stream. ## diff --git a/scripts/base/frameworks/logging/writers/sqlite.bro b/scripts/base/frameworks/logging/writers/sqlite.bro index d73e95ac0a..f7ebd9130c 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.bro +++ b/scripts/base/frameworks/logging/writers/sqlite.bro @@ -19,7 +19,7 @@ export { const unset_field = Log::unset_field &redef; ## String to use for empty fields. This should be different from - ## *unset_field* to make the output unambiguous. + ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; } From 260b25f20a5668a7eba9074569dcb0458a226a30 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Sat, 30 May 2015 00:18:04 -0500 Subject: [PATCH 18/52] Fix typos in the "writing bro plugins" doc --- doc/devel/plugins.rst | 62 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst index 5c963a1552..091a0090d1 100644 --- a/doc/devel/plugins.rst +++ b/doc/devel/plugins.rst @@ -3,7 +3,7 @@ Writing Bro Plugins =================== -Bro internally provides plugin API that enables extending +Bro internally provides a plugin API that enables extending the system dynamically, without modifying the core code base. That way custom code remains self-contained and can be maintained, compiled, and installed independently. Currently, plugins can add the following @@ -32,7 +32,7 @@ Quick Start =========== Writing a basic plugin is quite straight-forward as long as one -follows a few conventions. In the following we walk a simple example +follows a few conventions. In the following we create a simple example plugin that adds a new built-in function (bif) to Bro: we'll add ``rot13(s: string) : string``, a function that rotates every character in a string by 13 places. @@ -81,7 +81,7 @@ The syntax of this file is just like any other ``*.bif`` file; we won't go into it here. Now we can already compile our plugin, we just need to tell the -configure script that ``init-plugin`` put in place where the Bro +configure script (that ``init-plugin`` created) where the Bro source tree is located (Bro needs to have been built there first):: # cd rot13-plugin @@ -99,7 +99,7 @@ option:: # export BRO_PLUGIN_PATH=/path/to/rot13-plugin/build # bro -N [...] - Plugin: Demo::Rot13 - (dynamic, version 1) + Demo::Rot13 - (dynamic, version 0.1) [...] That looks quite good, except for the dummy description that we should @@ -108,28 +108,30 @@ is about. We do this by editing the ``config.description`` line in ``src/Plugin.cc``, like this:: [...] - plugin::Configuration Configure() + plugin::Configuration Plugin::Configure() { plugin::Configuration config; config.name = "Demo::Rot13"; config.description = "Caesar cipher rotating a string's characters by 13 places."; - config.version.major = 1; - config.version.minor = 0; + config.version.major = 0; + config.version.minor = 1; return config; } [...] +Now rebuild and verify that the description is visible:: + # make [...] # bro -N | grep Rot13 - Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 0.1) -Better. Bro can also show us what exactly the plugin provides with the +Bro can also show us what exactly the plugin provides with the more verbose option ``-NN``:: # bro -NN [...] - Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 0.1) [Function] Demo::rot13 [...] @@ -157,10 +159,12 @@ The installed version went into ``/lib/bro/plugins/Demo_Rot13``. One can distribute the plugin independently of Bro for others to use. -To distribute in source form, just remove the ``build/`` (``make -distclean`` does that) and then tar up the whole ``rot13-plugin/`` +To distribute in source form, just remove the ``build/`` directory +(``make distclean`` does that) and then tar up the whole ``rot13-plugin/`` directory. Others then follow the same process as above after -unpacking. To distribute the plugin in binary form, the build process +unpacking. + +To distribute the plugin in binary form, the build process conveniently creates a corresponding tarball in ``build/dist/``. In this case, it's called ``Demo_Rot13-0.1.tar.gz``, with the version number coming out of the ``VERSION`` file that ``init-plugin`` put @@ -169,14 +173,14 @@ plugin, but no further source files. Optionally, one can include further files by specifying them in the plugin's ``CMakeLists.txt`` through the ``bro_plugin_dist_files`` macro; the skeleton does that for ``README``, ``VERSION``, ``CHANGES``, and ``COPYING``. To use the -plugin through the binary tarball, just unpack it and point -``BRO_PLUGIN_PATH`` there; or copy it into -``/lib/bro/plugins/`` directly. +plugin through the binary tarball, just unpack it into +``/lib/bro/plugins/``. Alternatively, if you unpack +it in another location, then you need to point ``BRO_PLUGIN_PATH`` there. Before distributing your plugin, you should edit some of the meta files that ``init-plugin`` puts in place. Edit ``README`` and ``VERSION``, and update ``CHANGES`` when you make changes. Also put a -license file in place as ``COPYING``; if BSD is fine, you find a +license file in place as ``COPYING``; if BSD is fine, you will find a template in ``COPYING.edit-me``. Plugin Directory Layout @@ -193,7 +197,7 @@ directory. With the skeleton, ```` corresponds to ``build/``. must exist, and its content must consist of a single line with the qualified name of the plugin (e.g., "Demo::Rot13"). -``/lib/--.so`` +``/lib/.-.so`` The shared library containing the plugin's compiled code. Bro will load this in dynamically at run-time if OS and architecture match the current platform. @@ -215,8 +219,8 @@ directory. With the skeleton, ```` corresponds to ``build/``. Any other files in ```` are ignored by Bro. By convention, a plugin should put its custom scripts into sub folders -of ``scripts/``, i.e., ``scripts//