diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 3bfd302ab5..a66a1bdbf5 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -48,6 +48,10 @@ export { status_code: count &log &optional; ## The status message returned by the server. status_msg: string &log &optional; + ## The last 1xx informational reply code returned by the server. + info_code: count &log &optional; + ## The last 1xx informational reply message returned by the server. + info_msg: string &log &optional; ## The filename given in the Content-Disposition header ## sent by the server. filename: string &log &optional; @@ -111,6 +115,11 @@ redef capture_filters += { ["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)" }; +function code_in_range(c: count, min: count, max: count) : bool + { + return c >= min && c <= max; + } + function new_http_session(c: connection): Info { local tmp: Info; @@ -163,12 +172,21 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p local s: State; c$http_state = s; } - - ++c$http_state$current_response; + + # If the last response was an informational 1xx, we're still expecting + # the real response to the request, so don't create a new Info record yet. + if ( c$http_state$current_response !in c$http_state$pending || + ! code_in_range(c$http_state$pending[c$http_state$current_response]$status_code, 100, 199) ) + ++c$http_state$current_response; set_state(c, F, F); c$http$status_code = code; c$http$status_msg = reason; + if ( code_in_range(code, 100, 199) ) + { + c$http$info_code = code; + c$http$info_msg = reason; + } } event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5 @@ -245,8 +263,13 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) & # The reply body is done so we're ready to log. if ( ! is_orig ) { - Log::write(HTTP::LOG, c$http); - delete c$http_state$pending[c$http_state$current_response]; + # If the response was an informational 1xx, we're still expecting + # the real response later, so we'll continue using the same record. + if ( ! code_in_range(c$http$status_code, 100, 199) ) + { + Log::write(HTTP::LOG, c$http); + delete c$http_state$pending[c$http_state$current_response]; + } } } diff --git a/src/HTTP.cc b/src/HTTP.cc index 0d154f1873..e94665bb2a 100644 --- a/src/HTTP.cc +++ b/src/HTTP.cc @@ -1305,7 +1305,9 @@ void HTTP_Analyzer::ReplyMade(const int interrupted, const char* msg) if ( reply_message ) reply_message->Done(interrupted, msg); - if ( ! unanswered_requests.empty() ) + // 1xx replies do not indicate the final response to a request, + // so don't pop an unanswered request in that case. + if ( (reply_code < 100 || reply_code >= 200) && ! unanswered_requests.empty() ) { Unref(unanswered_requests.front()); unanswered_requests.pop(); diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 4d8421ac86..a18242e8da 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -1,5 +1,5 @@ #separator \x09 #path http -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file -1316124231.969273 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file +1317149787.593092 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 4d8421ac86..a18242e8da 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -1,5 +1,5 @@ #separator \x09 #path http -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file -1316124231.969273 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file +1317149787.593092 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index f1b0fd02ab..3896ee8c3a 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -1,5 +1,5 @@ #separator \x09 #path http -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file -1316124240.720195 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file +1317149750.648989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index f1b0fd02ab..3896ee8c3a 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -1,5 +1,5 @@ #separator \x09 #path http -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file -1316124240.720195 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - text/html - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file +1317149750.648989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 0 F 9130 F 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index 4f7215e3c7..0f167ded6a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -1,5 +1,5 @@ #separator \x09 #path http -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file -1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 F 346 F 404 Not Found - - - - - text/html - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file +1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 F 346 F 404 Not Found - - - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log new file mode 100644 index 0000000000..10a6afcb37 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -0,0 +1,5 @@ +#separator \x09 +#path http +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 F 60731 F 200 OK 100 Continue - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index d69b3e5b1a..3c58932940 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -1,9 +1,9 @@ #separator \x09 #path http -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port string string string string string count bool count bool count string string table string string table string string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - FAKE_MIME - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - FAKE_MIME - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - FAKE_MIME - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - image/png e0029eea80812e9a8e57b8d05d52938a - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - image/png 30aa926344f58019d047e85ba049ca1e - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string string file +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - - - FAKE_MIME - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - - - FAKE_MIME - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - - - FAKE_MIME - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - - - image/png e0029eea80812e9a8e57b8d05d52938a - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - - - image/png 30aa926344f58019d047e85ba049ca1e - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index 4ef6d2ee1c..e5e3579a7b 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -1,9 +1,9 @@ #separator \x09 #path http -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg filename tags username password proxied md5 extraction_file -#types time string addr port addr port string string string string string count bool count bool count string string table string string table string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_body_len request_body_interrupted response_body_len response_body_interrupted status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file +#types time string addr port addr port string string string string string count bool count bool count string count string string table string string table string file +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2675 F 200 OK - - - - - - - - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 21421 F 200 OK - - - - - - - - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 94 F 200 OK - - - - - - - - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 2349 F 200 OK - - - - - - - - - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 F 27579 F 200 OK - - - - - - - - - diff --git a/testing/btest/Traces/http-100-continue.trace b/testing/btest/Traces/http-100-continue.trace new file mode 100644 index 0000000000..3ff38fa5c5 Binary files /dev/null and b/testing/btest/Traces/http-100-continue.trace differ diff --git a/testing/btest/scripts/base/protocols/http/100-continue.bro b/testing/btest/scripts/base/protocols/http/100-continue.bro new file mode 100644 index 0000000000..7b7b5bde89 --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/100-continue.bro @@ -0,0 +1,12 @@ +# This tests that the HTTP analyzer does not generate an unmatched_HTTP_reply +# weird as a result of seeing both a 1xx response and the real response to +# a given request. The http scripts should also be able log such replies +# in a way that correlates the final response with the request. +# +# @TEST-EXEC: bro -r $TRACES/http-100-continue.trace %INPUT +# @TEST-EXEC: grep -q unmatched_HTTP_reply weird.log && exit 1 || exit 0 +# @TEST-EXEC: btest-diff http.log + +# The base analysis scripts are loaded by default. +#@load base/protocols/http +