diff --git a/NEWS b/NEWS index 06c03755ca..c986b21e4d 100644 --- a/NEWS +++ b/NEWS @@ -147,6 +147,10 @@ Changed Functionality --enable-cpp-tests configure flag. Unit tests now also work in (static and dynamic) Zeek plugins. +- Entries in ``http.log`` now record the original ``HOST`` headers. + Previously, they would skip any port specification a header might + include. + Removed Functionality --------------------- diff --git a/scripts/base/protocols/http/main.zeek b/scripts/base/protocols/http/main.zeek index d63a5efa43..c12da898d5 100644 --- a/scripts/base/protocols/http/main.zeek +++ b/scripts/base/protocols/http/main.zeek @@ -263,8 +263,9 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr c$http$referrer = value; else if ( name == "HOST" ) - # The split is done to remove the occasional port value that shows up here. - c$http$host = split_string1(value, /:/)[0]; + # Per #1844, we record the original host header, including any port + # specification if present. + c$http$host = value; else if ( name == "RANGE" ) c$http$range_request = T; diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 45ee2b9c00..6ce199cb81 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -4460,7 +4460,6 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(http_request, , ([id=[orig_ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(network_time, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) -> -XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(split_string1, , (bro.org, /^?(:)$?/)) -> XXXXXXXXXX.XXXXXX MetaHookPost DrainEvents() -> XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> false @@ -4500,7 +4499,6 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(http_request, , ([id=[orig_ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(network_time, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) -XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(split_string1, , (bro.org, /^?(:)$?/)) XXXXXXXXXX.XXXXXX MetaHookPre DrainEvents() XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) @@ -4541,7 +4539,6 @@ XXXXXXXXXX.XXXXXX | HookCallFunction http_request([id=[orig_h=141.142.228.5, ori XXXXXXXXXX.XXXXXX | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]) XXXXXXXXXX.XXXXXX | HookCallFunction network_time() XXXXXXXXXX.XXXXXX | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80) -XXXXXXXXXX.XXXXXX | HookCallFunction split_string1(bro.org, /^?(:)$?/) XXXXXXXXXX.XXXXXX | HookDrainEvents XXXXXXXXXX.XXXXXX | HookQueueEvent analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3) XXXXXXXXXX.XXXXXX | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log index cfb29a7388..d0ddc6b67f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log @@ -7,5 +7,5 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52522 ::1 80 1 CONNECT secure.newegg.com secure.newegg.com:443 - 1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0 - 0 0 200 Connection Established - - (empty) - - PROXY-CONNECTION -> keep-alive - - - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52522 ::1 80 1 CONNECT secure.newegg.com:443 secure.newegg.com:443 - 1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0 - 0 0 200 Connection Established - - (empty) - - PROXY-CONNECTION -> keep-alive - - - - - - #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log b/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log index 9f9419a32d..9a74c0750e 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log @@ -7,5 +7,5 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 45376 127.0.0.1 8000 1 POST localhost / - - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 null 767 0 - - - - (empty) - - - FeaAeS2KrXTNRvTtBh test.txt text/plain - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 45376 127.0.0.1 8000 1 POST localhost:8000 / - - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 null 767 0 - - - - (empty) - - - FeaAeS2KrXTNRvTtBh test.txt text/plain - - - #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 9590a998f2..78c1099b34 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -67f592e6a84d236aaf5cc08c91c71625a095e49a +7c40cc2c3709fc54e5c75c119d1d01ed8a3ceb93