From a7427e95bf405fb595d7a6484d349f47796dc62e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 21 Dec 2021 08:32:32 +0100 Subject: [PATCH] Switch to recording unmodified HTTP header. We used to attempt to remove any port specification before recording HTTP host headers in logs. Doing so would (1) remove potentially useful information, (2) not match what the documentation seemed to suggest, and (3) fail for IP6 addresses containing colons. We now record the original HOST header as is. Addresses #1844. --- NEWS | 4 ++++ scripts/base/protocols/http/main.zeek | 5 +++-- testing/btest/Baseline/plugins.hooks/output | 3 --- .../http.log | 2 +- .../http.log | 2 +- testing/external/commit-hash.zeek-testing-private | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 06c03755ca..c986b21e4d 100644 --- a/NEWS +++ b/NEWS @@ -147,6 +147,10 @@ Changed Functionality --enable-cpp-tests configure flag. Unit tests now also work in (static and dynamic) Zeek plugins. +- Entries in ``http.log`` now record the original ``HOST`` headers. + Previously, they would skip any port specification a header might + include. + Removed Functionality --------------------- diff --git a/scripts/base/protocols/http/main.zeek b/scripts/base/protocols/http/main.zeek index d63a5efa43..c12da898d5 100644 --- a/scripts/base/protocols/http/main.zeek +++ b/scripts/base/protocols/http/main.zeek @@ -263,8 +263,9 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr c$http$referrer = value; else if ( name == "HOST" ) - # The split is done to remove the occasional port value that shows up here. - c$http$host = split_string1(value, /:/)[0]; + # Per #1844, we record the original host header, including any port + # specification if present. + c$http$host = value; else if ( name == "RANGE" ) c$http$range_request = T; diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 45ee2b9c00..6ce199cb81 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -4460,7 +4460,6 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(http_request, , ([id=[orig_ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(network_time, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) -> -XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(split_string1, , (bro.org, /^?(:)$?/)) -> XXXXXXXXXX.XXXXXX MetaHookPost DrainEvents() -> XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> false @@ -4500,7 +4499,6 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(http_request, , ([id=[orig_ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(network_time, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) -XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(split_string1, , (bro.org, /^?(:)$?/)) XXXXXXXXXX.XXXXXX MetaHookPre DrainEvents() XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) @@ -4541,7 +4539,6 @@ XXXXXXXXXX.XXXXXX | HookCallFunction http_request([id=[orig_h=141.142.228.5, ori XXXXXXXXXX.XXXXXX | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]) XXXXXXXXXX.XXXXXX | HookCallFunction network_time() XXXXXXXXXX.XXXXXX | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80) -XXXXXXXXXX.XXXXXX | HookCallFunction split_string1(bro.org, /^?(:)$?/) XXXXXXXXXX.XXXXXX | HookDrainEvents XXXXXXXXXX.XXXXXX | HookQueueEvent analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3) XXXXXXXXXX.XXXXXX | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log index cfb29a7388..d0ddc6b67f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log @@ -7,5 +7,5 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52522 ::1 80 1 CONNECT secure.newegg.com secure.newegg.com:443 - 1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0 - 0 0 200 Connection Established - - (empty) - - PROXY-CONNECTION -> keep-alive - - - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52522 ::1 80 1 CONNECT secure.newegg.com:443 secure.newegg.com:443 - 1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0 - 0 0 200 Connection Established - - (empty) - - PROXY-CONNECTION -> keep-alive - - - - - - #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log b/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log index 9f9419a32d..9a74c0750e 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log @@ -7,5 +7,5 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 45376 127.0.0.1 8000 1 POST localhost / - - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 null 767 0 - - - - (empty) - - - FeaAeS2KrXTNRvTtBh test.txt text/plain - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 45376 127.0.0.1 8000 1 POST localhost:8000 / - - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 null 767 0 - - - - (empty) - - - FeaAeS2KrXTNRvTtBh test.txt text/plain - - - #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 9590a998f2..78c1099b34 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -67f592e6a84d236aaf5cc08c91c71625a095e49a +7c40cc2c3709fc54e5c75c119d1d01ed8a3ceb93