From 1778bc5fa8959aa75ed16dcdf77adc225fd583bf Mon Sep 17 00:00:00 2001 From: Craig Leres Date: Fri, 3 Dec 2021 14:46:09 -0800 Subject: [PATCH 01/18] FreeBSD: don't look for jemalloc as a package, it's in the base system --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 494cb2efa2..4bc747f904 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -294,7 +294,7 @@ if ( NOT BIFCL_EXE_PATH ) add_subdirectory(auxil/bifcl) endif () -if (ENABLE_JEMALLOC) +if (ENABLE_JEMALLOC AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") find_package(JeMalloc) if (NOT JEMALLOC_FOUND) From 47a267130bcbf8035c9e34031038bb1167b6290f Mon Sep 17 00:00:00 2001 From: Craig Leres Date: Fri, 3 Dec 2021 15:23:50 -0800 Subject: [PATCH 02/18] Set JEMALLOC_FOUND when on FreeBSD (suggested by Tim Wojtulewicz) --- CMakeLists.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bc747f904..bf7a793910 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -294,11 +294,15 @@ if ( NOT BIFCL_EXE_PATH ) add_subdirectory(auxil/bifcl) endif () -if (ENABLE_JEMALLOC AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") - find_package(JeMalloc) +if (ENABLE_JEMALLOC) + if (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") + set(JEMALLOC_FOUND true) + else () + find_package(JeMalloc) - if (NOT JEMALLOC_FOUND) - message(FATAL_ERROR "Could not find requested JeMalloc") + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "Could not find requested JeMalloc") + endif() endif() endif () From a7427e95bf405fb595d7a6484d349f47796dc62e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 21 Dec 2021 08:32:32 +0100 Subject: [PATCH 03/18] Switch to recording unmodified HTTP header. We used to attempt to remove any port specification before recording HTTP host headers in logs. Doing so would (1) remove potentially useful information, (2) not match what the documentation seemed to suggest, and (3) fail for IP6 addresses containing colons. We now record the original HOST header as is. Addresses #1844. --- NEWS | 4 ++++ scripts/base/protocols/http/main.zeek | 5 +++-- testing/btest/Baseline/plugins.hooks/output | 3 --- .../http.log | 2 +- .../http.log | 2 +- testing/external/commit-hash.zeek-testing-private | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 06c03755ca..c986b21e4d 100644 --- a/NEWS +++ b/NEWS @@ -147,6 +147,10 @@ Changed Functionality --enable-cpp-tests configure flag. Unit tests now also work in (static and dynamic) Zeek plugins. +- Entries in ``http.log`` now record the original ``HOST`` headers. + Previously, they would skip any port specification a header might + include. + Removed Functionality --------------------- diff --git a/scripts/base/protocols/http/main.zeek b/scripts/base/protocols/http/main.zeek index d63a5efa43..c12da898d5 100644 --- a/scripts/base/protocols/http/main.zeek +++ b/scripts/base/protocols/http/main.zeek @@ -263,8 +263,9 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr c$http$referrer = value; else if ( name == "HOST" ) - # The split is done to remove the occasional port value that shows up here. - c$http$host = split_string1(value, /:/)[0]; + # Per #1844, we record the original host header, including any port + # specification if present. + c$http$host = value; else if ( name == "RANGE" ) c$http$range_request = T; diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 45ee2b9c00..6ce199cb81 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -4460,7 +4460,6 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(http_request, , ([id=[orig_ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(network_time, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) -> -XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(split_string1, , (bro.org, /^?(:)$?/)) -> XXXXXXXXXX.XXXXXX MetaHookPost DrainEvents() -> XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> false @@ -4500,7 +4499,6 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(http_request, , ([id=[orig_ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(network_time, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) -XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(split_string1, , (bro.org, /^?(:)$?/)) XXXXXXXXXX.XXXXXX MetaHookPre DrainEvents() XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) @@ -4541,7 +4539,6 @@ XXXXXXXXXX.XXXXXX | HookCallFunction http_request([id=[orig_h=141.142.228.5, ori XXXXXXXXXX.XXXXXX | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]) XXXXXXXXXX.XXXXXX | HookCallFunction network_time() XXXXXXXXXX.XXXXXX | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80) -XXXXXXXXXX.XXXXXX | HookCallFunction split_string1(bro.org, /^?(:)$?/) XXXXXXXXXX.XXXXXX | HookDrainEvents XXXXXXXXXX.XXXXXX | HookQueueEvent analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3) XXXXXXXXXX.XXXXXX | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log index cfb29a7388..d0ddc6b67f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-connect-with-header/http.log @@ -7,5 +7,5 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52522 ::1 80 1 CONNECT secure.newegg.com secure.newegg.com:443 - 1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0 - 0 0 200 Connection Established - - (empty) - - PROXY-CONNECTION -> keep-alive - - - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52522 ::1 80 1 CONNECT secure.newegg.com:443 secure.newegg.com:443 - 1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0 - 0 0 200 Connection Established - - (empty) - - PROXY-CONNECTION -> keep-alive - - - - - - #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log b/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log index 9f9419a32d..9a74c0750e 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log @@ -7,5 +7,5 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 45376 127.0.0.1 8000 1 POST localhost / - - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 null 767 0 - - - - (empty) - - - FeaAeS2KrXTNRvTtBh test.txt text/plain - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 45376 127.0.0.1 8000 1 POST localhost:8000 / - - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 null 767 0 - - - - (empty) - - - FeaAeS2KrXTNRvTtBh test.txt text/plain - - - #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 9590a998f2..78c1099b34 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -67f592e6a84d236aaf5cc08c91c71625a095e49a +7c40cc2c3709fc54e5c75c119d1d01ed8a3ceb93 From 07045ec254dffec4f02ef684c30180ae388445f0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 21 Dec 2021 08:49:46 +0100 Subject: [PATCH 04/18] Fix host header normalization in intel framework. The way we were splitting off ports from host names could fail for IPv6 addresses. Closes #1844. --- scripts/policy/frameworks/intel/seen/http-headers.zeek | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/policy/frameworks/intel/seen/http-headers.zeek b/scripts/policy/frameworks/intel/seen/http-headers.zeek index 382bae439e..066d10c0fc 100644 --- a/scripts/policy/frameworks/intel/seen/http-headers.zeek +++ b/scripts/policy/frameworks/intel/seen/http-headers.zeek @@ -9,8 +9,8 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) switch ( name ) { case "HOST": - # The split is done to remove the occasional port value that shows up here (see also base script) - local host = split_string1(value, /:/)[0]; + # Remove the occasional port value that shows up here. + local host = gsub(value, /:[[:digit:]]+$/, ""); if ( is_valid_ip(host) ) Intel::seen([$host=to_addr(host), $indicator_type=Intel::ADDR, From 4f566f35ee0e9ef83f881b0054784f9eb8612ff4 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Wed, 22 Dec 2021 14:26:03 -0800 Subject: [PATCH 05/18] bug fix for reporting poorly formed record constructors --- src/Expr.cc | 9 ++++++++- .../btest/Baseline/language.record-bad-ctor3/out | 2 ++ testing/btest/language/record-bad-ctor3.zeek | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/language.record-bad-ctor3/out create mode 100644 testing/btest/language/record-bad-ctor3.zeek diff --git a/src/Expr.cc b/src/Expr.cc index 9271cf7e22..c291f288ae 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3307,11 +3307,15 @@ RecordConstructorExpr::RecordConstructorExpr(ListExprPtr constructor_list) const ExprPList& exprs = op->AsListExpr()->Exprs(); type_decl_list* record_types = new type_decl_list(exprs.length()); + const Expr* constructor_error_expr = nullptr; + for ( const auto& e : exprs ) { if ( e->Tag() != EXPR_FIELD_ASSIGN ) { - Error("bad type in record constructor", e); + // Don't generate the error yet, as reporting it + // requires that we have a well-formed type. + constructor_error_expr = e; SetError(); continue; } @@ -3323,6 +3327,9 @@ RecordConstructorExpr::RecordConstructorExpr(ListExprPtr constructor_list) } SetType(make_intrusive(record_types)); + + if ( constructor_error_expr ) + Error("bad type in record constructor", constructor_error_expr); } RecordConstructorExpr::RecordConstructorExpr(RecordTypePtr known_rt, ListExprPtr constructor_list) diff --git a/testing/btest/Baseline/language.record-bad-ctor3/out b/testing/btest/Baseline/language.record-bad-ctor3/out new file mode 100644 index 0000000000..4903963619 --- /dev/null +++ b/testing/btest/Baseline/language.record-bad-ctor3/out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/record-bad-ctor3.zeek, line 13: bad type in record constructor ([$x=a + 5, a + 9] and a + 9) diff --git a/testing/btest/language/record-bad-ctor3.zeek b/testing/btest/language/record-bad-ctor3.zeek new file mode 100644 index 0000000000..539aa5361a --- /dev/null +++ b/testing/btest/language/record-bad-ctor3.zeek @@ -0,0 +1,15 @@ +# @TEST-EXEC-FAIL: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +# Every element in a record ctor's expression list should have an assignment +# form. Make sure we correctly report errors when that's not the case. + +global a = 3; + +type r: record { x: count; y: count; }; + +event zeek_init() + { + local b: r = record($x = a + 5, a + 9); + print b; + } From 8f2ac836f81b59ba90172501d2880e987ca80f83 Mon Sep 17 00:00:00 2001 From: Craig Leres Date: Tue, 4 Jan 2022 17:19:43 -0800 Subject: [PATCH 06/18] Make FreeBSD more flexible, don't look for jemalloc unless we were given a specific path for it. --- CMakeLists.txt | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf7a793910..7f7907392e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -296,13 +296,19 @@ endif () if (ENABLE_JEMALLOC) if (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") - set(JEMALLOC_FOUND true) - else () - find_package(JeMalloc) - - if (NOT JEMALLOC_FOUND) - message(FATAL_ERROR "Could not find requested JeMalloc") + if (DEFINED ${JEMALLOC_ROOT_DIR}) + # Look for jemalloc at a specific path + find_package(JeMalloc) + else() + # jemalloc is in the base system + set(JEMALLOC_FOUND true) endif() + else() + find_package(JeMalloc) + endif() + + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "Could not find requested JeMalloc") endif() endif () From b9ba362948d81d45c1180d4f9b422aeb4de78c9a Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 5 Jan 2022 10:09:08 +0100 Subject: [PATCH 07/18] Fix return type of double_to_int Fixes GH-1919 --- src/zeek.bif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zeek.bif b/src/zeek.bif index fffb310412..3d28df7011 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -2424,7 +2424,7 @@ function int_to_count%(n: int%): count ## Returns: The :zeek:type:`double` *d* as unsigned integer, or 0 if *d* < 0.0. ## The value returned follows typical rounding rules, as implemented ## by rint(). -function double_to_int%(d: double%): count +function double_to_int%(d: double%): int %{ return zeek::val_mgr->Int(bro_int_t(rint(d))); %} From 46df2edfb1ff863db198e149b0282f30722f476a Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 5 Jan 2022 10:24:46 +0100 Subject: [PATCH 08/18] Remove CentOS 8 from CI. EOL was 2021-12-31 --- .cirrus.yml | 12 ------------ ci/centos-8/Dockerfile | 30 ------------------------------ 2 files changed, 42 deletions(-) delete mode 100644 ci/centos-8/Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml index c1f963cf6c..219a5b7456 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -122,18 +122,6 @@ centosstream8_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE -centos8_task: - container: - # CentOS 8 EOL: May 31, 2029 - dockerfile: ci/centos-8/Dockerfile - << : *RESOURCES_TEMPLATE - env: - ZEEK_CI_CREATE_ARTIFACT: 1 - << : *CI_TEMPLATE - upload_binary_artifacts: - path: build.tgz - benchmark_script: ./ci/benchmark.sh - centos7_task: container: # CentOS 7 EOL: June 30, 2024 diff --git a/ci/centos-8/Dockerfile b/ci/centos-8/Dockerfile deleted file mode 100644 index 6456e718a4..0000000000 --- a/ci/centos-8/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM centos:8 - -RUN dnf -y install epel-release dnf-plugins-core \ - && dnf clean all && rm -rf /var/cache/dnf - -RUN dnf config-manager --set-enabled powertools - -RUN dnf -y update && dnf -y install \ - git \ - cmake \ - make \ - gcc \ - gcc-c++ \ - flex \ - bison \ - swig \ - openssl \ - openssl-devel \ - libpcap-devel \ - python3 \ - python3-devel \ - python3-pip \ - zlib-devel \ - libsqlite3x-devel \ - findutils \ - diffutils \ - which \ - && dnf clean all && rm -rf /var/cache/dnf - -RUN pip3 install junit2html From 03b2adb3120d186e0e75350d41691650d850e41b Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 5 Jan 2022 09:11:45 -0700 Subject: [PATCH 09/18] Fixed local-compat test and added NEWS section for 5.0 --- NEWS | 15 +++++++++++++++ testing/btest/scripts/site/local-compat.test | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 49f003d071..ef23f2065d 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,21 @@ This document summarizes the most important changes in the current Zeek release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as Broker, come with their own ``CHANGES``.) +Zeek 5.0.0 +========== + +Breaking Changes +---------------- + +New Functionality +----------------- + +Changed Functionality +--------------------- + +Deprecated Functionality +------------------------ + Zeek 4.2.0 ========== diff --git a/testing/btest/scripts/site/local-compat.test b/testing/btest/scripts/site/local-compat.test index da3ef2510b..41cbeb5711 100644 --- a/testing/btest/scripts/site/local-compat.test +++ b/testing/btest/scripts/site/local-compat.test @@ -15,7 +15,7 @@ # # simply update this test's TEST-START-FILE with the latest contents # site/local.zeek. -@TEST-START-FILE local-4.2.zeek +@TEST-START-FILE local-5.0.zeek ##! Local site policy. Customize as appropriate. ##! ##! This file will not be overwritten when upgrading or reinstalling! From 412a053942c82a905fbddd52a58e1363b3313b83 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 5 Jan 2022 11:05:56 -0700 Subject: [PATCH 10/18] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index b8ae1f3362..1263619ce3 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit b8ae1f336272371d6c46fda133e472a075f69e3d +Subproject commit 1263619ce3279415314355032b855206f7e3e632 From 3d16baae83e7115fe644ac0859d4af45b45a0598 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 6 Jan 2022 12:05:31 -0700 Subject: [PATCH 11/18] Updating submodule(s). [nomail] --- auxil/bifcl | 2 +- auxil/binpac | 2 +- auxil/broker | 2 +- auxil/package-manager | 2 +- auxil/zeek-aux | 2 +- auxil/zeekctl | 2 +- cmake | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/auxil/bifcl b/auxil/bifcl index a7d9233b37..eed5effea5 160000 --- a/auxil/bifcl +++ b/auxil/bifcl @@ -1 +1 @@ -Subproject commit a7d9233b37daac558314625566bb8c8a993f2904 +Subproject commit eed5effea5661e03b50d0436fecb620a05fb1250 diff --git a/auxil/binpac b/auxil/binpac index 8b1322d306..e751fa36b1 160000 --- a/auxil/binpac +++ b/auxil/binpac @@ -1 +1 @@ -Subproject commit 8b1322d3060a1fecdc586693e6215ad7ef8ab0e9 +Subproject commit e751fa36b107c64bc9ac22c14fc694cd091ee526 diff --git a/auxil/broker b/auxil/broker index d9e8440053..41c524f172 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit d9e84400534b968e33ab01cfadfb569c0d7b2929 +Subproject commit 41c524f172aa357422f1ccf7e806b448d5def08e diff --git a/auxil/package-manager b/auxil/package-manager index feffa1d51e..f7a9da292c 160000 --- a/auxil/package-manager +++ b/auxil/package-manager @@ -1 +1 @@ -Subproject commit feffa1d51e4e5494fef7daf2bd044138cb04f621 +Subproject commit f7a9da292cc97567473d8656afbcced3c6d6ebfa diff --git a/auxil/zeek-aux b/auxil/zeek-aux index 12be5e3e51..f6cdb4b3fa 160000 --- a/auxil/zeek-aux +++ b/auxil/zeek-aux @@ -1 +1 @@ -Subproject commit 12be5e3e51a4a97ab3aa0fa4a02da194a83c7f24 +Subproject commit f6cdb4b3fa14065a135cbe8db0502419b3f9218b diff --git a/auxil/zeekctl b/auxil/zeekctl index 95b048298a..e7fd4d552e 160000 --- a/auxil/zeekctl +++ b/auxil/zeekctl @@ -1 +1 @@ -Subproject commit 95b048298a77bb14d2c54dcca8bb549c86eb96b9 +Subproject commit e7fd4d552ec7c3e55cb556943bf8b499d2db17e1 diff --git a/cmake b/cmake index 12fbc1a3bc..105f6c9df6 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 12fbc1a3bc206a57b079505e3df938c3a993ba58 +Subproject commit 105f6c9df616a4c2286d5ef38c2b31a718192301 From 334f7f4ada5d2a905654559f0baf1ec33c6b1a87 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Tue, 4 Jan 2022 17:00:49 -0800 Subject: [PATCH 12/18] addressed a couple of memory leaks in ZAM execution --- src/script_opt/ZAM/Gen-ZAM.cc | 4 ++-- src/script_opt/ZAM/Ops.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/script_opt/ZAM/Gen-ZAM.cc b/src/script_opt/ZAM/Gen-ZAM.cc index f81e74b7bc..92948b4d84 100644 --- a/src/script_opt/ZAM/Gen-ZAM.cc +++ b/src/script_opt/ZAM/Gen-ZAM.cc @@ -1386,8 +1386,8 @@ void ZAM_ExprOpTemplate::InstantiateEval(const vector& ot_orig, auto is_none = ei.LHS_ET() == ZAM_EXPR_TYPE_NONE; auto is_default = ei.LHS_ET() == ZAM_EXPR_TYPE_DEFAULT; - if ( zc != ZIC_FIELD && ! is_none && ! is_default && - find_type_info(ei.LHS_ET()).is_managed && ! HasExplicitResultType() ) + if ( ! is_none && ! is_default && find_type_info(ei.LHS_ET()).is_managed && + ! HasExplicitResultType() ) { auto delim = zc == ZIC_VEC ? "->" : "."; auto pre = "auto hold_lhs = " + lhs + delim + "ManagedVal();\n\t"; diff --git a/src/script_opt/ZAM/Ops.in b/src/script_opt/ZAM/Ops.in index 71f1760694..2ba97d526d 100644 --- a/src/script_opt/ZAM/Ops.in +++ b/src/script_opt/ZAM/Ops.in @@ -1669,7 +1669,7 @@ internal-op Next-Table-Iter-Val-Var # v3 = branch target if loop done type VVV eval NextTableIterPre(v2, v3) - frame[z.v1] = ti.IterValue(); + AssignV1(ti.IterValue()); ti.NextIter(frame); internal-op Next-Table-Iter-Val-Var-No-Vars @@ -1678,7 +1678,7 @@ internal-op Next-Table-Iter-Val-Var-No-Vars # v3 = branch target if loop done type VVV eval NextTableIterPre(v2, v3) - frame[z.v1] = ti.IterValue(); + AssignV1(ti.IterValue()); ti.IterFinished(); From 16be867fefd2851cbf46578c301a7818cd33b1e5 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Tue, 4 Jan 2022 17:01:06 -0800 Subject: [PATCH 13/18] test suite alternative baseline fixes for recent test renaming --- testing/btest/Baseline.opt/language.unused-assignment/out | 2 +- testing/btest/Baseline.zam/language.unused-assignment/out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/Baseline.opt/language.unused-assignment/out b/testing/btest/Baseline.opt/language.unused-assignment/out index dc6567e06a..3e67ae5ef6 100644 --- a/testing/btest/Baseline.opt/language.unused-assignment/out +++ b/testing/btest/Baseline.opt/language.unused-assignment/out @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. warning: "-O optimize-AST" option is incompatible with -u option, deactivating optimization -warning: please_warn assignment unused: please_warn = test; <...>/unused-assignement.zeek, line 7 +warning: please_warn assignment unused: please_warn = test; <...>/unused-assignment.zeek, line 7 diff --git a/testing/btest/Baseline.zam/language.unused-assignment/out b/testing/btest/Baseline.zam/language.unused-assignment/out index dc6567e06a..3e67ae5ef6 100644 --- a/testing/btest/Baseline.zam/language.unused-assignment/out +++ b/testing/btest/Baseline.zam/language.unused-assignment/out @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. warning: "-O optimize-AST" option is incompatible with -u option, deactivating optimization -warning: please_warn assignment unused: please_warn = test; <...>/unused-assignement.zeek, line 7 +warning: please_warn assignment unused: please_warn = test; <...>/unused-assignment.zeek, line 7 From 75b7bd0c6e5e2393758a15be355171312d7baeac Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 7 Jan 2022 11:50:21 -0700 Subject: [PATCH 14/18] Update zeek-aux submodule [nomail] [skip ci] --- auxil/zeek-aux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/zeek-aux b/auxil/zeek-aux index f6cdb4b3fa..0f120aa00c 160000 --- a/auxil/zeek-aux +++ b/auxil/zeek-aux @@ -1 +1 @@ -Subproject commit f6cdb4b3fa14065a135cbe8db0502419b3f9218b +Subproject commit 0f120aa00c2b666ed5c430a6bcf1043b82f17e64 From edf90a51e45c1b5b683243064748c66539d97598 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 13 Dec 2021 13:19:12 -0700 Subject: [PATCH 15/18] GH-1624: Migrate check_and_promote and a few Expr methods to IntrusivePtr --- src/Expr.cc | 85 +++++++++++++++----------------- src/Expr.h | 26 +++++----- src/Type.cc | 6 +-- src/Val.cc | 13 ++++- src/Val.h | 5 +- src/Var.cc | 4 +- src/script_opt/CPP/RuntimeOps.cc | 2 +- 7 files changed, 76 insertions(+), 65 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index c291f288ae..d996eadb88 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -263,7 +263,7 @@ bool Expr::InvertSense() return false; } -void Expr::EvalIntoAggregate(const zeek::Type* /* t */, Val* /* aggr */, Frame* /* f */) const +void Expr::EvalIntoAggregate(const TypePtr& /* t */, ValPtr /* aggr */, Frame* /* f */) const { Internal("Expr::EvalIntoAggregate called"); } @@ -416,7 +416,7 @@ bool Expr::IsPure() const return true; } -ValPtr Expr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr Expr::InitVal(const TypePtr& t, ValPtr aggr) const { if ( aggr ) { @@ -2637,7 +2637,7 @@ TypePtr AssignExpr::InitType() const return make_intrusive(IntrusivePtr{NewRef{}, tl->AsTypeList()}, op2->GetType()); } -void AssignExpr::EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const +void AssignExpr::EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const { if ( IsError() ) return; @@ -2677,7 +2677,7 @@ void AssignExpr::EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) con TableVal* tv = aggr->AsTableVal(); auto index = op1->Eval(f); - auto v = check_and_promote(op2->Eval(f), t->Yield().get(), true); + auto v = check_and_promote(op2->Eval(f), t->Yield(), true); if ( ! index || ! v ) return; @@ -2686,7 +2686,7 @@ void AssignExpr::EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) con RuntimeError("type clash in table assignment"); } -ValPtr AssignExpr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr AssignExpr::InitVal(const TypePtr& t, ValPtr aggr) const { if ( ! aggr ) { @@ -2703,7 +2703,7 @@ ValPtr AssignExpr::InitVal(const zeek::Type* t, ValPtr aggr) const { if ( t->Tag() != TYPE_RECORD ) { - Error("not a record initializer", t); + Error("not a record initializer", t.get()); return nullptr; } @@ -2721,7 +2721,7 @@ ValPtr AssignExpr::InitVal(const zeek::Type* t, ValPtr aggr) const RecordVal* aggr_r = aggr->AsRecordVal(); - auto v = op2->InitVal(rt->GetFieldType(td.id).get(), nullptr); + auto v = op2->InitVal(rt->GetFieldType(td.id), nullptr); if ( ! v ) return nullptr; @@ -2734,7 +2734,7 @@ ValPtr AssignExpr::InitVal(const zeek::Type* t, ValPtr aggr) const { if ( t->Tag() != TYPE_TABLE ) { - Error("not a table initialization", t); + Error("not a table initialization", t.get()); return nullptr; } @@ -2745,7 +2745,7 @@ ValPtr AssignExpr::InitVal(const zeek::Type* t, ValPtr aggr) const const TableType* tt = tv->GetType()->AsTableType(); const auto& yt = tv->GetType()->Yield(); - auto index = op1->InitVal(tt->GetIndices().get(), nullptr); + auto index = op1->InitVal(tt->GetIndices(), nullptr); if ( yt->Tag() == TYPE_RECORD ) { @@ -2776,7 +2776,7 @@ ValPtr AssignExpr::InitVal(const zeek::Type* t, ValPtr aggr) const } } - auto v = op2->InitVal(yt.get(), nullptr); + auto v = op2->InitVal(yt, nullptr); if ( ! index || ! v ) return nullptr; @@ -3372,7 +3372,7 @@ RecordConstructorExpr::RecordConstructorExpr(RecordTypePtr known_rt, ListExprPtr } } -ValPtr RecordConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr RecordConstructorExpr::InitVal(const TypePtr& t, ValPtr aggr) const { if ( IsError() ) { @@ -3388,8 +3388,7 @@ ValPtr RecordConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const if ( v ) { RecordVal* rv = v->AsRecordVal(); - auto bt = const_cast(t); - RecordTypePtr rt{NewRef{}, bt->AsRecordType()}; + RecordTypePtr rt{NewRef{}, t->AsRecordType()}; auto aggr_rec = cast_intrusive(std::move(aggr)); auto ar = rv->CoerceTo(std::move(rt), std::move(aggr_rec)); @@ -3573,14 +3572,14 @@ ValPtr TableConstructorExpr::Eval(Frame* f) const const ExprPList& exprs = op->AsListExpr()->Exprs(); for ( const auto& expr : exprs ) - expr->EvalIntoAggregate(type.get(), aggr.get(), f); + expr->EvalIntoAggregate(type, aggr, f); aggr->InitDefaultFunc(f); return aggr; } -ValPtr TableConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr TableConstructorExpr::InitVal(const TypePtr& t, ValPtr aggr) const { if ( IsError() ) return nullptr; @@ -3595,7 +3594,7 @@ ValPtr TableConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const const ExprPList& exprs = op->AsListExpr()->Exprs(); for ( const auto& expr : exprs ) - expr->EvalIntoAggregate(t, tval.get(), nullptr); + expr->EvalIntoAggregate(t, tval, nullptr); return tval; } @@ -3664,7 +3663,7 @@ SetConstructorExpr::SetConstructorExpr(ListExprPtr constructor_list, ListExpr* le = ce->AsListExpr(); assert(ce->Tag() == EXPR_LIST); - if ( check_and_promote_exprs(le, type->AsTableType()->GetIndices().get()) ) + if ( check_and_promote_exprs(le, type->AsTableType()->GetIndices()) ) { if ( le != cle[i] ) cle.replace(i, le); @@ -3694,7 +3693,7 @@ ValPtr SetConstructorExpr::Eval(Frame* f) const return aggr; } -ValPtr SetConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr SetConstructorExpr::InitVal(const TypePtr& t, ValPtr aggr) const { if ( IsError() ) return nullptr; @@ -3710,7 +3709,7 @@ ValPtr SetConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const for ( const auto& e : exprs ) { - auto element = check_and_promote(e->Eval(nullptr), index_type.get(), true); + auto element = check_and_promote(e->Eval(nullptr), index_type, true); if ( ! element || ! tval->Assign(std::move(element), nullptr) ) { @@ -3792,7 +3791,7 @@ ValPtr VectorConstructorExpr::Eval(Frame* f) const return vec; } -ValPtr VectorConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr VectorConstructorExpr::InitVal(const TypePtr& t, ValPtr aggr) const { if ( IsError() ) return nullptr; @@ -3808,7 +3807,7 @@ ValPtr VectorConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const loop_over_list(exprs, i) { Expr* e = exprs[i]; - auto v = check_and_promote(e->Eval(nullptr), t->Yield().get(), true); + auto v = check_and_promote(e->Eval(nullptr), t->Yield(), true); if ( ! v || ! vec->Assign(i, std::move(v)) ) { @@ -3839,7 +3838,7 @@ bool FieldAssignExpr::PromoteTo(TypePtr t) return op != nullptr; } -void FieldAssignExpr::EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const +void FieldAssignExpr::EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const { if ( IsError() ) return; @@ -3910,7 +3909,7 @@ ArithCoerceExpr::ArithCoerceExpr(ExprPtr arg_op, TypeTag t) ValPtr ArithCoerceExpr::FoldSingleVal(ValPtr v, const TypePtr& t) const { - return check_and_promote(v, t.get(), false, location); + return check_and_promote(v, t, false, location); } ValPtr ArithCoerceExpr::Fold(Val* v) const @@ -4047,7 +4046,7 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r) } } -ValPtr RecordCoerceExpr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr RecordCoerceExpr::InitVal(const TypePtr& t, ValPtr aggr) const { if ( IsError() ) { @@ -4061,8 +4060,7 @@ ValPtr RecordCoerceExpr::InitVal(const zeek::Type* t, ValPtr aggr) const if ( auto v = Eval(nullptr) ) { RecordVal* rv = v->AsRecordVal(); - auto bt = const_cast(t); - RecordTypePtr rt{NewRef{}, bt->AsRecordType()}; + RecordTypePtr rt{NewRef{}, t->AsRecordType()}; auto aggr_rec = cast_intrusive(std::move(aggr)); if ( auto ar = rv->CoerceTo(std::move(rt), std::move(aggr_rec)) ) @@ -4134,7 +4132,7 @@ RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v, const std::vector& else if ( BothArithmetic(rhs_type->Tag(), field_type->Tag()) && ! same_type(rhs_type, field_type) ) { - auto new_val = check_and_promote(rhs, field_type.get(), false); + auto new_val = check_and_promote(rhs, field_type, false); rhs = std::move(new_val); } @@ -4929,7 +4927,7 @@ TypePtr ListExpr::InitType() const } } -ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const +ValPtr ListExpr::InitVal(const TypePtr& t, ValPtr aggr) const { // While fairly similar to the EvalIntoAggregate() code, // we keep this separate since it also deals with initialization @@ -4947,13 +4945,13 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const if ( exprs.length() != static_cast(tl.size()) ) { - Error("index mismatch", t); + Error("index mismatch", t.get()); return nullptr; } loop_over_list(exprs, i) { - auto vi = exprs[i]->InitVal(tl[i].get(), nullptr); + auto vi = exprs[i]->InitVal(tl[i], nullptr); if ( ! vi ) return nullptr; @@ -4967,7 +4965,7 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const { if ( aggr ) { - Error("bad use of list in initialization", t); + Error("bad use of list in initialization", t.get()); return nullptr; } @@ -4975,7 +4973,7 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const if ( exprs.length() != static_cast(tl.size()) ) { - Error("index mismatch", t); + Error("index mismatch", t.get()); return nullptr; } @@ -4983,7 +4981,7 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const loop_over_list(exprs, i) { - auto vi = exprs[i]->InitVal(tl[i].get(), nullptr); + auto vi = exprs[i]->InitVal(tl[i], nullptr); if ( ! vi ) return nullptr; @@ -5001,7 +4999,7 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const return exprs[0]->InitVal(t, aggr); else { - Error("aggregate initializer for scalar type", t); + Error("aggregate initializer for scalar type", t.get()); return nullptr; } } @@ -5052,7 +5050,7 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const { if ( t->Tag() == TYPE_RECORD ) { - e->Error("bad record initializer", t); + e->Error("bad record initializer", t.get()); return nullptr; } @@ -5060,7 +5058,7 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const if ( ! same_type(v->GetType(), t) ) { - v->GetType()->Error("type clash in table initializer", t); + v->GetType()->Error("type clash in table initializer", t.get()); return nullptr; } @@ -5072,14 +5070,14 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const return aggr; } -ValPtr ListExpr::AddSetInit(const zeek::Type* t, ValPtr aggr) const +ValPtr ListExpr::AddSetInit(TypePtr t, ValPtr aggr) const { if ( aggr->GetType()->Tag() != TYPE_TABLE ) Internal("bad aggregate in ListExpr::AddSetInit"); TableVal* tv = aggr->AsTableVal(); const TableType* tt = tv->GetType()->AsTableType(); - const TypeList* it = tt->GetIndices().get(); + TypeListPtr it = tt->GetIndices(); for ( const auto& expr : exprs ) { @@ -5091,7 +5089,7 @@ ValPtr ListExpr::AddSetInit(const zeek::Type* t, ValPtr aggr) const else if ( expr->GetType()->Tag() == TYPE_LIST ) element = expr->InitVal(it, nullptr); else - element = expr->InitVal(it->GetTypes()[0].get(), nullptr); + element = expr->InitVal(it->GetTypes()[0], nullptr); if ( ! element ) return nullptr; @@ -5100,7 +5098,7 @@ ValPtr ListExpr::AddSetInit(const zeek::Type* t, ValPtr aggr) const { if ( ! same_type(element->GetType(), t) ) { - element->Error("type clash in set initializer", t); + element->Error("type clash in set initializer", t.get()); return nullptr; } @@ -5113,7 +5111,7 @@ ValPtr ListExpr::AddSetInit(const zeek::Type* t, ValPtr aggr) const if ( expr->GetType()->Tag() == TYPE_LIST ) element = check_and_promote(std::move(element), it, true); else - element = check_and_promote(std::move(element), it->GetTypes()[0].get(), true); + element = check_and_promote(std::move(element), it->GetTypes()[0], true); if ( ! element ) return nullptr; @@ -5389,7 +5387,7 @@ ExprPtr check_and_promote_expr(ExprPtr e, TypePtr t) return e; } -bool check_and_promote_exprs(ListExpr* const elements, TypeList* types) +bool check_and_promote_exprs(ListExpr* const elements, const TypeListPtr& types) { ExprPList& el = elements->Exprs(); const auto& tl = types->GetTypes(); @@ -5464,13 +5462,12 @@ bool check_and_promote_args(ListExpr* const args, const RecordType* types) el.push_back(def_elements[--ne].release()); } - TypeList* tl = new TypeList(); + auto tl = make_intrusive(); for ( int i = 0; i < types->NumFields(); ++i ) tl->Append(types->GetFieldType(i)); int rval = check_and_promote_exprs(args, tl); - Unref(tl); return rval; } diff --git a/src/Expr.h b/src/Expr.h index 83c34616f5..84b8b3d777 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -167,7 +167,7 @@ public: // into the given aggregate of the given type. Note that // return type is void since it's updating an existing // value, rather than creating a new one. - virtual void EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const; + virtual void EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const; // Assign to the given value, if appropriate. virtual void Assign(Frame* f, ValPtr v); @@ -187,7 +187,7 @@ public: // with the given type. If "aggr" is non-nil, then this expression // is an element of the given aggregate, and it is added to it // accordingly. - virtual ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const; + virtual ValPtr InitVal(const TypePtr& t, ValPtr aggr) const; // True if the expression has no side effects, false otherwise. virtual bool IsPure() const; @@ -939,10 +939,10 @@ public: const AttributesPtr& attrs = nullptr, bool type_check = true); ValPtr Eval(Frame* f) const override; - void EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const override; + void EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const override; TypePtr InitType() const override; bool IsRecordElement(TypeDecl* td) const override; - ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; + ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override; bool IsPure() const override; // Optimization-related: @@ -1149,7 +1149,7 @@ public: StmtPtr ReduceToSingletons(Reducer* c) override; protected: - ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; + ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override; void ExprDescribe(ODesc* d) const override; @@ -1175,7 +1175,7 @@ public: StmtPtr ReduceToSingletons(Reducer* c) override; protected: - ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; + ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override; void ExprDescribe(ODesc* d) const override; @@ -1200,7 +1200,7 @@ public: StmtPtr ReduceToSingletons(Reducer* c) override; protected: - ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; + ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override; void ExprDescribe(ODesc* d) const override; @@ -1220,7 +1220,7 @@ public: bool HasReducedOps(Reducer* c) const override; protected: - ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; + ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override; void ExprDescribe(ODesc* d) const override; }; @@ -1240,7 +1240,7 @@ public: // (in which case an error is reported). bool PromoteTo(TypePtr t); - void EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const override; + void EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const override; bool IsRecordElement(TypeDecl* td) const override; // Optimization-related: @@ -1282,7 +1282,7 @@ public: const std::vector& Map() const { return map; } protected: - ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; + ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override; ValPtr Fold(Val* v) const override; // For each super-record slot, gives subrecord slot with which to @@ -1470,7 +1470,7 @@ public: ValPtr Eval(Frame* f) const override; TypePtr InitType() const override; - ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; + ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override; ExprPtr MakeLvalue() override; void Assign(Frame* f, ValPtr v) override; @@ -1486,7 +1486,7 @@ public: StmtPtr ReduceToSingletons(Reducer* c) override; protected: - ValPtr AddSetInit(const zeek::Type* t, ValPtr aggr) const; + ValPtr AddSetInit(TypePtr t, ValPtr aggr) const; void ExprDescribe(ODesc* d) const override; @@ -1767,7 +1767,7 @@ ExprPtr get_assign_expr(ExprPtr op1, ExprPtr op2, bool is_init); */ extern ExprPtr check_and_promote_expr(ExprPtr e, TypePtr t); -extern bool check_and_promote_exprs(ListExpr* elements, TypeList* types); +extern bool check_and_promote_exprs(ListExpr* elements, const TypeListPtr& types); extern bool check_and_promote_args(ListExpr* args, const RecordType* types); extern bool check_and_promote_exprs_to_type(ListExpr* elements, TypePtr type); diff --git a/src/Type.cc b/src/Type.cc index 308b8b64e5..cb459533a6 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -345,8 +345,8 @@ int IndexType::MatchesIndex(detail::ListExpr* const index) const exprs[0]->GetType()->Tag() == TYPE_ADDR ) return MATCHES_INDEX_SCALAR; - return check_and_promote_exprs(index, GetIndices().get()) ? MATCHES_INDEX_SCALAR - : DOES_NOT_MATCH_INDEX; + return check_and_promote_exprs(index, GetIndices()) ? MATCHES_INDEX_SCALAR + : DOES_NOT_MATCH_INDEX; } void IndexType::Describe(ODesc* d) const @@ -590,7 +590,7 @@ SetType::SetType(TypeListPtr ind, detail::ListExprPtr arg_elements) { if ( indices ) { // We already have a type. - if ( ! check_and_promote_exprs(elements.get(), indices.get()) ) + if ( ! check_and_promote_exprs(elements.get(), indices) ) SetError(); } else diff --git a/src/Val.cc b/src/Val.cc index 3ad6169149..2b76743983 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3730,8 +3730,19 @@ void VectorVal::ValDescribe(ODesc* d) const d->Add("]"); } -ValPtr check_and_promote(ValPtr v, const Type* t, bool is_init, +ValPtr check_and_promote(ValPtr v, const TypePtr& t, bool is_init, const detail::Location* expr_location) + { + // Once 5.0 comes out, this function can merge with the deprecated one below it, and this + // pragma block can go away. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + return check_and_promote(v, t.get(), is_init, expr_location); +#pragma GCC diagnostic pop + } + +[[deprecated("Remove in v5.1. Use version that takes TypePtr instead.")]] ValPtr +check_and_promote(ValPtr v, const Type* t, bool is_init, const detail::Location* expr_location) { if ( ! v ) return nullptr; diff --git a/src/Val.h b/src/Val.h index 6ece8c90de..9f7d1bd018 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1699,8 +1699,11 @@ UNDERLYING_ACCESSOR_DEF(TypeVal, zeek::Type*, AsType) // exact match, returns it. If promotable, returns the promoted version. // If not a match, generates an error message and return nil. If is_init is // true, then the checking is done in the context of an initialization. -extern ValPtr check_and_promote(ValPtr v, const Type* t, bool is_init, +extern ValPtr check_and_promote(ValPtr v, const TypePtr& t, bool is_init, const detail::Location* expr_location = nullptr); +[[deprecated("Remove in v5.1. Use version that takes TypePtr instead.")]] extern ValPtr +check_and_promote(ValPtr v, const Type* t, bool is_init, + const detail::Location* expr_location = nullptr); extern bool same_val(const Val* v1, const Val* v2); extern bool same_atomic_val(const Val* v1, const Val* v2); diff --git a/src/Var.cc b/src/Var.cc index 5437b11c33..0797ba819d 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -23,7 +23,7 @@ namespace zeek::detail { -static ValPtr init_val(Expr* init, const Type* t, ValPtr aggr) +static ValPtr init_val(ExprPtr init, TypePtr t, ValPtr aggr) { try { @@ -289,7 +289,7 @@ static void make_var(const IDPtr& id, TypePtr t, InitClass c, ExprPtr init, if ( init ) { - v = init_val(init.get(), t.get(), aggr); + v = init_val(init, t, aggr); if ( ! v ) return; diff --git a/src/script_opt/CPP/RuntimeOps.cc b/src/script_opt/CPP/RuntimeOps.cc index 13cb3f5092..35dd89dc6f 100644 --- a/src/script_opt/CPP/RuntimeOps.cc +++ b/src/script_opt/CPP/RuntimeOps.cc @@ -197,7 +197,7 @@ TableValPtr set_constructor__CPP(vector elements, TableTypePtr t, vector TableValPtr table_constructor__CPP(vector indices, vector vals, TableTypePtr t, vector attr_tags, vector attr_vals) { - const auto& yt = t->Yield().get(); + const auto& yt = t->Yield(); auto n = indices.size(); auto attrs = build_attrs__CPP(move(attr_tags), move(attr_vals)); From 253630311e4779bb354d7716bd50cd105beb347a Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 7 Jan 2022 18:18:00 -0800 Subject: [PATCH 16/18] fix for avoiding de-ref of nil pointer --- src/Func.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Func.cc b/src/Func.cc index d50221132c..282c65dde8 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -706,8 +706,12 @@ broker::expected ScriptFunc::SerializeClosure() const { if ( captures_frame ) return captures_frame->SerializeCopyFrame(); - else + + if ( closure ) return closure->SerializeClosureFrame(outer_ids); + + // No captures/closures, return an empty vector. + return broker::vector{}; } void ScriptFunc::Describe(ODesc* d) const From e9803113d12d636a6ddab3c9f0035efe49fcc1b8 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 10 Jan 2022 10:22:32 -0700 Subject: [PATCH 17/18] NEWS: move note for http header changes into 4.2 --- NEWS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 6cdcafc0f2..9c5eec6d6f 100644 --- a/NEWS +++ b/NEWS @@ -15,10 +15,6 @@ New Functionality Changed Functionality --------------------- -- Entries in ``http.log`` now record the original ``HOST`` headers. - Previously, they would skip any port specification a header might - include. - Deprecated Functionality ------------------------ @@ -182,6 +178,10 @@ Changed Functionality ZeekControl's functionality. ZeekControl remains the recommended tool for maintaining your cluster. +- Entries in ``http.log`` now record the original ``HOST`` headers. + Previously, they would skip any port specification a header might + include. + Deprecated Functionality ------------------------ From 1dfc9176d52207b9a5198c6875ad96f7ead09324 Mon Sep 17 00:00:00 2001 From: zeek-bot Date: Tue, 11 Jan 2022 00:37:24 +0000 Subject: [PATCH 18/18] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 1263619ce3..d7a74c1b81 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 1263619ce3279415314355032b855206f7e3e632 +Subproject commit d7a74c1b81270fa98e48f89e9a2629516c8b39f7