Merge remote-tracking branch 'origin/topic/vern/id-redecl'

* origin/topic/vern/id-redecl:
  fixes for re-declaring type identifiers in inconsistent ways - addresses GH-2686

CHANGES

@@ -1,3 +1,21 @@
+8.1.0-dev.621 | 2025-09-30 20:46:27 +0000
+
+  * GH-2686: fixes for re-declaring type identifiers in inconsistent ways - addresses GH-2686 (Vern Paxson, Corelight)
+
+8.1.0-dev.619 | 2025-09-30 20:45:19 +0000
+
+  * Fix for standalone initializations that require BiFs, and streamlining of standalone BiF-tracking (Vern Paxson, Corelight)
+
+8.1.0-dev.617 | 2025-09-30 20:12:14 +0000
+
+  * fixed bug in logic for including/excluding files for script optimization (Vern Paxson, Corelight)
+
+8.1.0-dev.615 | 2025-09-30 19:12:05 +0000
+
+  * Remove checks for OpenSSL 1.x versions (Tim Wojtulewicz, Corelight)
+
+  * Remove some additional LibreSSL checks (Tim Wojtulewicz, Corelight)
+
 8.1.0-dev.612 | 2025-09-29 18:04:24 +0200
 
   * Supervisor: Make last_signal atomic to squelch data race (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-8.1.0-dev.612
+8.1.0-dev.621

src/OpaqueVal.cc

@@ -27,10 +27,6 @@
 #include "zeek/probabilistic/BloomFilter.h"
 #include "zeek/probabilistic/CardinalityCounter.h"
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-inline void* EVP_MD_CTX_md_data(const EVP_MD_CTX* ctx) { return ctx->md_data; }
-#endif
-
 #if ( OPENSSL_VERSION_NUMBER < 0x30000000L )
 #include <openssl/md5.h>
 #endif

src/Var.cc

@@ -348,6 +348,35 @@ extern ExprPtr add_and_assign_local(IDPtr id, ExprPtr init, ValPtr val) {
 }
 
 void add_type(ID* id, TypePtr t, std::unique_ptr<std::vector<AttrPtr>> attr) {
+    if ( const auto& old_t = id->GetType() ) {
+        // The identifier already has a type associated with it. This can
+        // be okay if (1) it's already been marked as a Type identifier,
+        // (2) the previous type is a stub, or an equivalent enum.
+        if ( ! id->IsType() ) {
+            reporter->Error("Identifier %s has already been declared and is not a type", id->Name());
+            return;
+        }
+
+        if ( old_t->Tag() == t->Tag() && ((old_t->Tag() == TYPE_RECORD && old_t->AsRecordType()->NumFields() == 0) ||
+                                          (t->Tag() == TYPE_ENUM && same_type(t, old_t))) )
+            // It has a consistent tag and is either redeclaring a stub
+            // record (used in init-bare.zeek) or an equivalent enum
+            // (which can appear due to specifiers in BiFs, for example).
+            ;
+
+        else {
+            std::string loc;
+            auto li = id->GetLocationInfo();
+            auto fn = li->FileName();
+            int ln = li->FirstLine();
+            if ( fn && fn[0] != '\0' )
+                loc = " at " + std::string(fn) + ":" + std::to_string(ln);
+
+            reporter->Error("Type %s has already been declared%s", id->Name(), loc.c_str());
+            return;
+        }
+    }
+
     std::string new_type_name = id->Name();
     std::string old_type_name = t->GetName();
 

src/digest.cc

@@ -12,11 +12,6 @@
 
 #include "zeek/Reporter.h"
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
-#define EVP_MD_CTX_new EVP_MD_CTX_create
-#define EVP_MD_CTX_free EVP_MD_CTX_destroy
-#endif
-
 static_assert(ZEEK_MD5_DIGEST_LENGTH == MD5_DIGEST_LENGTH);
 
 static_assert(ZEEK_SHA_DIGEST_LENGTH == SHA_DIGEST_LENGTH);

src/file_analysis/analyzer/x509/OCSP.cc

@@ -26,28 +26,11 @@ namespace zeek::file_analysis::detail {
 static constexpr size_t OCSP_STRING_BUF_SIZE = 2048;
 
 static bool OCSP_RESPID_bio(OCSP_BASICRESP* basic_resp, BIO* bio) {
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    ASN1_OCTET_STRING* key = nullptr;
-    X509_NAME* name = nullptr;
-
-    if ( ! basic_resp->tbsResponseData )
-        return false;
-
-    auto resp_id = basic_resp->tbsResponseData->responderId;
-
-    if ( resp_id->type == V_OCSP_RESPID_NAME )
-        name = resp_id->value.byName;
-    else if ( resp_id->type == V_OCSP_RESPID_KEY )
-        key = resp_id->value.byKey;
-    else
-        return false;
-#else
     const ASN1_OCTET_STRING* key = nullptr;
     const X509_NAME* name = nullptr;
 
     if ( ! OCSP_resp_get0_id(basic_resp, &key, &name) )
         return false;
-#endif
 
     if ( name )
         X509_NAME_print_ex(bio, name, 0, XN_FLAG_ONELINE);
@@ -150,8 +133,6 @@ bool OCSP::EndOfFile() {
     return true;
 }
 
-#if ( OPENSSL_VERSION_NUMBER >= 0x10100000L )
-
 struct ASN1Seq {
     ASN1Seq(const unsigned char** der_in, long length) { decoded = d2i_ASN1_SEQUENCE_ANY(nullptr, der_in, length); }
 
@@ -345,7 +326,6 @@ static uint64_t parse_request_version(OCSP_REQUEST* req) {
     OPENSSL_free(der_req_dat);
     return asn1_int;
 }
-#endif
 
 void OCSP::ParseRequest(OCSP_REQUEST* req) {
     char buf[OCSP_STRING_BUF_SIZE]; // we need a buffer for some of the openssl functions
@@ -353,13 +333,8 @@ void OCSP::ParseRequest(OCSP_REQUEST* req) {
 
     uint64_t version = 0;
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    if ( req->tbsRequest->version )
-        version = (uint64_t)ASN1_INTEGER_get(req->tbsRequest->version);
-#else
     version = parse_request_version(req);
     // TODO: try to parse out general name ?
-#endif
 
     if ( ocsp_request )
         event_mgr.Enqueue(ocsp_request, GetFile()->ToVal(), val_mgr->Count(version));
@@ -425,20 +400,10 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) {
     if ( ! basic_resp )
         goto clean_up;
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    resp_data = basic_resp->tbsResponseData;
-    if ( ! resp_data )
-        goto clean_up;
-#endif
-
     vl.emplace_back(GetFile()->ToVal());
     vl.emplace_back(std::move(status_val));
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    vl.emplace_back(val_mgr->Count((uint64_t)ASN1_INTEGER_get(resp_data->version)));
-#else
     vl.emplace_back(parse_basic_resp_data_version(basic_resp));
-#endif
 
     // responderID
     if ( OCSP_RESPID_bio(basic_resp, bio) ) {
@@ -452,11 +417,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) {
     }
 
     // producedAt
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    produced_at = resp_data->producedAt;
-#else
     produced_at = OCSP_resp_get0_produced_at(basic_resp);
-#endif
 
     vl.emplace_back(make_intrusive<TimeVal>(GetTimeFromAsn1(produced_at, GetFile(), reporter)));
 
@@ -477,11 +438,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) {
         // cert id
         const OCSP_CERTID* cert_id = nullptr;
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-        cert_id = single_resp->certId;
-#else
         cert_id = OCSP_SINGLERESP_get0_id(single_resp);
-#endif
 
         ocsp_add_cert_id(cert_id, &rvl, bio);
         BIO_reset(bio);
@@ -550,14 +507,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) {
         }
     }
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    i2a_ASN1_OBJECT(bio, basic_resp->signatureAlgorithm->algorithm);
-    len = BIO_read(bio, buf, sizeof(buf));
-    vl.emplace_back(make_intrusive<StringVal>(len, buf));
-    BIO_reset(bio);
-#else
     vl.emplace_back(parse_basic_resp_sig_alg(basic_resp, bio, buf, sizeof(buf)));
-#endif
 
     // i2a_ASN1_OBJECT(bio, basic_resp->signature);
     // len = BIO_read(bio, buf, sizeof(buf));
@@ -567,11 +517,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) {
     certs_vector = new VectorVal(id::find_type<VectorType>("x509_opaque_vector"));
     vl.emplace_back(AdoptRef{}, certs_vector);
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    certs = basic_resp->certs;
-#else
     certs = OCSP_resp_get0_certs(basic_resp);
-#endif
 
     if ( certs ) {
         int num_certs = sk_X509_num(certs);

src/file_analysis/analyzer/x509/X509.cc

@@ -161,13 +161,9 @@ RecordValPtr X509::ParseCertificate(X509Val* cert_val, file_analysis::File* f) {
 
     pX509Cert->Assign(7, buf);
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-    i2a_ASN1_OBJECT(bio, ssl_cert->sig_alg->algorithm);
-#else
     const ASN1_OBJECT* alg;
     X509_ALGOR_get0(&alg, nullptr, nullptr, X509_get0_tbs_sigalg(ssl_cert));
     i2a_ASN1_OBJECT(bio, alg);
-#endif
     len = BIO_gets(bio, buf, sizeof(buf));
     pX509Cert->Assign(13, make_intrusive<StringVal>(len, buf));
     BIO_free(bio);
@@ -349,11 +345,7 @@ void X509::ParseSAN(X509_EXTENSION* ext) {
             }
 
             auto len = ASN1_STRING_length(gen->d.ia5);
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
-            const char* name = (const char*)ASN1_STRING_data(gen->d.ia5);
-#else
             const char* name = (const char*)ASN1_STRING_get0_data(gen->d.ia5);
-#endif
             auto bs = make_intrusive<StringVal>(len, name);
 
             switch ( gen->type ) {

src/file_analysis/analyzer/x509/X509.h

@@ -9,55 +9,6 @@
 #include "zeek/OpaqueVal.h"
 #include "zeek/file_analysis/analyzer/x509/X509Common.h"
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10002000L )
-
-#define X509_get_signature_nid(x) OBJ_obj2nid((x)->sig_alg->algorithm)
-
-#endif
-
-#if ( OPENSSL_VERSION_NUMBER < 0x1010000fL )
-
-#define X509_OBJECT_new() (X509_OBJECT*)malloc(sizeof(X509_OBJECT))
-#define X509_OBJECT_free(a) free(a)
-
-#define OCSP_resp_get0_certs(x) (x)->certs
-
-#define EVP_PKEY_get0_DSA(p) ((p)->pkey.dsa)
-#define EVP_PKEY_get0_EC_KEY(p) ((p)->pkey.ec)
-#define EVP_PKEY_get0_RSA(p) ((p)->pkey.rsa)
-
-#if ! defined(LIBRESSL_VERSION_NUMBER) || (LIBRESSL_VERSION_NUMBER < 0x2070000fL)
-
-#define OCSP_SINGLERESP_get0_id(s) (s)->certId
-
-static X509* X509_OBJECT_get0_X509(const X509_OBJECT* a) {
-    if ( a == nullptr || a->type != X509_LU_X509 )
-        return nullptr;
-    return a->data.x509;
-}
-
-static void DSA_get0_pqg(const DSA* d, const BIGNUM** p, const BIGNUM** q, const BIGNUM** g) {
-    if ( p != nullptr )
-        *p = d->p;
-    if ( q != nullptr )
-        *q = d->q;
-    if ( g != nullptr )
-        *g = d->g;
-}
-
-static void RSA_get0_key(const RSA* r, const BIGNUM** n, const BIGNUM** e, const BIGNUM** d) {
-    if ( n != nullptr )
-        *n = r->n;
-    if ( e != nullptr )
-        *e = r->e;
-    if ( d != nullptr )
-        *d = r->d;
-}
-
-#endif
-
-#endif
-
 namespace zeek::file_analysis::detail {
 
 class X509Val;

src/file_analysis/analyzer/x509/functions.bif

@@ -65,19 +65,8 @@ X509* x509_get_ocsp_signer(const STACK_OF(X509)* certs,
 	const ASN1_OCTET_STRING* key  = nullptr;
 	const X509_NAME*         name = nullptr;
 
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-	OCSP_RESPID* resp_id = basic_resp->tbsResponseData->responderId;
-
-	if ( resp_id->type == V_OCSP_RESPID_NAME )
-		name = resp_id->value.byName;
-	else if ( resp_id->type == V_OCSP_RESPID_KEY )
-		key = resp_id->value.byKey;
-	else
-		return nullptr;
-#else
 	if ( ! OCSP_resp_get0_id(basic_resp, &key, &name) )
 		return nullptr;
-#endif
 
 	if ( name )
 		return X509_find_by_subject(const_cast<STACK_OF(X509)*>(certs),
@@ -359,11 +348,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
 
 	// Because we actually want to be able to give nice error messages that show why we were
 	// not able to verify the OCSP response - do our own verification logic first.
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-	signer = x509_get_ocsp_signer(basic->certs, basic);
-#else
 	signer = x509_get_ocsp_signer(OCSP_resp_get0_certs(basic), basic);
-#endif
 
 	/*
 	Do this perhaps - OpenSSL also cannot do it, so I do not really feel bad about it.
@@ -730,12 +715,7 @@ function sct_verify%(cert: opaque of x509, logid: string, log_key: string, signa
 	uint32_t cert_length;
 	if ( precert )
 		{
-#if ( OPENSSL_VERSION_NUMBER < 0x10002000L )
-		x->cert_info->enc.modified = 1;
-		cert_length = i2d_X509_CINF(x->cert_info, &cert_out);
-#else
 		cert_length = i2d_re_X509_tbs(x, &cert_out);
-#endif
 		data.append(reinterpret_cast<const char*>(issuer_key_hash->Bytes()), issuer_key_hash->Len());
 		}
 	else
@@ -1058,11 +1038,7 @@ function x509_check_cert_hostname%(cert_opaque: opaque of x509, hostname: string
 				continue;
 
 			std::size_t len = ASN1_STRING_length(gen->d.ia5);
-#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
-			auto* name = reinterpret_cast<const char*>(ASN1_STRING_data(gen->d.ia5));
-#else
 			auto* name = reinterpret_cast<const char*>(ASN1_STRING_get0_data(gen->d.ia5));
-#endif
 			std::string_view nameview {name, len};
 			if ( check_hostname(hostview, nameview) )
 				{

src/script_opt/CPP/Inits.cc

@@ -322,7 +322,9 @@ void CPPCompile::GenStandaloneActivation() {
     Emit("void standalone_init__CPP()");
     StartBlock();
     Emit("init__CPP();");
+    Emit("load_BiFs__CPP(); // support initializations that call BiFs ...");
     Emit("standalone_activation__CPP();");
+    Emit("// ... and later use of BiFs from plugins not initially available");
     Emit("standalone_finalizations.push_back(load_BiFs__CPP);");
     EndBlock();
 }

src/script_opt/ProfileFunc.cc

@@ -388,7 +388,10 @@ TraversalCode ProfileFunc::PreExpr(const Expr* e) {
                     auto sf = static_cast<ScriptFunc*>(func_vf);
                     script_calls.insert(sf);
                 }
-                else
+
+                // Track the BiF, though not if we know we're not going to
+                // compile the call to it.
+                else if ( obj_matches_opt_files(e) != AnalyzeDecision::SHOULD_NOT )
                     BiF_globals.insert(func);
             }
             else {

src/script_opt/ProfileFunc.h

@@ -256,7 +256,8 @@ protected:
     std::unordered_set<ScriptFunc*> script_calls;
 
     // Same for BiF's, though for them we record the corresponding global
-    // rather than the BuiltinFunc*.
+    // rather than the BuiltinFunc*. In addition, we only track BiFs germane
+    // to code we're compiling.
     IDSet BiF_globals;
 
     // Script functions appearing in "when" clauses.

src/script_opt/ScriptOpt.cc

@@ -160,15 +160,18 @@ AnalyzeDecision filename_matches_opt_files(const char* filename) {
 
     auto fin = util::detail::normalize_path(filename);
 
-    for ( auto& s : analysis_options.skip_files )
+    for ( auto& s : sfiles )
         if ( std::regex_match(fin, s) )
             return AnalyzeDecision::SHOULD_NOT;
 
+    if ( ofiles.empty() )
+        return AnalyzeDecision::DEFAULT;
+
     for ( auto& o : ofiles )
         if ( std::regex_match(fin, o) )
             return AnalyzeDecision::SHOULD;
 
-    return AnalyzeDecision::DEFAULT;
+    return AnalyzeDecision::SHOULD_NOT;
 }
 
 AnalyzeDecision obj_matches_opt_files(const Obj* obj) {

src/zeek-setup.cc

@@ -97,58 +97,6 @@ int perftools_leaks = 0;
 int perftools_profile = 0;
 #endif
 
-#if OPENSSL_VERSION_NUMBER < 0x10100000L
-struct CRYPTO_dynlock_value {
-    std::mutex mtx;
-};
-
-namespace {
-
-std::unique_ptr<std::mutex[]> ssl_mtx_tbl;
-
-void ssl_lock_fn(int mode, int n, const char*, int) {
-    if ( mode & CRYPTO_LOCK )
-        ssl_mtx_tbl[static_cast<size_t>(n)].lock();
-    else
-        ssl_mtx_tbl[static_cast<size_t>(n)].unlock();
-}
-
-CRYPTO_dynlock_value* ssl_dynlock_create(const char*, int) { return new CRYPTO_dynlock_value; }
-
-void ssl_dynlock_lock(int mode, CRYPTO_dynlock_value* ptr, const char*, int) {
-    if ( mode & CRYPTO_LOCK )
-        ptr->mtx.lock();
-    else
-        ptr->mtx.unlock();
-}
-
-void ssl_dynlock_destroy(CRYPTO_dynlock_value* ptr, const char*, int) { delete ptr; }
-
-void do_ssl_init() {
-    ERR_load_crypto_strings();
-    OPENSSL_add_all_algorithms_conf();
-    SSL_library_init();
-    SSL_load_error_strings();
-    ssl_mtx_tbl.reset(new std::mutex[CRYPTO_num_locks()]);
-    CRYPTO_set_locking_callback(ssl_lock_fn);
-    CRYPTO_set_dynlock_create_callback(ssl_dynlock_create);
-    CRYPTO_set_dynlock_lock_callback(ssl_dynlock_lock);
-    CRYPTO_set_dynlock_destroy_callback(ssl_dynlock_destroy);
-}
-
-void do_ssl_deinit() {
-    ERR_free_strings();
-    EVP_cleanup();
-    CRYPTO_cleanup_all_ex_data();
-    CRYPTO_set_locking_callback(nullptr);
-    CRYPTO_set_dynlock_create_callback(nullptr);
-    CRYPTO_set_dynlock_lock_callback(nullptr);
-    CRYPTO_set_dynlock_destroy_callback(nullptr);
-    ssl_mtx_tbl.reset();
-}
-
-} // namespace
-#else
 namespace {
 
 void do_ssl_init() { OPENSSL_init_ssl(0, nullptr); }
@@ -160,7 +108,6 @@ void do_ssl_deinit() {
 }
 
 } // namespace
-#endif
 
 zeek::ValManager* zeek::val_mgr = nullptr;
 zeek::packet_analysis::Manager* zeek::packet_mgr = nullptr;

testing/btest/Baseline/language.redeclaration-redefinition-errors-12/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/redeclaration-redefinition-errors.zeek, line 2: Identifier f has already been declared and is not a type

testing/btest/Baseline/language.redeclaration-redefinition-errors-13/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/redeclaration-redefinition-errors.zeek, line 2: Type f has already been declared at <...>/redeclaration-redefinition-errors.zeek:1

testing/btest/language/redeclaration-redefinition-errors.zeek

@@ -59,3 +59,11 @@ global f: function();
 global f = function() { };
 global f: hook();
 global f: event();
+
+# @TEST-START-NEXT
+global f = function() { };
+type f: bool;
+
+# @TEST-START-NEXT
+type f: record {};
+type f: bool;