Merge remote-tracking branch 'origin/topic/christian/mmdb-configurability'

* origin/topic/christian/mmdb-configurability:
  Modernize various C++/Zeek-isms in the MMDB code.
  Fix MMDB code to re-open explicitly opened DBs correctly
  Add btest to verify behavior of re-opened MMDBs opened directly via BIFs
  Simplify MMDB code by moving more lookup functionality into MMDB class
  Move MMDB logic out of mmdb.bif and into MMDB.cc/h.
  Fix mmdb.temporary-error testcase when MMDBs are installed on system
  Adapt MMDB BiF code to new script-layer variables
  Update btest baselines to reflect introduction of mmdb.bif
  Move MaxMind/GeoIP BiF functionality into separate file
  Provide script-level configurability of MaxMind DB placement on disk
  Sort toplevel .bif list in CMakeLists

CHANGES

@@ -1,3 +1,60 @@
+6.2.0-dev.375 | 2024-01-12 09:27:58 +0100
+
+  * Modernize various C++/Zeek-isms in the MMDB code. (Christian Kreibich, Corelight)
+
+  * Fix MMDB code to re-open explicitly opened DBs correctly (Christian Kreibich, Corelight)
+
+    The filename from which a DB first gets opened (either via an explicitly
+    specified filename, or via the path sequence now configurable at the script
+    layer) is now "sticky", meaning re-opening won't switch to a different file.
+
+    This was easiest by moving most state into the MMDB class itself. The previous
+    approach of tracking the two DB instances via a smart pointer and blowing the
+    pointed-to objects away as needed is now instead one of two objects fixed over
+    the lifetime of Zeek, able to open/close/reopen their underlying Maxmind DBs.
+
+    The MMDB class now only has one Lookup() method since there was no need to break
+    them apart -- it saves the return of a MMDB_lookup_result_s over the stack and
+    there's no need for throwing an exception.
+
+  * Add btest to verify behavior of re-opened MMDBs opened directly via BIFs (Christian Kreibich, Corelight)
+
+    The mmdb_open_location_db() and mmdb_open_asn_db() BiFs were untested, and Zeek
+    has a bug that makes any DBs opened that way fall back to looking up DBs via the
+    existing script-level config mechanism (via mmdb_dir), which is at least
+    unexpected and might well be unconfigured if somebody uses the direct BiFs.
+
+  * Simplify MMDB code by moving more lookup functionality into MMDB class (Christian Kreibich, Corelight)
+
+  * Move MMDB logic out of mmdb.bif and into MMDB.cc/h. (Christian Kreibich, Corelight)
+
+    This does not change the implementation except for some light renaming where
+    things are now naturally scoped within MMDB.cc.
+
+  * Fix mmdb.temporary-error testcase when MMDBs are installed on system (Christian Kreibich, Corelight)
+
+    The test would previously fail in settings where the user has Maxmind DBs
+    installed in the hardwired system locations, because the fallback logic still
+    picked those up.
+
+  * Adapt MMDB BiF code to new script-layer variables (Christian Kreibich, Corelight)
+
+  * Update btest baselines to reflect introduction of mmdb.bif (Christian Kreibich, Corelight)
+
+  * Move MaxMind/GeoIP BiF functionality into separate file (Christian Kreibich, Corelight)
+
+  * Provide script-level configurability of MaxMind DB placement on disk (Christian Kreibich, Corelight)
+
+    This lifts the list of fallback directories in which Zeek will look for Maxmind
+    DBs into the script layer, and makes the names of the DB files themselves
+    (previously hardwired) configurable as well.
+
+    This does not yet change the in-core code; that commit follows.
+
+  * Sort toplevel .bif list in CMakeLists (Christian Kreibich, Corelight)
+
+    Not important, but here it is safe to do so and the list is getting lengthy.
+
 6.2.0-dev.362 | 2024-01-11 20:17:45 +0100
 
   * GH-3540: Known: Keep &create_expire on local tables/sets valid (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-6.2.0-dev.362
+6.2.0-dev.375

scripts/base/init-bare.zeek

@@ -1107,6 +1107,25 @@ type geo_autonomous_system: record {
 ## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support.
 const mmdb_dir: string = "" &redef;
 
+## Default name of the MaxMind City database file:
+const mmdb_city_db: string = "GeoLite2-City.mmdb" &redef;
+## Default name of the MaxMind Country database file:
+const mmdb_country_db: string = "GeoLite2-Country.mmdb" &redef;
+## Default name of the MaxMind ASN database file:
+const mmdb_asn_db: string = "GeoLite2-ASN.mmdb" &redef;
+
+## Fallback locations for MaxMind databases. Zeek attempts these when
+## :zeek:see:`mmdb_dir` is not set, or it cannot read a DB file from it. For
+## geolocation lookups, Zeek will first attempt to locate the city database in
+## each of the fallback locations, and should this fail, attempt to locate the
+## country one.
+const mmdb_dir_fallbacks: vector of string = vector(
+	"/usr/share/GeoIP",
+	"/var/lib/GeoIP",
+	"/usr/local/share/GeoIP",
+	"/usr/local/var/GeoIP",
+) &redef;
+
 ## Sets the interval for MaxMind DB file staleness checks. When Zeek detects a
 ## change in inode or modification time, the database is re-opened. Setting
 ## a negative interval disables staleness checks.
@@ -2094,6 +2113,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 @load base/bif/supervisor.bif
 @load base/bif/packet_analysis.bif
 @load base/bif/CPP-load.bif
+@load base/bif/mmdb.bif
 
 ## Internal function.
 function add_interface(iold: string, inew: string): string

src/CMakeLists.txt

@@ -111,27 +111,28 @@ include(BifCl)
 set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc)
 
 set(BIF_SRCS
-    zeek.bif
     communityid.bif
-    stats.bif
-    event.bif
     const.bif
-    types.bif
+    event.bif
-    strings.bif
+    mmdb.bif
-    reporter.bif
     option.bif
-    # Note: the supervisor BIF file is treated like other top-level BIFs instead
+    reporter.bif
-    # of contained in its own subdirectory CMake logic because subdirectory BIFs
+    stats.bif
-    # are treated differently and don't support being called *during* parsing
+    strings.bif
-    # (e.g. within an @if directive).
+    types.bif
-    supervisor/supervisor.bif
+    zeek.bif
     # The packet analysis BIF is treated like other top-level BIFs because it's
     # needed before parsing the packet protocol scripts, which happen very near
     # to the start of parsing.
     packet_analysis/packet_analysis.bif
     # The C++ loading BIF is treated like other top-level BIFs to give us
     # flexibility regarding when it's called.
-    script_opt/CPP/CPP-load.bif)
+    script_opt/CPP/CPP-load.bif
+    # Note: the supervisor BIF file is treated like other top-level BIFs instead
+    # of contained in its own subdirectory CMake logic because subdirectory BIFs
+    # are treated differently and don't support being called *during* parsing
+    # (e.g. within an @if directive).
+    supervisor/supervisor.bif)
 
 foreach (bift ${BIF_SRCS})
     bif_target(${bift} "standard")
@@ -328,6 +329,7 @@ set(MAIN_SRCS
     IP.cc
     IPAddr.cc
     List.cc
+    MMDB.cc
     Reporter.cc
     NFA.cc
     NetVar.cc

src/Func.cc

@@ -52,6 +52,7 @@
 #include "supervisor.bif.func_h"
 #include "packet_analysis.bif.func_h"
 #include "CPP-load.bif.func_h"
+#include "mmdb.bif.func_h"
 
 #include "zeek.bif.func_def"
 #include "communityid.bif.func_def"
@@ -62,6 +63,7 @@
 #include "supervisor.bif.func_def"
 #include "packet_analysis.bif.func_def"
 #include "CPP-load.bif.func_def"
+#include "mmdb.bif.func_def"
 // clang-format on
 
 extern RETSIGTYPE sig_handler(int signo);
@@ -1049,6 +1051,7 @@ void init_primary_bifs() {
 
 #include "CPP-load.bif.func_init"
 #include "communityid.bif.func_init"
+#include "mmdb.bif.func_init"
 #include "option.bif.func_init"
 #include "packet_analysis.bif.func_init"
 #include "reporter.bif.func_init"

src/MMDB.cc

@@ -0,0 +1,357 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/MMDB.h"
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <chrono>
+
+#include "zeek/Func.h"
+#include "zeek/IPAddr.h"
+#include "zeek/ZeekString.h"
+
+namespace zeek {
+
+#ifdef USE_GEOIP
+
+static int msg_count = 0;
+static double msg_suppression_time = 0;
+static constexpr int msg_limit = 20;
+static constexpr double msg_suppression_duration = 300;
+
+LocDB mmdb_loc;
+AsnDB mmdb_asn;
+
+static void report_msg(const char* format, ...) {
+    if ( zeek::run_state::network_time > msg_suppression_time + msg_suppression_duration ) {
+        msg_count = 0;
+        msg_suppression_time = zeek::run_state::network_time;
+    }
+
+    if ( msg_count >= msg_limit )
+        return;
+
+    ++msg_count;
+
+    va_list al;
+    va_start(al, format);
+    std::string msg = zeek::util::vfmt(format, al);
+    va_end(al);
+
+    zeek::reporter->Info("%s", msg.data());
+}
+
+static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, int data_type) {
+    switch ( status ) {
+        case MMDB_SUCCESS:
+            if ( entry_data->has_data ) {
+                switch ( data_type ) {
+                    case MMDB_DATA_TYPE_UTF8_STRING:
+                        return zeek::make_intrusive<zeek::StringVal>(entry_data->data_size, entry_data->utf8_string);
+                        break;
+
+                    case MMDB_DATA_TYPE_DOUBLE:
+                        return zeek::make_intrusive<zeek::DoubleVal>(entry_data->double_value);
+                        break;
+
+                    case MMDB_DATA_TYPE_UINT32: return zeek::val_mgr->Count(entry_data->uint32);
+
+                    default: break;
+                }
+            }
+            break;
+
+        case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR:
+            // key doesn't exist, nothing to do
+            break;
+
+        default: report_msg("MaxMind DB error [%s]", MMDB_strerror(status)); break;
+    }
+
+    return nullptr;
+}
+
+MMDB::MMDB() : mmdb{}, file_info{}, reported_error{false}, last_check{zeek::run_state::network_time} {}
+
+MMDB::~MMDB() { Close(); }
+
+bool MMDB::OpenFile(const std::string& a_filename) {
+    filename = a_filename;
+    Close();
+
+    if ( 0 != stat(filename.data(), &file_info) ) {
+        return false;
+    }
+
+    int status = MMDB_open(a_filename.data(), MMDB_MODE_MMAP, &mmdb);
+
+    if ( MMDB_SUCCESS != status ) {
+        memset(&mmdb, 0, sizeof(mmdb));
+        report_msg("Failed to open MaxMind DB: %s [%s]", filename.data(), MMDB_strerror(status));
+        return false;
+    }
+
+    return true;
+}
+
+void MMDB::Close() {
+    if ( IsOpen() ) {
+        MMDB_close(&mmdb);
+        memset(&mmdb, 0, sizeof(mmdb));
+        reported_error = false;
+    }
+}
+
+bool MMDB::EnsureLoaded() {
+    bool res = true;
+
+    if ( filename.empty() )
+        res = OpenFromScriptConfig();
+    else if ( ! IsOpen() )
+        res = OpenFile(filename);
+    else if ( IsStaleDB() ) {
+        report_msg("Closing stale MaxMind DB [%s]", filename.data());
+        if ( ! OpenFile(filename) )
+            res = false;
+    }
+
+    if ( ! res && ! reported_error ) {
+        reported_error = true;
+        zeek::emit_builtin_error(zeek::util::fmt("Failed to open %s", Description().data()));
+    }
+
+    return res;
+}
+
+bool MMDB::Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) {
+    struct sockaddr_storage ss = {0};
+
+    if ( IPv4 == addr.GetFamily() ) {
+        struct sockaddr_in* sa = (struct sockaddr_in*)&ss;
+        sa->sin_family = AF_INET;
+        addr.CopyIPv4(&sa->sin_addr);
+    }
+    else {
+        struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss;
+        sa->sin6_family = AF_INET6;
+        addr.CopyIPv6(&sa->sin6_addr);
+    }
+
+    int mmdb_error;
+    result = MMDB_lookup_sockaddr(&mmdb, (struct sockaddr*)&ss, &mmdb_error);
+
+    if ( MMDB_SUCCESS != mmdb_error ) {
+        report_msg("MaxMind DB lookup location error [%s]", MMDB_strerror(mmdb_error));
+        Close();
+        return false;
+    }
+
+    return true;
+}
+
+// Check to see if the Maxmind DB should be closed and reopened.  This will
+// happen if there was a lookup error or if the mmap'd file has been replaced
+// by an external process.
+bool MMDB::IsStaleDB() {
+    if ( ! IsOpen() )
+        return false;
+
+    static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval();
+
+    if ( mmdb_stale_check_interval < 0.0 )
+        return false;
+
+    if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval )
+        return false;
+
+    last_check = zeek::run_state::network_time;
+    struct stat buf;
+
+    if ( 0 != stat(mmdb.filename, &buf) )
+        return true;
+
+    if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) {
+        report_msg("%s change detected for MaxMind DB [%s]",
+                   buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", mmdb.filename);
+        return true;
+    }
+
+    return false;
+}
+
+bool LocDB::OpenFromScriptConfig() {
+    // City database is always preferred over Country database.
+    const auto& mmdb_dir_val = zeek::id::find_val<StringVal>("mmdb_dir");
+    std::string mmdb_dir{mmdb_dir_val->ToStdStringView()};
+
+    const auto& mmdb_city_db_val = zeek::id::find_val<StringVal>("mmdb_city_db");
+    std::string mmdb_city_db{mmdb_city_db_val->ToStdStringView()};
+
+    const auto& mmdb_country_db_val = zeek::id::find_val<StringVal>("mmdb_country_db");
+    std::string mmdb_country_db{mmdb_country_db_val->ToStdStringView()};
+
+    if ( ! mmdb_dir.empty() ) {
+        auto d = mmdb_dir + "/" + mmdb_city_db;
+
+        if ( OpenFile(d) )
+            return true;
+
+        d = mmdb_dir + "/" + mmdb_country_db;
+
+        if ( OpenFile(d) )
+            return true;
+    }
+
+    const auto& mmdb_dir_fallbacks_val = zeek::id::find_val<VectorVal>("mmdb_dir_fallbacks");
+
+    for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) {
+        auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_city_db;
+        if ( OpenFile(d) )
+            return true;
+    }
+
+    for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) {
+        auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_country_db;
+        if ( OpenFile(d) )
+            return true;
+    }
+
+    return false;
+}
+bool AsnDB::OpenFromScriptConfig() {
+    const auto& mmdb_dir_val = zeek::id::find_val<StringVal>("mmdb_dir");
+    std::string mmdb_dir{mmdb_dir_val->ToStdStringView()};
+
+    const auto& mmdb_asn_db_val = zeek::id::find_val<StringVal>("mmdb_asn_db");
+    std::string mmdb_asn_db{mmdb_asn_db_val->ToStdStringView()};
+
+    if ( ! mmdb_dir.empty() ) {
+        auto d = mmdb_dir + "/" + mmdb_asn_db;
+
+        if ( OpenFile(d) )
+            return true;
+    }
+
+    const auto& mmdb_dir_fallbacks_val = zeek::id::find_val<VectorVal>("mmdb_dir_fallbacks");
+
+    for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) {
+        auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_asn_db;
+        if ( OpenFile(d) )
+            return true;
+    }
+
+    return false;
+}
+#endif // USE_GEOIP
+
+ValPtr mmdb_open_location_db(const StringValPtr& filename) {
+#ifdef USE_GEOIP
+    return zeek::val_mgr->Bool(mmdb_loc.OpenFile(filename->ToStdString()));
+#else
+    return zeek::val_mgr->False();
+#endif
+}
+
+ValPtr mmdb_open_asn_db(const StringValPtr& filename) {
+#ifdef USE_GEOIP
+    return zeek::val_mgr->Bool(mmdb_asn.OpenFile(filename->ToStdString()));
+#else
+    return zeek::val_mgr->False();
+#endif
+}
+
+RecordValPtr mmdb_lookup_location(const AddrValPtr& addr) {
+    static auto geo_location = zeek::id::find_type<zeek::RecordType>("geo_location");
+    auto location = zeek::make_intrusive<zeek::RecordVal>(geo_location);
+
+#ifdef USE_GEOIP
+    if ( ! mmdb_loc.EnsureLoaded() )
+        return location;
+
+    MMDB_lookup_result_s result;
+
+    if ( mmdb_loc.Lookup(addr->AsAddr(), result) ) {
+        MMDB_entry_data_s entry_data;
+        int status;
+
+        // Get Country ISO Code
+        status = MMDB_get_value(&result.entry, &entry_data, "country", "iso_code", nullptr);
+        location->Assign(0, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING));
+
+        // Get Major Subdivision ISO Code
+        status = MMDB_get_value(&result.entry, &entry_data, "subdivisions", "0", "iso_code", nullptr);
+        location->Assign(1, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING));
+
+        // Get City English Name
+        status = MMDB_get_value(&result.entry, &entry_data, "city", "names", "en", nullptr);
+        location->Assign(2, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING));
+
+        // Get Location Latitude
+        status = MMDB_get_value(&result.entry, &entry_data, "location", "latitude", nullptr);
+        location->Assign(3, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_DOUBLE));
+
+        // Get Location Longitude
+        status = MMDB_get_value(&result.entry, &entry_data, "location", "longitude", nullptr);
+        location->Assign(4, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_DOUBLE));
+
+        return location;
+    }
+
+#else // not USE_GEOIP
+    static int missing_geoip_reported = 0;
+
+    if ( ! missing_geoip_reported ) {
+        zeek::emit_builtin_error("Zeek was not configured for GeoIP support");
+        missing_geoip_reported = 1;
+    }
+#endif
+
+    // We can get here even if we have MMDB support if we weren't
+    // able to initialize it or it didn't return any information for
+    // the address.
+    return location;
+}
+
+RecordValPtr mmdb_lookup_autonomous_system(const AddrValPtr& addr) {
+    static auto geo_autonomous_system = zeek::id::find_type<zeek::RecordType>("geo_autonomous_system");
+    auto autonomous_system = zeek::make_intrusive<zeek::RecordVal>(geo_autonomous_system);
+
+#ifdef USE_GEOIP
+    if ( ! mmdb_asn.EnsureLoaded() )
+        return autonomous_system;
+
+    MMDB_lookup_result_s result;
+
+    if ( mmdb_asn.Lookup(addr->AsAddr(), result) ) {
+        MMDB_entry_data_s entry_data;
+        int status;
+
+        // Get Autonomous System Number
+        status = MMDB_get_value(&result.entry, &entry_data, "autonomous_system_number", nullptr);
+        autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UINT32));
+
+        // Get Autonomous System Organization
+        status = MMDB_get_value(&result.entry, &entry_data, "autonomous_system_organization", nullptr);
+        autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING));
+
+        return autonomous_system;
+    }
+
+#else // not USE_GEOIP
+    static int missing_geoip_reported = 0;
+
+    if ( ! missing_geoip_reported ) {
+        zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support");
+        missing_geoip_reported = 1;
+    }
+#endif
+
+    // We can get here even if we have GeoIP support, if we weren't
+    // able to initialize it or it didn't return any information for
+    // the address.
+    return autonomous_system;
+}
+
+} // namespace zeek

src/MMDB.h

@@ -0,0 +1,91 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/Val.h"
+
+namespace zeek {
+
+#ifdef USE_GEOIP
+
+#include <maxminddb.h>
+
+// The MMDB class encapsulates a low-level libmaxmind MMDB_s structure. It
+// tracks whether that DB is currently loaded, and can open it from a file in
+// two ways: (1) via explicit specification of a filename, (2) by determining
+// the configuration from configuration settings in the script layer (mmdb_dir
+// etc). This configuration depends on whether this is a geolocation DB or an
+// ASN one, so details are left to derived classes below that specialize.
+//
+// The class tracks the inode and modification time of a DB file to detect
+// "stale" DBs, which get reloaded (from the same location in the file system)
+// upon the first lookup that detects staleness.
+class MMDB {
+public:
+    MMDB();
+    virtual ~MMDB();
+
+    // Implements the logic to determine a file system path for the DB from
+    // script-layer configuration settings, and opens the DB from there. Returns
+    // true if successful, false otherwise.
+    virtual bool OpenFromScriptConfig() = 0;
+
+    // Helper string to identify the type of DB, useful in error messages.
+    virtual std::string_view Description() = 0;
+
+    // Opens the DB at the given location, closing and cleaning up any currently
+    // opened DB if there is one. Returns true if successful, false otherwise.
+    bool OpenFile(const std::string& filename);
+
+    // Closes a currently opened DB, releasing its state. Safe to call on a
+    // closed DB.
+    void Close();
+
+    // Predicate; returns true if the DB is currently opened.
+    bool IsOpen() const { return mmdb.filename != nullptr; }
+
+    // Ensures that upon return the underlying DB file is loaded. When no
+    // filename is configured for the DB (i.e. OpenFile() has never been called
+    // on it), this triggers the script-level configuration lookup via
+    // OpenFromScriptConfig(). When a filename is available but it's not
+    // currently loaded, it does so. Finally, if there's a loaded DB but it's
+    // found to be stale, it gets reloaded. When the load operation succeeds, or
+    // the DB was already loaded and not stale, this returns true, and false if
+    // anything went wrong.
+    bool EnsureLoaded();
+
+    // Looks up a given IP address in the DB, storing the result in the provided
+    // result structure.
+    bool Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result);
+
+private:
+    bool IsStaleDB();
+
+    std::string filename;
+    MMDB_s mmdb;
+    struct stat file_info;
+    bool reported_error; // to ensure we emit builtin errors during opening only once.
+    double last_check;
+};
+
+class LocDB : public MMDB {
+public:
+    bool OpenFromScriptConfig();
+    std::string_view Description() { return "GeoIP location database"; }
+};
+
+class AsnDB : public MMDB {
+public:
+    bool OpenFromScriptConfig();
+    std::string_view Description() { return "GeoIP ASN database"; }
+};
+
+#endif // USE_GEOIP
+
+ValPtr mmdb_open_location_db(const StringValPtr& filename);
+ValPtr mmdb_open_asn_db(const StringValPtr& filename);
+
+RecordValPtr mmdb_lookup_location(const AddrValPtr& addr);
+RecordValPtr mmdb_lookup_autonomous_system(const AddrValPtr& addr);
+
+} // namespace zeek

src/mmdb.bif

@@ -0,0 +1,55 @@
+%%{
+#include <zeek/MMDB.h>
+%%}
+
+## Initializes MMDB for later use of lookup_location.
+## Requires Zeek to be built with ``libmaxminddb``.
+##
+## f: The filename of the MaxMind City or Country DB.
+##
+## Returns: A boolean indicating whether the db was successfully opened.
+##
+## .. zeek:see:: lookup_autonomous_system
+function mmdb_open_location_db%(f: string%) : bool
+	%{
+	return zeek::mmdb_open_location_db(StringValPtr(NewRef(), f));
+	%}
+
+## Initializes MMDB for later use of lookup_autonomous_system.
+## Requires Zeek to be built with ``libmaxminddb``.
+##
+## f: The filename of the MaxMind ASN DB.
+##
+## Returns: A boolean indicating whether the db was successfully opened.
+##
+## .. zeek:see:: lookup_autonomous_system
+function mmdb_open_asn_db%(f: string%) : bool
+	%{
+	return zeek::mmdb_open_asn_db(StringValPtr(NewRef(), f));
+	%}
+
+## Performs a geo-lookup of an IP address.
+## Requires Zeek to be built with ``libmaxminddb``.
+##
+## a: The IP address to lookup.
+##
+## Returns: A record with country, region, city, latitude, and longitude.
+##
+## .. zeek:see:: lookup_autonomous_system
+function lookup_location%(a: addr%) : geo_location
+	%{
+	return zeek::mmdb_lookup_location(AddrValPtr(NewRef(), a));
+	%}
+
+## Performs an lookup of AS number & organization of an IP address.
+## Requires Zeek to be built with ``libmaxminddb``.
+##
+## a: The IP address to lookup.
+##
+## Returns: A record with autonomous system number and organization that contains *a*.
+##
+## .. zeek:see:: lookup_location
+function lookup_autonomous_system%(a: addr%) : geo_autonomous_system
+	%{
+	return zeek::mmdb_lookup_autonomous_system(AddrValPtr(NewRef(), a));
+	%}

src/zeek.bif

@@ -3980,525 +3980,6 @@ function lookup_hostname%(host: string%) : addr_set
 	return nullptr;
 	%}
 
-%%{
-#ifdef USE_GEOIP
-#include <chrono>
-
-extern "C" {
-#include <maxminddb.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/ip.h>
-}
-
-static int mmdb_msg_count = 0;
-static constexpr int mmdb_msg_limit = 20;
-static double mmdb_msg_suppression_time = 0;
-static constexpr double mmdb_msg_suppression_duration = 300;
-
-static void report_mmdb_msg(const char* format, ...)
-	{
-	if ( zeek::run_state::network_time > mmdb_msg_suppression_time + mmdb_msg_suppression_duration )
-		{
-		mmdb_msg_count = 0;
-		mmdb_msg_suppression_time = zeek::run_state::network_time;
-		}
-
-	if ( mmdb_msg_count >= mmdb_msg_limit )
-		return;
-
-	++mmdb_msg_count;
-
-	va_list al;
-	va_start(al, format);
-	std::string msg = zeek::util::vfmt(format, al);
-	va_end(al);
-
-	zeek::reporter->Info("%s", msg.data());
-	}
-
-class MMDB {
-public:
-	MMDB(const char* filename, struct stat info);
-
-	~MMDB();
-
-	MMDB_lookup_result_s Lookup(const struct sockaddr* const sa);
-	bool StaleDB();
-	const char* Filename();
-
-private:
-	MMDB_s mmdb;
-	struct stat file_info;
-	bool lookup_error;
-	double last_check;
-};
-
-MMDB::MMDB(const char* filename, struct stat info)
-	: file_info(info), lookup_error{false},
-	  last_check{zeek::run_state::network_time}
-	{
-	int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb);
-
-	if ( MMDB_SUCCESS != status )
-		{
-		throw std::runtime_error(MMDB_strerror(status));
-		}
-	}
-
-MMDB::~MMDB()
-	{
-	MMDB_close(&mmdb);
-	}
-
-MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa)
-	{
-	int mmdb_error;
-	MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error);
-
-	if ( MMDB_SUCCESS != mmdb_error )
-		{
-		lookup_error = true;
-		throw std::runtime_error(MMDB_strerror(mmdb_error));
-		}
-
-	return result;
-	}
-
-// Check to see if the Maxmind DB should be closed and reopened.  This will
-// happen if there was a lookup error or if the mmap'd file has been replaced
-// by an external process.
-bool MMDB::StaleDB()
-	{
-	struct stat buf;
-
-	if ( lookup_error )
-		return true;
-
-
-	static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval();
-
-	if ( mmdb_stale_check_interval < 0.0 )
-		return false;
-
-	if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval )
-		return false;
-
-	last_check = zeek::run_state::network_time;
-
-	if ( 0 != stat(mmdb.filename, &buf) )
-		return true;
-
-	if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime )
-		{
-		report_mmdb_msg("%s change detected for MaxMind DB [%s]",
-				buf.st_ino != file_info.st_ino ? "Inode" : "Modification time",
-				mmdb.filename);
-		return true;
-		}
-
-	return false;
-	}
-
-const char* MMDB::Filename()
-	{
-	return mmdb.filename;
-	}
-
-std::unique_ptr<MMDB> mmdb_loc;
-std::unique_ptr<MMDB> mmdb_asn;
-static bool did_mmdb_loc_db_error = false;
-static bool did_mmdb_asn_db_error = false;
-
-static bool mmdb_open(const char* filename, bool asn)
-	{
-	struct stat buf;
-
-	if ( 0 != stat(filename, &buf) )
-		{
-		return false;
-		}
-
-	try
-		{
-		if ( asn )
-			{
-			mmdb_asn.reset(new MMDB(filename, buf));
-			}
-		else
-			{
-			mmdb_loc.reset(new MMDB(filename, buf));
-			}
-		}
-
-	catch ( const std::exception& e )
-		{
-		if ( asn )
-			did_mmdb_asn_db_error = false;
-		else
-			did_mmdb_loc_db_error = false;
-
-		report_mmdb_msg("Failed to open MaxMind DB: %s [%s]", filename,
-		                e.what());
-		return false;
-		}
-
-	return true;
-	}
-
-static bool mmdb_open_loc(const char* filename)
-	{
-	return mmdb_open(filename, false);
-	}
-
-static bool mmdb_open_asn(const char* filename)
-	{
-	return mmdb_open(filename, true);
-	}
-
-static void mmdb_check_loc()
-	{
-	if ( mmdb_loc && mmdb_loc->StaleDB() )
-		{
-		report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename());
-		did_mmdb_loc_db_error = false;
-		mmdb_loc.reset();
-		}
-	}
-
-static void mmdb_check_asn()
-	{
-	if ( mmdb_asn && mmdb_asn->StaleDB() )
-		{
-		report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename());
-		did_mmdb_asn_db_error = false;
-		mmdb_asn.reset();
-		}
-	}
-
-static bool mmdb_lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result,
-                        bool asn)
-	{
-	struct sockaddr_storage ss = {0};
-
-	if ( IPv4 == addr.GetFamily() )
-		{
-		struct sockaddr_in* sa = (struct sockaddr_in*)&ss;
-		sa->sin_family = AF_INET;
-		addr.CopyIPv4(&sa->sin_addr);
-		}
-
-	else
-		{
-		struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss;
-		sa->sin6_family = AF_INET6;
-		addr.CopyIPv6(&sa->sin6_addr);
-		}
-
-	try
-		{
-		result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss)
-		             : mmdb_loc->Lookup((struct sockaddr*)&ss);
-		}
-
-	catch ( const std::exception& e )
-		{
-		report_mmdb_msg("MaxMind DB lookup location error [%s]", e.what());
-		return false;
-		}
-
-	return result.found_entry;
-	}
-
-static bool mmdb_lookup_loc(const zeek::IPAddr& addr, MMDB_lookup_result_s& result)
-	{
-	return mmdb_lookup(addr, result, false);
-	}
-
-static bool mmdb_lookup_asn(const zeek::IPAddr& addr, MMDB_lookup_result_s& result)
-	{
-	return mmdb_lookup(addr, result, true);
-	}
-
-static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status,
-                            int data_type )
-	{
-	switch (status)
-		{
-		case MMDB_SUCCESS:
-			if ( entry_data->has_data )
-				{
-				switch (data_type)
-					{
-					case MMDB_DATA_TYPE_UTF8_STRING:
-						return zeek::make_intrusive<zeek::StringVal>(
-						    entry_data->data_size, entry_data->utf8_string);
-						break;
-
-					case MMDB_DATA_TYPE_DOUBLE:
-						return zeek::make_intrusive<zeek::DoubleVal>(entry_data->double_value);
-						break;
-
-					case MMDB_DATA_TYPE_UINT32:
-						return zeek::val_mgr->Count(entry_data->uint32);
-
-					default:
-						break;
-					}
-				}
-			break;
-
-		case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR:
-			// key doesn't exist, nothing to do
-			break;
-
-		default:
-			report_mmdb_msg("MaxMind DB error [%s]", MMDB_strerror(status));
-			break;
-		}
-
-	return nullptr;
-	}
-
-static bool mmdb_try_open_loc ()
-	{
-	// City database is always preferred over Country database.
-	const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal();
-	std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString();
-
-	if ( ! mmdb_dir.empty() )
-		{
-		auto d = mmdb_dir + "/GeoLite2-City.mmdb";
-
-		if ( mmdb_open_loc(d.data()) )
-			return true;
-
-		d = mmdb_dir + "/GeoLite2-Country.mmdb";
-
-		if ( mmdb_open_loc(d.data()) )
-			return true;;
-		}
-
-	return mmdb_open_loc("/usr/share/GeoIP/GeoLite2-City.mmdb")
-	    || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-City.mmdb")
-	    || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-City.mmdb")
-	    || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-City.mmdb")
-	    || mmdb_open_loc("/usr/share/GeoIP/GeoLite2-Country.mmdb")
-	    || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-Country.mmdb")
-	    || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-Country.mmdb")
-	    || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-Country.mmdb");
-	}
-
-static bool mmdb_try_open_asn ()
-	{
-	const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal();
-	std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString();
-
-	if ( ! mmdb_dir.empty() )
-		{
-		auto d = mmdb_dir + "/GeoLite2-ASN.mmdb";
-
-		if ( mmdb_open_asn(d.data()) )
-			return true;
-		}
-
-	return mmdb_open_asn("/usr/share/GeoIP/GeoLite2-ASN.mmdb")
-	    || mmdb_open_asn("/var/lib/GeoIP/GeoLite2-ASN.mmdb")
-	    || mmdb_open_asn("/usr/local/share/GeoIP/GeoLite2-ASN.mmdb")
-	    || mmdb_open_asn("/usr/local/var/GeoIP/GeoLite2-ASN.mmdb");
-	}
-
-#endif
-%%}
-
-## Initializes MMDB for later use of lookup_location.
-## Requires Zeek to be built with ``libmaxminddb``.
-##
-## f: The filename of the MaxMind City or Country DB.
-##
-## Returns: A boolean indicating whether the db was successfully opened.
-##
-## .. zeek:see:: lookup_autonomous_system
-function mmdb_open_location_db%(f: string%) : bool
-	%{
-#ifdef USE_GEOIP
-	return zeek::val_mgr->Bool(mmdb_open_loc(f->CheckString()));
-#else
-	return zeek::val_mgr->False();
-#endif
-	%}
-
-## Initializes MMDB for later use of lookup_autonomous_system.
-## Requires Zeek to be built with ``libmaxminddb``.
-##
-## f: The filename of the MaxMind ASN DB.
-##
-## Returns: A boolean indicating whether the db was successfully opened.
-##
-## .. zeek:see:: lookup_autonomous_system
-function mmdb_open_asn_db%(f: string%) : bool
-	%{
-#ifdef USE_GEOIP
-	return zeek::val_mgr->Bool(mmdb_open_asn(f->CheckString()));
-#else
-	return zeek::val_mgr->False();
-#endif
-	%}
-
-## Performs a geo-lookup of an IP address.
-## Requires Zeek to be built with ``libmaxminddb``.
-##
-## a: The IP address to lookup.
-##
-## Returns: A record with country, region, city, latitude, and longitude.
-##
-## .. zeek:see:: lookup_autonomous_system
-function lookup_location%(a: addr%) : geo_location
-	%{
-	static auto geo_location = zeek::id::find_type<zeek::RecordType>("geo_location");
-	auto location = zeek::make_intrusive<zeek::RecordVal>(geo_location);
-
-#ifdef USE_GEOIP
-	mmdb_check_loc();
-	if ( ! mmdb_loc )
-		{
-		if ( ! mmdb_try_open_loc() )
-			{
-			if ( ! did_mmdb_loc_db_error )
-				{
-				did_mmdb_loc_db_error = true;
-				zeek::emit_builtin_error("Failed to open GeoIP location database");
-				}
-
-			return location;
-			}
-		}
-
-	MMDB_lookup_result_s result;
-
-	if ( mmdb_lookup_loc(a->AsAddr(), result) )
-		{
-		MMDB_entry_data_s entry_data;
-		int status;
-
-		// Get Country ISO Code
-		status = MMDB_get_value(&result.entry, &entry_data,
-		                        "country", "iso_code", nullptr);
-		location->Assign(0, mmdb_getvalue(&entry_data, status,
-		                 MMDB_DATA_TYPE_UTF8_STRING));
-
-		// Get Major Subdivision ISO Code
-		status = MMDB_get_value(&result.entry, &entry_data,
-		                        "subdivisions", "0", "iso_code", nullptr);
-		location->Assign(1, mmdb_getvalue(&entry_data, status,
-		                 MMDB_DATA_TYPE_UTF8_STRING));
-
-		// Get City English Name
-		status = MMDB_get_value(&result.entry, &entry_data,
-		                        "city", "names", "en", nullptr);
-		location->Assign(2, mmdb_getvalue(&entry_data, status,
-		                 MMDB_DATA_TYPE_UTF8_STRING));
-
-		// Get Location Latitude
-		status = MMDB_get_value(&result.entry, &entry_data,
-		                        "location", "latitude", nullptr);
-		location->Assign(3, mmdb_getvalue(&entry_data, status,
-		                 MMDB_DATA_TYPE_DOUBLE));
-
-		// Get Location Longitude
-		status = MMDB_get_value(&result.entry, &entry_data,
-		                        "location", "longitude", nullptr);
-		location->Assign(4, mmdb_getvalue(&entry_data, status,
-		                 MMDB_DATA_TYPE_DOUBLE));
-
-		return location;
-		}
-
-#else // not USE_GEOIP
-	static int missing_geoip_reported = 0;
-
-	if ( ! missing_geoip_reported )
-		{
-		zeek::emit_builtin_error("Zeek was not configured for GeoIP support");
-		missing_geoip_reported = 1;
-		}
-#endif
-
-	// We can get here even if we have MMDB support if we weren't
-	// able to initialize it or it didn't return any information for
-	// the address.
-
-	return std::move(location);
-	%}
-
-## Performs an lookup of AS number & organization of an IP address.
-## Requires Zeek to be built with ``libmaxminddb``.
-##
-## a: The IP address to lookup.
-##
-## Returns: A record with autonomous system number and organization that contains *a*.
-##
-## .. zeek:see:: lookup_location
-function lookup_autonomous_system%(a: addr%) : geo_autonomous_system
-	%{
-	static auto geo_autonomous_system = zeek::id::find_type<zeek::RecordType>("geo_autonomous_system");
-	auto autonomous_system = zeek::make_intrusive<zeek::RecordVal>(geo_autonomous_system);
-
-#ifdef USE_GEOIP
-	mmdb_check_asn();
-	if ( ! mmdb_asn )
-		{
-		if ( ! mmdb_try_open_asn() )
-			{
-			if ( ! did_mmdb_asn_db_error )
-				{
-				did_mmdb_asn_db_error = true;
-				zeek::emit_builtin_error("Failed to open GeoIP ASN database");
-				}
-
-			return autonomous_system;
-			}
-		}
-
-	MMDB_lookup_result_s result;
-
-	if ( mmdb_lookup_asn(a->AsAddr(), result) )
-		{
-		MMDB_entry_data_s entry_data;
-		int status;
-
-		// Get Autonomous System Number
-		status = MMDB_get_value(&result.entry, &entry_data,
-		                        "autonomous_system_number", nullptr);
-		autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status,
-		                          MMDB_DATA_TYPE_UINT32));
-
-		// Get Autonomous System Organization
-		status = MMDB_get_value(&result.entry, &entry_data,
-		                        "autonomous_system_organization", nullptr);
-		autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status,
-		                          MMDB_DATA_TYPE_UTF8_STRING));
-
-		return autonomous_system;
-		}
-
-#else // not USE_GEOIP
-	static int missing_geoip_reported = 0;
-
-	if ( ! missing_geoip_reported )
-		{
-		zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support");
-		missing_geoip_reported = 1;
-		}
-#endif
-
-	// We can get here even if we have GeoIP support, if we weren't
-	// able to initialize it or it didn't return any information for
-	// the address.
-	return std::move(autonomous_system);
-	%}
-
 ## Calculates distance between two geographic locations using the haversine
 ## formula.  Latitudes and longitudes must be given in degrees, where southern
 ## hemisphere latitudes are negative and western hemisphere longitudes are

testing/btest/Baseline/core.mmdb.explicit-open/out

@@ -0,0 +1,17 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
+1299466805.0, 1, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299466805.0, 1, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
+1299470395.0, 2, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299470395.0, 2, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
+1299470395.0, 2, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299470395.0, 2, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
+1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
+1299470405.0, 3, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299470405.0, 3, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
+1299473995.0, 4, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299473995.0, 4, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
+1299473995.0, 4, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
+1299473995.0, 4, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]

testing/btest/Baseline/core.mmdb.explicit-open/reporter.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	level	message	location
+1299470395.000000	Reporter::INFO	Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	(empty)
+1299470395.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	(empty)
+1299470395.000000	Reporter::INFO	Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb]	(empty)
+1299470395.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb]	(empty)
+1299473995.000000	Reporter::INFO	Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	(empty)
+1299473995.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	(empty)
+1299473995.000000	Reporter::INFO	Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb]	(empty)
+1299473995.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb]	(empty)
+1299473995.000000	Reporter::INFO	received termination signal	(empty)

testing/btest/Baseline/core.mmdb.temporary-error/reporter.log

@@ -3,15 +3,15 @@ ts	level	message	location
 1299470395.000000	Reporter::INFO	Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	<params>, line 1
 1299470395.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	<params>, line 1
 1299470395.000000	Reporter::INFO	Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata]	<params>, line 1
-1299470395.000000	Reporter::ERROR	Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1))	<...>/temporary-error.zeek, line 100
+1299470395.000000	Reporter::ERROR	Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1))	<...>/temporary-error.zeek, line 101
 1299470395.000000	Reporter::INFO	Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb]	<params>, line 1
 1299470395.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb]	<params>, line 1
 1299470395.000000	Reporter::INFO	Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata]	<params>, line 1
-1299470395.000000	Reporter::ERROR	Failed to open GeoIP location database (lookup_location(128.3.0.1))	<...>/temporary-error.zeek, line 101
+1299470395.000000	Reporter::ERROR	Failed to open GeoIP location database (lookup_location(128.3.0.1))	<...>/temporary-error.zeek, line 102
 1299473995.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	<params>, line 1
-1299473995.000000	Reporter::ERROR	Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1))	<...>/temporary-error.zeek, line 100
+1299473995.000000	Reporter::ERROR	Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1))	<...>/temporary-error.zeek, line 101
 1299473995.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb]	<params>, line 1
-1299473995.000000	Reporter::ERROR	Failed to open GeoIP location database (lookup_location(128.3.0.1))	<...>/temporary-error.zeek, line 101
+1299473995.000000	Reporter::ERROR	Failed to open GeoIP location database (lookup_location(128.3.0.1))	<...>/temporary-error.zeek, line 102
 1299477595.000000	Reporter::INFO	Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	<params>, line 1
 1299477595.000000	Reporter::INFO	Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb]	<params>, line 1
 1299477595.000000	Reporter::INFO	Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb]	<params>, line 1

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -20,6 +20,7 @@ scripts/base/init-bare.zeek
   build/scripts/base/bif/supervisor.bif.zeek
   build/scripts/base/bif/packet_analysis.bif.zeek
   build/scripts/base/bif/CPP-load.bif.zeek
+  build/scripts/base/bif/mmdb.bif.zeek
   build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek
   build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek
   build/scripts/base/bif/event.bif.zeek

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -20,6 +20,7 @@ scripts/base/init-bare.zeek
   build/scripts/base/bif/supervisor.bif.zeek
   build/scripts/base/bif/packet_analysis.bif.zeek
   build/scripts/base/bif/CPP-load.bif.zeek
+  build/scripts/base/bif/mmdb.bif.zeek
   build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek
   build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek
   build/scripts/base/bif/event.bif.zeek

testing/btest/Baseline/plugins.hooks/output

@@ -493,6 +493,7 @@
 0.000000   MetaHookPost  LoadFile(0, ./main, <...>/main.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./main.zeek, <...>/main.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./patterns, <...>/patterns.zeek) -> -1
@@ -580,6 +581,7 @@
 0.000000   MetaHookPost  LoadFile(0, base<...>/main, <...>/main.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/main.zeek, <...>/main.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/mpls, <...>/mpls) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/nflog, <...>/nflog) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3) -> -1
@@ -778,6 +780,7 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./main, <...>/main.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./main.zeek, <...>/main.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./patterns, <...>/patterns.zeek) -> (-1, <no content>)
@@ -865,6 +868,7 @@
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/main, <...>/main.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/mpls, <...>/mpls) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/nflog, <...>/nflog) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3) -> (-1, <no content>)
@@ -1411,6 +1415,7 @@
 0.000000   MetaHookPre   LoadFile(0, ./main, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./main.zeek, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./patterns, <...>/patterns.zeek)
@@ -1498,6 +1503,7 @@
 0.000000   MetaHookPre   LoadFile(0, base<...>/main, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/main.zeek, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/mpls, <...>/mpls)
 0.000000   MetaHookPre   LoadFile(0, base<...>/nflog, <...>/nflog)
 0.000000   MetaHookPre   LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3)
@@ -1696,6 +1702,7 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./main, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./main.zeek, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./patterns, <...>/patterns.zeek)
@@ -1783,6 +1790,7 @@
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/main, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/mpls, <...>/mpls)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/nflog, <...>/nflog)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3)
@@ -2337,6 +2345,7 @@
 0.000000 | HookLoadFile  ./main <...>/main.zeek
 0.000000 | HookLoadFile  ./main.zeek <...>/main.zeek
 0.000000 | HookLoadFile  ./messaging.bif.zeek <...>/messaging.bif.zeek
+0.000000 | HookLoadFile  ./mmdb.bif.zeek <...>/mmdb.bif.zeek
 0.000000 | HookLoadFile  ./office <...>/office.sig
 0.000000 | HookLoadFile  ./option.bif.zeek <...>/option.bif.zeek
 0.000000 | HookLoadFile  ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek
@@ -2427,6 +2436,7 @@
 0.000000 | HookLoadFile  base<...>/main <...>/main.zeek
 0.000000 | HookLoadFile  base<...>/main.zeek <...>/main.zeek
 0.000000 | HookLoadFile  base<...>/messaging.bif <...>/messaging.bif.zeek
+0.000000 | HookLoadFile  base<...>/mmdb.bif <...>/mmdb.bif.zeek
 0.000000 | HookLoadFile  base<...>/mpls <...>/mpls
 0.000000 | HookLoadFile  base<...>/nflog <...>/nflog
 0.000000 | HookLoadFile  base<...>/novell_802_3 <...>/novell_802_3
@@ -2622,6 +2632,7 @@
 0.000000 | HookLoadFileExtended ./main <...>/main.zeek
 0.000000 | HookLoadFileExtended ./main.zeek <...>/main.zeek
 0.000000 | HookLoadFileExtended ./messaging.bif.zeek <...>/messaging.bif.zeek
+0.000000 | HookLoadFileExtended ./mmdb.bif.zeek <...>/mmdb.bif.zeek
 0.000000 | HookLoadFileExtended ./office <...>/office.sig
 0.000000 | HookLoadFileExtended ./option.bif.zeek <...>/option.bif.zeek
 0.000000 | HookLoadFileExtended ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek
@@ -2712,6 +2723,7 @@
 0.000000 | HookLoadFileExtended base<...>/main <...>/main.zeek
 0.000000 | HookLoadFileExtended base<...>/main.zeek <...>/main.zeek
 0.000000 | HookLoadFileExtended base<...>/messaging.bif <...>/messaging.bif.zeek
+0.000000 | HookLoadFileExtended base<...>/mmdb.bif <...>/mmdb.bif.zeek
 0.000000 | HookLoadFileExtended base<...>/mpls <...>/mpls
 0.000000 | HookLoadFileExtended base<...>/nflog <...>/nflog
 0.000000 | HookLoadFileExtended base<...>/novell_802_3 <...>/novell_802_3

testing/btest/core/mmdb/explicit-open.zeek

@@ -0,0 +1,50 @@
+# @TEST-DOC: verifies that the explicit BiFs for loading MMDBs work, including when re-opening.
+#
+# Like other MMDB tests, this uses a pcap to use each packet as a driver to
+# touch the DBs involved upon each packet, triggering DB reloads.
+#
+# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h
+#
+# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb
+# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT >out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+
+@load base/frameworks/reporter
+
+global pkt = 0;
+global asn_fn = "./mmdb/GeoLite2-ASN.mmdb";
+global city_fn = "./mmdb/GeoLite2-City.mmdb";
+
+function timestamp(n: count): string
+	{
+	assert n <= 60;
+	return fmt("2020-01-01T00:%s:00", n);
+	}
+
+event new_packet(c: connection, p: pkt_hdr)
+	{
+	++pkt;
+
+	print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1);
+	print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1);
+	print network_time(), pkt, 131.243.0.1, "asn", lookup_autonomous_system(131.243.0.1);
+	print network_time(), pkt, 131.243.0.1, "location", lookup_location(131.243.0.1);
+
+	# Increment MMDBs' modification time, triggering a re-open.
+	if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(asn_fn)), "") )
+		exit(1);
+
+	if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(city_fn)), "") )
+		exit(1);
+
+	if ( pkt == 4 )
+		terminate();
+	}
+
+event zeek_init()
+	{
+	assert mmdb_open_asn_db(asn_fn);
+	assert mmdb_open_location_db(city_fn);
+	}

testing/btest/core/mmdb/temporary-error.zeek

@@ -13,6 +13,7 @@
 @load base/frameworks/reporter
 
 redef mmdb_dir = "./mmdb";
+redef mmdb_dir_fallbacks = vector(); # Clear out fallbacks to avoid influence on tests
 
 global pkt = 0;