diff --git a/CHANGES b/CHANGES index 40c8bac779..56c3b12d67 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,60 @@ +6.2.0-dev.375 | 2024-01-12 09:27:58 +0100 + + * Modernize various C++/Zeek-isms in the MMDB code. (Christian Kreibich, Corelight) + + * Fix MMDB code to re-open explicitly opened DBs correctly (Christian Kreibich, Corelight) + + The filename from which a DB first gets opened (either via an explicitly + specified filename, or via the path sequence now configurable at the script + layer) is now "sticky", meaning re-opening won't switch to a different file. + + This was easiest by moving most state into the MMDB class itself. The previous + approach of tracking the two DB instances via a smart pointer and blowing the + pointed-to objects away as needed is now instead one of two objects fixed over + the lifetime of Zeek, able to open/close/reopen their underlying Maxmind DBs. + + The MMDB class now only has one Lookup() method since there was no need to break + them apart -- it saves the return of a MMDB_lookup_result_s over the stack and + there's no need for throwing an exception. + + * Add btest to verify behavior of re-opened MMDBs opened directly via BIFs (Christian Kreibich, Corelight) + + The mmdb_open_location_db() and mmdb_open_asn_db() BiFs were untested, and Zeek + has a bug that makes any DBs opened that way fall back to looking up DBs via the + existing script-level config mechanism (via mmdb_dir), which is at least + unexpected and might well be unconfigured if somebody uses the direct BiFs. + + * Simplify MMDB code by moving more lookup functionality into MMDB class (Christian Kreibich, Corelight) + + * Move MMDB logic out of mmdb.bif and into MMDB.cc/h. (Christian Kreibich, Corelight) + + This does not change the implementation except for some light renaming where + things are now naturally scoped within MMDB.cc. + + * Fix mmdb.temporary-error testcase when MMDBs are installed on system (Christian Kreibich, Corelight) + + The test would previously fail in settings where the user has Maxmind DBs + installed in the hardwired system locations, because the fallback logic still + picked those up. + + * Adapt MMDB BiF code to new script-layer variables (Christian Kreibich, Corelight) + + * Update btest baselines to reflect introduction of mmdb.bif (Christian Kreibich, Corelight) + + * Move MaxMind/GeoIP BiF functionality into separate file (Christian Kreibich, Corelight) + + * Provide script-level configurability of MaxMind DB placement on disk (Christian Kreibich, Corelight) + + This lifts the list of fallback directories in which Zeek will look for Maxmind + DBs into the script layer, and makes the names of the DB files themselves + (previously hardwired) configurable as well. + + This does not yet change the in-core code; that commit follows. + + * Sort toplevel .bif list in CMakeLists (Christian Kreibich, Corelight) + + Not important, but here it is safe to do so and the list is getting lengthy. + 6.2.0-dev.362 | 2024-01-11 20:17:45 +0100 * GH-3540: Known: Keep &create_expire on local tables/sets valid (Arne Welzel, Corelight) diff --git a/VERSION b/VERSION index 2a0bb87848..968f8f8a09 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.2.0-dev.362 +6.2.0-dev.375 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 41b918992d..0313d63ae5 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -1107,6 +1107,25 @@ type geo_autonomous_system: record { ## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support. const mmdb_dir: string = "" &redef; +## Default name of the MaxMind City database file: +const mmdb_city_db: string = "GeoLite2-City.mmdb" &redef; +## Default name of the MaxMind Country database file: +const mmdb_country_db: string = "GeoLite2-Country.mmdb" &redef; +## Default name of the MaxMind ASN database file: +const mmdb_asn_db: string = "GeoLite2-ASN.mmdb" &redef; + +## Fallback locations for MaxMind databases. Zeek attempts these when +## :zeek:see:`mmdb_dir` is not set, or it cannot read a DB file from it. For +## geolocation lookups, Zeek will first attempt to locate the city database in +## each of the fallback locations, and should this fail, attempt to locate the +## country one. +const mmdb_dir_fallbacks: vector of string = vector( + "/usr/share/GeoIP", + "/var/lib/GeoIP", + "/usr/local/share/GeoIP", + "/usr/local/var/GeoIP", +) &redef; + ## Sets the interval for MaxMind DB file staleness checks. When Zeek detects a ## change in inode or modification time, the database is re-opened. Setting ## a negative interval disables staleness checks. @@ -2094,6 +2113,7 @@ type gtp_delete_pdp_ctx_response_elements: record { @load base/bif/supervisor.bif @load base/bif/packet_analysis.bif @load base/bif/CPP-load.bif +@load base/bif/mmdb.bif ## Internal function. function add_interface(iold: string, inew: string): string diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4101e97be1..8b1007416f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -111,27 +111,28 @@ include(BifCl) set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc) set(BIF_SRCS - zeek.bif communityid.bif - stats.bif - event.bif const.bif - types.bif - strings.bif - reporter.bif + event.bif + mmdb.bif option.bif - # Note: the supervisor BIF file is treated like other top-level BIFs instead - # of contained in its own subdirectory CMake logic because subdirectory BIFs - # are treated differently and don't support being called *during* parsing - # (e.g. within an @if directive). - supervisor/supervisor.bif + reporter.bif + stats.bif + strings.bif + types.bif + zeek.bif # The packet analysis BIF is treated like other top-level BIFs because it's # needed before parsing the packet protocol scripts, which happen very near # to the start of parsing. packet_analysis/packet_analysis.bif # The C++ loading BIF is treated like other top-level BIFs to give us # flexibility regarding when it's called. - script_opt/CPP/CPP-load.bif) + script_opt/CPP/CPP-load.bif + # Note: the supervisor BIF file is treated like other top-level BIFs instead + # of contained in its own subdirectory CMake logic because subdirectory BIFs + # are treated differently and don't support being called *during* parsing + # (e.g. within an @if directive). + supervisor/supervisor.bif) foreach (bift ${BIF_SRCS}) bif_target(${bift} "standard") @@ -328,6 +329,7 @@ set(MAIN_SRCS IP.cc IPAddr.cc List.cc + MMDB.cc Reporter.cc NFA.cc NetVar.cc diff --git a/src/Func.cc b/src/Func.cc index 20a4a79164..b399c80600 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -52,6 +52,7 @@ #include "supervisor.bif.func_h" #include "packet_analysis.bif.func_h" #include "CPP-load.bif.func_h" +#include "mmdb.bif.func_h" #include "zeek.bif.func_def" #include "communityid.bif.func_def" @@ -62,6 +63,7 @@ #include "supervisor.bif.func_def" #include "packet_analysis.bif.func_def" #include "CPP-load.bif.func_def" +#include "mmdb.bif.func_def" // clang-format on extern RETSIGTYPE sig_handler(int signo); @@ -1049,6 +1051,7 @@ void init_primary_bifs() { #include "CPP-load.bif.func_init" #include "communityid.bif.func_init" +#include "mmdb.bif.func_init" #include "option.bif.func_init" #include "packet_analysis.bif.func_init" #include "reporter.bif.func_init" diff --git a/src/MMDB.cc b/src/MMDB.cc new file mode 100644 index 0000000000..45b6b0052b --- /dev/null +++ b/src/MMDB.cc @@ -0,0 +1,357 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/MMDB.h" + +#include +#include +#include +#include +#include + +#include "zeek/Func.h" +#include "zeek/IPAddr.h" +#include "zeek/ZeekString.h" + +namespace zeek { + +#ifdef USE_GEOIP + +static int msg_count = 0; +static double msg_suppression_time = 0; +static constexpr int msg_limit = 20; +static constexpr double msg_suppression_duration = 300; + +LocDB mmdb_loc; +AsnDB mmdb_asn; + +static void report_msg(const char* format, ...) { + if ( zeek::run_state::network_time > msg_suppression_time + msg_suppression_duration ) { + msg_count = 0; + msg_suppression_time = zeek::run_state::network_time; + } + + if ( msg_count >= msg_limit ) + return; + + ++msg_count; + + va_list al; + va_start(al, format); + std::string msg = zeek::util::vfmt(format, al); + va_end(al); + + zeek::reporter->Info("%s", msg.data()); +} + +static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, int data_type) { + switch ( status ) { + case MMDB_SUCCESS: + if ( entry_data->has_data ) { + switch ( data_type ) { + case MMDB_DATA_TYPE_UTF8_STRING: + return zeek::make_intrusive(entry_data->data_size, entry_data->utf8_string); + break; + + case MMDB_DATA_TYPE_DOUBLE: + return zeek::make_intrusive(entry_data->double_value); + break; + + case MMDB_DATA_TYPE_UINT32: return zeek::val_mgr->Count(entry_data->uint32); + + default: break; + } + } + break; + + case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: + // key doesn't exist, nothing to do + break; + + default: report_msg("MaxMind DB error [%s]", MMDB_strerror(status)); break; + } + + return nullptr; +} + +MMDB::MMDB() : mmdb{}, file_info{}, reported_error{false}, last_check{zeek::run_state::network_time} {} + +MMDB::~MMDB() { Close(); } + +bool MMDB::OpenFile(const std::string& a_filename) { + filename = a_filename; + Close(); + + if ( 0 != stat(filename.data(), &file_info) ) { + return false; + } + + int status = MMDB_open(a_filename.data(), MMDB_MODE_MMAP, &mmdb); + + if ( MMDB_SUCCESS != status ) { + memset(&mmdb, 0, sizeof(mmdb)); + report_msg("Failed to open MaxMind DB: %s [%s]", filename.data(), MMDB_strerror(status)); + return false; + } + + return true; +} + +void MMDB::Close() { + if ( IsOpen() ) { + MMDB_close(&mmdb); + memset(&mmdb, 0, sizeof(mmdb)); + reported_error = false; + } +} + +bool MMDB::EnsureLoaded() { + bool res = true; + + if ( filename.empty() ) + res = OpenFromScriptConfig(); + else if ( ! IsOpen() ) + res = OpenFile(filename); + else if ( IsStaleDB() ) { + report_msg("Closing stale MaxMind DB [%s]", filename.data()); + if ( ! OpenFile(filename) ) + res = false; + } + + if ( ! res && ! reported_error ) { + reported_error = true; + zeek::emit_builtin_error(zeek::util::fmt("Failed to open %s", Description().data())); + } + + return res; +} + +bool MMDB::Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { + struct sockaddr_storage ss = {0}; + + if ( IPv4 == addr.GetFamily() ) { + struct sockaddr_in* sa = (struct sockaddr_in*)&ss; + sa->sin_family = AF_INET; + addr.CopyIPv4(&sa->sin_addr); + } + else { + struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; + sa->sin6_family = AF_INET6; + addr.CopyIPv6(&sa->sin6_addr); + } + + int mmdb_error; + result = MMDB_lookup_sockaddr(&mmdb, (struct sockaddr*)&ss, &mmdb_error); + + if ( MMDB_SUCCESS != mmdb_error ) { + report_msg("MaxMind DB lookup location error [%s]", MMDB_strerror(mmdb_error)); + Close(); + return false; + } + + return true; +} + +// Check to see if the Maxmind DB should be closed and reopened. This will +// happen if there was a lookup error or if the mmap'd file has been replaced +// by an external process. +bool MMDB::IsStaleDB() { + if ( ! IsOpen() ) + return false; + + static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); + + if ( mmdb_stale_check_interval < 0.0 ) + return false; + + if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) + return false; + + last_check = zeek::run_state::network_time; + struct stat buf; + + if ( 0 != stat(mmdb.filename, &buf) ) + return true; + + if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) { + report_msg("%s change detected for MaxMind DB [%s]", + buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", mmdb.filename); + return true; + } + + return false; +} + +bool LocDB::OpenFromScriptConfig() { + // City database is always preferred over Country database. + const auto& mmdb_dir_val = zeek::id::find_val("mmdb_dir"); + std::string mmdb_dir{mmdb_dir_val->ToStdStringView()}; + + const auto& mmdb_city_db_val = zeek::id::find_val("mmdb_city_db"); + std::string mmdb_city_db{mmdb_city_db_val->ToStdStringView()}; + + const auto& mmdb_country_db_val = zeek::id::find_val("mmdb_country_db"); + std::string mmdb_country_db{mmdb_country_db_val->ToStdStringView()}; + + if ( ! mmdb_dir.empty() ) { + auto d = mmdb_dir + "/" + mmdb_city_db; + + if ( OpenFile(d) ) + return true; + + d = mmdb_dir + "/" + mmdb_country_db; + + if ( OpenFile(d) ) + return true; + } + + const auto& mmdb_dir_fallbacks_val = zeek::id::find_val("mmdb_dir_fallbacks"); + + for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) { + auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_city_db; + if ( OpenFile(d) ) + return true; + } + + for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) { + auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_country_db; + if ( OpenFile(d) ) + return true; + } + + return false; +} +bool AsnDB::OpenFromScriptConfig() { + const auto& mmdb_dir_val = zeek::id::find_val("mmdb_dir"); + std::string mmdb_dir{mmdb_dir_val->ToStdStringView()}; + + const auto& mmdb_asn_db_val = zeek::id::find_val("mmdb_asn_db"); + std::string mmdb_asn_db{mmdb_asn_db_val->ToStdStringView()}; + + if ( ! mmdb_dir.empty() ) { + auto d = mmdb_dir + "/" + mmdb_asn_db; + + if ( OpenFile(d) ) + return true; + } + + const auto& mmdb_dir_fallbacks_val = zeek::id::find_val("mmdb_dir_fallbacks"); + + for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) { + auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_asn_db; + if ( OpenFile(d) ) + return true; + } + + return false; +} +#endif // USE_GEOIP + +ValPtr mmdb_open_location_db(const StringValPtr& filename) { +#ifdef USE_GEOIP + return zeek::val_mgr->Bool(mmdb_loc.OpenFile(filename->ToStdString())); +#else + return zeek::val_mgr->False(); +#endif +} + +ValPtr mmdb_open_asn_db(const StringValPtr& filename) { +#ifdef USE_GEOIP + return zeek::val_mgr->Bool(mmdb_asn.OpenFile(filename->ToStdString())); +#else + return zeek::val_mgr->False(); +#endif +} + +RecordValPtr mmdb_lookup_location(const AddrValPtr& addr) { + static auto geo_location = zeek::id::find_type("geo_location"); + auto location = zeek::make_intrusive(geo_location); + +#ifdef USE_GEOIP + if ( ! mmdb_loc.EnsureLoaded() ) + return location; + + MMDB_lookup_result_s result; + + if ( mmdb_loc.Lookup(addr->AsAddr(), result) ) { + MMDB_entry_data_s entry_data; + int status; + + // Get Country ISO Code + status = MMDB_get_value(&result.entry, &entry_data, "country", "iso_code", nullptr); + location->Assign(0, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + // Get Major Subdivision ISO Code + status = MMDB_get_value(&result.entry, &entry_data, "subdivisions", "0", "iso_code", nullptr); + location->Assign(1, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + // Get City English Name + status = MMDB_get_value(&result.entry, &entry_data, "city", "names", "en", nullptr); + location->Assign(2, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + // Get Location Latitude + status = MMDB_get_value(&result.entry, &entry_data, "location", "latitude", nullptr); + location->Assign(3, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_DOUBLE)); + + // Get Location Longitude + status = MMDB_get_value(&result.entry, &entry_data, "location", "longitude", nullptr); + location->Assign(4, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_DOUBLE)); + + return location; + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) { + zeek::emit_builtin_error("Zeek was not configured for GeoIP support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have MMDB support if we weren't + // able to initialize it or it didn't return any information for + // the address. + return location; +} + +RecordValPtr mmdb_lookup_autonomous_system(const AddrValPtr& addr) { + static auto geo_autonomous_system = zeek::id::find_type("geo_autonomous_system"); + auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); + +#ifdef USE_GEOIP + if ( ! mmdb_asn.EnsureLoaded() ) + return autonomous_system; + + MMDB_lookup_result_s result; + + if ( mmdb_asn.Lookup(addr->AsAddr(), result) ) { + MMDB_entry_data_s entry_data; + int status; + + // Get Autonomous System Number + status = MMDB_get_value(&result.entry, &entry_data, "autonomous_system_number", nullptr); + autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UINT32)); + + // Get Autonomous System Organization + status = MMDB_get_value(&result.entry, &entry_data, "autonomous_system_organization", nullptr); + autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + return autonomous_system; + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) { + zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have GeoIP support, if we weren't + // able to initialize it or it didn't return any information for + // the address. + return autonomous_system; +} + +} // namespace zeek diff --git a/src/MMDB.h b/src/MMDB.h new file mode 100644 index 0000000000..0d3af59638 --- /dev/null +++ b/src/MMDB.h @@ -0,0 +1,91 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/Val.h" + +namespace zeek { + +#ifdef USE_GEOIP + +#include + +// The MMDB class encapsulates a low-level libmaxmind MMDB_s structure. It +// tracks whether that DB is currently loaded, and can open it from a file in +// two ways: (1) via explicit specification of a filename, (2) by determining +// the configuration from configuration settings in the script layer (mmdb_dir +// etc). This configuration depends on whether this is a geolocation DB or an +// ASN one, so details are left to derived classes below that specialize. +// +// The class tracks the inode and modification time of a DB file to detect +// "stale" DBs, which get reloaded (from the same location in the file system) +// upon the first lookup that detects staleness. +class MMDB { +public: + MMDB(); + virtual ~MMDB(); + + // Implements the logic to determine a file system path for the DB from + // script-layer configuration settings, and opens the DB from there. Returns + // true if successful, false otherwise. + virtual bool OpenFromScriptConfig() = 0; + + // Helper string to identify the type of DB, useful in error messages. + virtual std::string_view Description() = 0; + + // Opens the DB at the given location, closing and cleaning up any currently + // opened DB if there is one. Returns true if successful, false otherwise. + bool OpenFile(const std::string& filename); + + // Closes a currently opened DB, releasing its state. Safe to call on a + // closed DB. + void Close(); + + // Predicate; returns true if the DB is currently opened. + bool IsOpen() const { return mmdb.filename != nullptr; } + + // Ensures that upon return the underlying DB file is loaded. When no + // filename is configured for the DB (i.e. OpenFile() has never been called + // on it), this triggers the script-level configuration lookup via + // OpenFromScriptConfig(). When a filename is available but it's not + // currently loaded, it does so. Finally, if there's a loaded DB but it's + // found to be stale, it gets reloaded. When the load operation succeeds, or + // the DB was already loaded and not stale, this returns true, and false if + // anything went wrong. + bool EnsureLoaded(); + + // Looks up a given IP address in the DB, storing the result in the provided + // result structure. + bool Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result); + +private: + bool IsStaleDB(); + + std::string filename; + MMDB_s mmdb; + struct stat file_info; + bool reported_error; // to ensure we emit builtin errors during opening only once. + double last_check; +}; + +class LocDB : public MMDB { +public: + bool OpenFromScriptConfig(); + std::string_view Description() { return "GeoIP location database"; } +}; + +class AsnDB : public MMDB { +public: + bool OpenFromScriptConfig(); + std::string_view Description() { return "GeoIP ASN database"; } +}; + +#endif // USE_GEOIP + +ValPtr mmdb_open_location_db(const StringValPtr& filename); +ValPtr mmdb_open_asn_db(const StringValPtr& filename); + +RecordValPtr mmdb_lookup_location(const AddrValPtr& addr); +RecordValPtr mmdb_lookup_autonomous_system(const AddrValPtr& addr); + +} // namespace zeek diff --git a/src/mmdb.bif b/src/mmdb.bif new file mode 100644 index 0000000000..76ff974e07 --- /dev/null +++ b/src/mmdb.bif @@ -0,0 +1,55 @@ +%%{ +#include +%%} + +## Initializes MMDB for later use of lookup_location. +## Requires Zeek to be built with ``libmaxminddb``. +## +## f: The filename of the MaxMind City or Country DB. +## +## Returns: A boolean indicating whether the db was successfully opened. +## +## .. zeek:see:: lookup_autonomous_system +function mmdb_open_location_db%(f: string%) : bool + %{ + return zeek::mmdb_open_location_db(StringValPtr(NewRef(), f)); + %} + +## Initializes MMDB for later use of lookup_autonomous_system. +## Requires Zeek to be built with ``libmaxminddb``. +## +## f: The filename of the MaxMind ASN DB. +## +## Returns: A boolean indicating whether the db was successfully opened. +## +## .. zeek:see:: lookup_autonomous_system +function mmdb_open_asn_db%(f: string%) : bool + %{ + return zeek::mmdb_open_asn_db(StringValPtr(NewRef(), f)); + %} + +## Performs a geo-lookup of an IP address. +## Requires Zeek to be built with ``libmaxminddb``. +## +## a: The IP address to lookup. +## +## Returns: A record with country, region, city, latitude, and longitude. +## +## .. zeek:see:: lookup_autonomous_system +function lookup_location%(a: addr%) : geo_location + %{ + return zeek::mmdb_lookup_location(AddrValPtr(NewRef(), a)); + %} + +## Performs an lookup of AS number & organization of an IP address. +## Requires Zeek to be built with ``libmaxminddb``. +## +## a: The IP address to lookup. +## +## Returns: A record with autonomous system number and organization that contains *a*. +## +## .. zeek:see:: lookup_location +function lookup_autonomous_system%(a: addr%) : geo_autonomous_system + %{ + return zeek::mmdb_lookup_autonomous_system(AddrValPtr(NewRef(), a)); + %} diff --git a/src/zeek.bif b/src/zeek.bif index dd04941170..770524ebda 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -3980,525 +3980,6 @@ function lookup_hostname%(host: string%) : addr_set return nullptr; %} -%%{ -#ifdef USE_GEOIP -#include - -extern "C" { -#include -#include -#include -#include -#include -} - -static int mmdb_msg_count = 0; -static constexpr int mmdb_msg_limit = 20; -static double mmdb_msg_suppression_time = 0; -static constexpr double mmdb_msg_suppression_duration = 300; - -static void report_mmdb_msg(const char* format, ...) - { - if ( zeek::run_state::network_time > mmdb_msg_suppression_time + mmdb_msg_suppression_duration ) - { - mmdb_msg_count = 0; - mmdb_msg_suppression_time = zeek::run_state::network_time; - } - - if ( mmdb_msg_count >= mmdb_msg_limit ) - return; - - ++mmdb_msg_count; - - va_list al; - va_start(al, format); - std::string msg = zeek::util::vfmt(format, al); - va_end(al); - - zeek::reporter->Info("%s", msg.data()); - } - -class MMDB { -public: - MMDB(const char* filename, struct stat info); - - ~MMDB(); - - MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); - bool StaleDB(); - const char* Filename(); - -private: - MMDB_s mmdb; - struct stat file_info; - bool lookup_error; - double last_check; -}; - -MMDB::MMDB(const char* filename, struct stat info) - : file_info(info), lookup_error{false}, - last_check{zeek::run_state::network_time} - { - int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); - - if ( MMDB_SUCCESS != status ) - { - throw std::runtime_error(MMDB_strerror(status)); - } - } - -MMDB::~MMDB() - { - MMDB_close(&mmdb); - } - -MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) - { - int mmdb_error; - MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); - - if ( MMDB_SUCCESS != mmdb_error ) - { - lookup_error = true; - throw std::runtime_error(MMDB_strerror(mmdb_error)); - } - - return result; - } - -// Check to see if the Maxmind DB should be closed and reopened. This will -// happen if there was a lookup error or if the mmap'd file has been replaced -// by an external process. -bool MMDB::StaleDB() - { - struct stat buf; - - if ( lookup_error ) - return true; - - - static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); - - if ( mmdb_stale_check_interval < 0.0 ) - return false; - - if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) - return false; - - last_check = zeek::run_state::network_time; - - if ( 0 != stat(mmdb.filename, &buf) ) - return true; - - if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) - { - report_mmdb_msg("%s change detected for MaxMind DB [%s]", - buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", - mmdb.filename); - return true; - } - - return false; - } - -const char* MMDB::Filename() - { - return mmdb.filename; - } - -std::unique_ptr mmdb_loc; -std::unique_ptr mmdb_asn; -static bool did_mmdb_loc_db_error = false; -static bool did_mmdb_asn_db_error = false; - -static bool mmdb_open(const char* filename, bool asn) - { - struct stat buf; - - if ( 0 != stat(filename, &buf) ) - { - return false; - } - - try - { - if ( asn ) - { - mmdb_asn.reset(new MMDB(filename, buf)); - } - else - { - mmdb_loc.reset(new MMDB(filename, buf)); - } - } - - catch ( const std::exception& e ) - { - if ( asn ) - did_mmdb_asn_db_error = false; - else - did_mmdb_loc_db_error = false; - - report_mmdb_msg("Failed to open MaxMind DB: %s [%s]", filename, - e.what()); - return false; - } - - return true; - } - -static bool mmdb_open_loc(const char* filename) - { - return mmdb_open(filename, false); - } - -static bool mmdb_open_asn(const char* filename) - { - return mmdb_open(filename, true); - } - -static void mmdb_check_loc() - { - if ( mmdb_loc && mmdb_loc->StaleDB() ) - { - report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); - did_mmdb_loc_db_error = false; - mmdb_loc.reset(); - } - } - -static void mmdb_check_asn() - { - if ( mmdb_asn && mmdb_asn->StaleDB() ) - { - report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); - did_mmdb_asn_db_error = false; - mmdb_asn.reset(); - } - } - -static bool mmdb_lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result, - bool asn) - { - struct sockaddr_storage ss = {0}; - - if ( IPv4 == addr.GetFamily() ) - { - struct sockaddr_in* sa = (struct sockaddr_in*)&ss; - sa->sin_family = AF_INET; - addr.CopyIPv4(&sa->sin_addr); - } - - else - { - struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; - sa->sin6_family = AF_INET6; - addr.CopyIPv6(&sa->sin6_addr); - } - - try - { - result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss) - : mmdb_loc->Lookup((struct sockaddr*)&ss); - } - - catch ( const std::exception& e ) - { - report_mmdb_msg("MaxMind DB lookup location error [%s]", e.what()); - return false; - } - - return result.found_entry; - } - -static bool mmdb_lookup_loc(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) - { - return mmdb_lookup(addr, result, false); - } - -static bool mmdb_lookup_asn(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) - { - return mmdb_lookup(addr, result, true); - } - -static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, - int data_type ) - { - switch (status) - { - case MMDB_SUCCESS: - if ( entry_data->has_data ) - { - switch (data_type) - { - case MMDB_DATA_TYPE_UTF8_STRING: - return zeek::make_intrusive( - entry_data->data_size, entry_data->utf8_string); - break; - - case MMDB_DATA_TYPE_DOUBLE: - return zeek::make_intrusive(entry_data->double_value); - break; - - case MMDB_DATA_TYPE_UINT32: - return zeek::val_mgr->Count(entry_data->uint32); - - default: - break; - } - } - break; - - case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: - // key doesn't exist, nothing to do - break; - - default: - report_mmdb_msg("MaxMind DB error [%s]", MMDB_strerror(status)); - break; - } - - return nullptr; - } - -static bool mmdb_try_open_loc () - { - // City database is always preferred over Country database. - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); - - if ( ! mmdb_dir.empty() ) - { - auto d = mmdb_dir + "/GeoLite2-City.mmdb"; - - if ( mmdb_open_loc(d.data()) ) - return true; - - d = mmdb_dir + "/GeoLite2-Country.mmdb"; - - if ( mmdb_open_loc(d.data()) ) - return true;; - } - - return mmdb_open_loc("/usr/share/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/share/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-Country.mmdb"); - } - -static bool mmdb_try_open_asn () - { - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); - - if ( ! mmdb_dir.empty() ) - { - auto d = mmdb_dir + "/GeoLite2-ASN.mmdb"; - - if ( mmdb_open_asn(d.data()) ) - return true; - } - - return mmdb_open_asn("/usr/share/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/var/lib/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/usr/local/share/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/usr/local/var/GeoIP/GeoLite2-ASN.mmdb"); - } - -#endif -%%} - -## Initializes MMDB for later use of lookup_location. -## Requires Zeek to be built with ``libmaxminddb``. -## -## f: The filename of the MaxMind City or Country DB. -## -## Returns: A boolean indicating whether the db was successfully opened. -## -## .. zeek:see:: lookup_autonomous_system -function mmdb_open_location_db%(f: string%) : bool - %{ -#ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_loc(f->CheckString())); -#else - return zeek::val_mgr->False(); -#endif - %} - -## Initializes MMDB for later use of lookup_autonomous_system. -## Requires Zeek to be built with ``libmaxminddb``. -## -## f: The filename of the MaxMind ASN DB. -## -## Returns: A boolean indicating whether the db was successfully opened. -## -## .. zeek:see:: lookup_autonomous_system -function mmdb_open_asn_db%(f: string%) : bool - %{ -#ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_asn(f->CheckString())); -#else - return zeek::val_mgr->False(); -#endif - %} - -## Performs a geo-lookup of an IP address. -## Requires Zeek to be built with ``libmaxminddb``. -## -## a: The IP address to lookup. -## -## Returns: A record with country, region, city, latitude, and longitude. -## -## .. zeek:see:: lookup_autonomous_system -function lookup_location%(a: addr%) : geo_location - %{ - static auto geo_location = zeek::id::find_type("geo_location"); - auto location = zeek::make_intrusive(geo_location); - -#ifdef USE_GEOIP - mmdb_check_loc(); - if ( ! mmdb_loc ) - { - if ( ! mmdb_try_open_loc() ) - { - if ( ! did_mmdb_loc_db_error ) - { - did_mmdb_loc_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP location database"); - } - - return location; - } - } - - MMDB_lookup_result_s result; - - if ( mmdb_lookup_loc(a->AsAddr(), result) ) - { - MMDB_entry_data_s entry_data; - int status; - - // Get Country ISO Code - status = MMDB_get_value(&result.entry, &entry_data, - "country", "iso_code", nullptr); - location->Assign(0, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get Major Subdivision ISO Code - status = MMDB_get_value(&result.entry, &entry_data, - "subdivisions", "0", "iso_code", nullptr); - location->Assign(1, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get City English Name - status = MMDB_get_value(&result.entry, &entry_data, - "city", "names", "en", nullptr); - location->Assign(2, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get Location Latitude - status = MMDB_get_value(&result.entry, &entry_data, - "location", "latitude", nullptr); - location->Assign(3, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_DOUBLE)); - - // Get Location Longitude - status = MMDB_get_value(&result.entry, &entry_data, - "location", "longitude", nullptr); - location->Assign(4, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_DOUBLE)); - - return location; - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - zeek::emit_builtin_error("Zeek was not configured for GeoIP support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have MMDB support if we weren't - // able to initialize it or it didn't return any information for - // the address. - - return std::move(location); - %} - -## Performs an lookup of AS number & organization of an IP address. -## Requires Zeek to be built with ``libmaxminddb``. -## -## a: The IP address to lookup. -## -## Returns: A record with autonomous system number and organization that contains *a*. -## -## .. zeek:see:: lookup_location -function lookup_autonomous_system%(a: addr%) : geo_autonomous_system - %{ - static auto geo_autonomous_system = zeek::id::find_type("geo_autonomous_system"); - auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); - -#ifdef USE_GEOIP - mmdb_check_asn(); - if ( ! mmdb_asn ) - { - if ( ! mmdb_try_open_asn() ) - { - if ( ! did_mmdb_asn_db_error ) - { - did_mmdb_asn_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP ASN database"); - } - - return autonomous_system; - } - } - - MMDB_lookup_result_s result; - - if ( mmdb_lookup_asn(a->AsAddr(), result) ) - { - MMDB_entry_data_s entry_data; - int status; - - // Get Autonomous System Number - status = MMDB_get_value(&result.entry, &entry_data, - "autonomous_system_number", nullptr); - autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UINT32)); - - // Get Autonomous System Organization - status = MMDB_get_value(&result.entry, &entry_data, - "autonomous_system_organization", nullptr); - autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - return autonomous_system; - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have GeoIP support, if we weren't - // able to initialize it or it didn't return any information for - // the address. - return std::move(autonomous_system); - %} - ## Calculates distance between two geographic locations using the haversine ## formula. Latitudes and longitudes must be given in degrees, where southern ## hemisphere latitudes are negative and western hemisphere longitudes are diff --git a/testing/btest/Baseline/core.mmdb.explicit-open/out b/testing/btest/Baseline/core.mmdb.explicit-open/out new file mode 100644 index 0000000000..5641085ef3 --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.explicit-open/out @@ -0,0 +1,17 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299466805.0, 1, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299466805.0, 1, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470395.0, 2, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470395.0, 2, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470395.0, 2, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470395.0, 2, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470405.0, 3, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470405.0, 3, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299473995.0, 4, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299473995.0, 4, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299473995.0, 4, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299473995.0, 4, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] diff --git a/testing/btest/Baseline/core.mmdb.explicit-open/reporter.log b/testing/btest/Baseline/core.mmdb.explicit-open/reporter.log new file mode 100644 index 0000000000..91b9edb7f8 --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.explicit-open/reporter.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts level message location +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO received termination signal (empty) diff --git a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log index 8dcd9fb5c1..6c9ef5749e 100644 --- a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log +++ b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log @@ -3,15 +3,15 @@ ts level message location 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 100 +1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 101 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 101 +1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 102 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 100 +1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 101 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 101 +1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 102 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index f7ca106dc0..0bdbda72a4 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -20,6 +20,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/supervisor.bif.zeek build/scripts/base/bif/packet_analysis.bif.zeek build/scripts/base/bif/CPP-load.bif.zeek + build/scripts/base/bif/mmdb.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek build/scripts/base/bif/event.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index ce766b6a49..29feb83dfb 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -20,6 +20,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/supervisor.bif.zeek build/scripts/base/bif/packet_analysis.bif.zeek build/scripts/base/bif/CPP-load.bif.zeek + build/scripts/base/bif/mmdb.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek build/scripts/base/bif/event.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 5bebc930f3..33c34f0daf 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -493,6 +493,7 @@ 0.000000 MetaHookPost LoadFile(0, ./main, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./main.zeek, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./patterns, <...>/patterns.zeek) -> -1 @@ -580,6 +581,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/main, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/main.zeek, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/mpls, <...>/mpls) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/nflog, <...>/nflog) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3) -> -1 @@ -778,6 +780,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./main, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./main.zeek, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./patterns, <...>/patterns.zeek) -> (-1, ) @@ -865,6 +868,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/main, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/mpls, <...>/mpls) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/nflog, <...>/nflog) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3) -> (-1, ) @@ -1411,6 +1415,7 @@ 0.000000 MetaHookPre LoadFile(0, ./main, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, ./main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./patterns, <...>/patterns.zeek) @@ -1498,6 +1503,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/main, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/mpls, <...>/mpls) 0.000000 MetaHookPre LoadFile(0, base<...>/nflog, <...>/nflog) 0.000000 MetaHookPre LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3) @@ -1696,6 +1702,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./main, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./patterns, <...>/patterns.zeek) @@ -1783,6 +1790,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/main, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/mpls, <...>/mpls) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/nflog, <...>/nflog) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3) @@ -2337,6 +2345,7 @@ 0.000000 | HookLoadFile ./main <...>/main.zeek 0.000000 | HookLoadFile ./main.zeek <...>/main.zeek 0.000000 | HookLoadFile ./messaging.bif.zeek <...>/messaging.bif.zeek +0.000000 | HookLoadFile ./mmdb.bif.zeek <...>/mmdb.bif.zeek 0.000000 | HookLoadFile ./office <...>/office.sig 0.000000 | HookLoadFile ./option.bif.zeek <...>/option.bif.zeek 0.000000 | HookLoadFile ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek @@ -2427,6 +2436,7 @@ 0.000000 | HookLoadFile base<...>/main <...>/main.zeek 0.000000 | HookLoadFile base<...>/main.zeek <...>/main.zeek 0.000000 | HookLoadFile base<...>/messaging.bif <...>/messaging.bif.zeek +0.000000 | HookLoadFile base<...>/mmdb.bif <...>/mmdb.bif.zeek 0.000000 | HookLoadFile base<...>/mpls <...>/mpls 0.000000 | HookLoadFile base<...>/nflog <...>/nflog 0.000000 | HookLoadFile base<...>/novell_802_3 <...>/novell_802_3 @@ -2622,6 +2632,7 @@ 0.000000 | HookLoadFileExtended ./main <...>/main.zeek 0.000000 | HookLoadFileExtended ./main.zeek <...>/main.zeek 0.000000 | HookLoadFileExtended ./messaging.bif.zeek <...>/messaging.bif.zeek +0.000000 | HookLoadFileExtended ./mmdb.bif.zeek <...>/mmdb.bif.zeek 0.000000 | HookLoadFileExtended ./office <...>/office.sig 0.000000 | HookLoadFileExtended ./option.bif.zeek <...>/option.bif.zeek 0.000000 | HookLoadFileExtended ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek @@ -2712,6 +2723,7 @@ 0.000000 | HookLoadFileExtended base<...>/main <...>/main.zeek 0.000000 | HookLoadFileExtended base<...>/main.zeek <...>/main.zeek 0.000000 | HookLoadFileExtended base<...>/messaging.bif <...>/messaging.bif.zeek +0.000000 | HookLoadFileExtended base<...>/mmdb.bif <...>/mmdb.bif.zeek 0.000000 | HookLoadFileExtended base<...>/mpls <...>/mpls 0.000000 | HookLoadFileExtended base<...>/nflog <...>/nflog 0.000000 | HookLoadFileExtended base<...>/novell_802_3 <...>/novell_802_3 diff --git a/testing/btest/core/mmdb/explicit-open.zeek b/testing/btest/core/mmdb/explicit-open.zeek new file mode 100644 index 0000000000..9b8c6ddebb --- /dev/null +++ b/testing/btest/core/mmdb/explicit-open.zeek @@ -0,0 +1,50 @@ +# @TEST-DOC: verifies that the explicit BiFs for loading MMDBs work, including when re-opening. +# +# Like other MMDB tests, this uses a pcap to use each packet as a driver to +# touch the DBs involved upon each packet, triggering DB reloads. +# +# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h +# +# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb +# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log + +@load base/frameworks/reporter + +global pkt = 0; +global asn_fn = "./mmdb/GeoLite2-ASN.mmdb"; +global city_fn = "./mmdb/GeoLite2-City.mmdb"; + +function timestamp(n: count): string + { + assert n <= 60; + return fmt("2020-01-01T00:%s:00", n); + } + +event new_packet(c: connection, p: pkt_hdr) + { + ++pkt; + + print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1); + print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1); + print network_time(), pkt, 131.243.0.1, "asn", lookup_autonomous_system(131.243.0.1); + print network_time(), pkt, 131.243.0.1, "location", lookup_location(131.243.0.1); + + # Increment MMDBs' modification time, triggering a re-open. + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(asn_fn)), "") ) + exit(1); + + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(city_fn)), "") ) + exit(1); + + if ( pkt == 4 ) + terminate(); + } + +event zeek_init() + { + assert mmdb_open_asn_db(asn_fn); + assert mmdb_open_location_db(city_fn); + } diff --git a/testing/btest/core/mmdb/temporary-error.zeek b/testing/btest/core/mmdb/temporary-error.zeek index 4af1b05fa4..78dea4af2a 100644 --- a/testing/btest/core/mmdb/temporary-error.zeek +++ b/testing/btest/core/mmdb/temporary-error.zeek @@ -13,6 +13,7 @@ @load base/frameworks/reporter redef mmdb_dir = "./mmdb"; +redef mmdb_dir_fallbacks = vector(); # Clear out fallbacks to avoid influence on tests global pkt = 0;