From b4725c113f9a5837d3419a0f7476d3d77e989a27 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 5 Jan 2024 16:52:28 -0800 Subject: [PATCH 01/11] Sort toplevel .bif list in CMakeLists Not important, but here it is safe to do so and the list is getting lengthy. --- src/CMakeLists.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4101e97be1..ef779e13e7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -111,27 +111,27 @@ include(BifCl) set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc) set(BIF_SRCS - zeek.bif communityid.bif - stats.bif - event.bif const.bif - types.bif - strings.bif - reporter.bif + event.bif option.bif - # Note: the supervisor BIF file is treated like other top-level BIFs instead - # of contained in its own subdirectory CMake logic because subdirectory BIFs - # are treated differently and don't support being called *during* parsing - # (e.g. within an @if directive). - supervisor/supervisor.bif + reporter.bif + stats.bif + strings.bif + types.bif + zeek.bif # The packet analysis BIF is treated like other top-level BIFs because it's # needed before parsing the packet protocol scripts, which happen very near # to the start of parsing. packet_analysis/packet_analysis.bif # The C++ loading BIF is treated like other top-level BIFs to give us # flexibility regarding when it's called. - script_opt/CPP/CPP-load.bif) + script_opt/CPP/CPP-load.bif + # Note: the supervisor BIF file is treated like other top-level BIFs instead + # of contained in its own subdirectory CMake logic because subdirectory BIFs + # are treated differently and don't support being called *during* parsing + # (e.g. within an @if directive). + supervisor/supervisor.bif) foreach (bift ${BIF_SRCS}) bif_target(${bift} "standard") From 06642d185b35f98dd61b5690bf74dc4f90989585 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 5 Jan 2024 12:31:14 -0800 Subject: [PATCH 02/11] Provide script-level configurability of MaxMind DB placement on disk This lifts the list of fallback directories in which Zeek will look for Maxmind DBs into the script layer, and makes the names of the DB files themselves (previously hardwired) configurable as well. This does not yet change the in-core code; that commit follows. --- scripts/base/init-bare.zeek | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 81c3d1dc31..74bd2288e4 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -1107,6 +1107,25 @@ type geo_autonomous_system: record { ## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support. const mmdb_dir: string = "" &redef; +## Default name of the MaxMind City database file: +const mmdb_city_db: string = "GeoLite2-City.mmdb" &redef; +## Default name of the MaxMind Country database file: +const mmdb_country_db: string = "GeoLite2-Country.mmdb" &redef; +## Default name of the MaxMind ASN database file: +const mmdb_asn_db: string = "GeoLite2-ASN.mmdb" &redef; + +## Fallback locations for MaxMind databases. Zeek attempts these when +## :zeek:see:`mmdb_dir` is not set, or it cannot read a DB file from it. For +## geolocation lookups, Zeek will first attempt to locate the city database in +## each of the fallback locations, and should this fail, attempt to locate the +## country one. +const mmdb_dir_fallbacks: vector of string = vector( + "/usr/share/GeoIP", + "/var/lib/GeoIP", + "/usr/local/share/GeoIP", + "/usr/local/var/GeoIP", +) &redef; + ## Sets the interval for MaxMind DB file staleness checks. When Zeek detects a ## change in inode or modification time, the database is re-opened. Setting ## a negative interval disables staleness checks. From 8406959ae2ffe4795c83e08ecdda17188deb09c8 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 5 Jan 2024 16:54:46 -0800 Subject: [PATCH 03/11] Move MaxMind/GeoIP BiF functionality into separate file --- scripts/base/init-bare.zeek | 1 + src/CMakeLists.txt | 1 + src/Func.cc | 3 + src/mmdb.bif | 518 +++++++++++++++++++++++++++++++++++ src/zeek.bif | 519 ------------------------------------ 5 files changed, 523 insertions(+), 519 deletions(-) create mode 100644 src/mmdb.bif diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 74bd2288e4..9c97e65982 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -2113,6 +2113,7 @@ type gtp_delete_pdp_ctx_response_elements: record { @load base/bif/supervisor.bif @load base/bif/packet_analysis.bif @load base/bif/CPP-load.bif +@load base/bif/mmdb.bif ## Internal function. function add_interface(iold: string, inew: string): string diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef779e13e7..1208c07b76 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -114,6 +114,7 @@ set(BIF_SRCS communityid.bif const.bif event.bif + mmdb.bif option.bif reporter.bif stats.bif diff --git a/src/Func.cc b/src/Func.cc index 20a4a79164..b399c80600 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -52,6 +52,7 @@ #include "supervisor.bif.func_h" #include "packet_analysis.bif.func_h" #include "CPP-load.bif.func_h" +#include "mmdb.bif.func_h" #include "zeek.bif.func_def" #include "communityid.bif.func_def" @@ -62,6 +63,7 @@ #include "supervisor.bif.func_def" #include "packet_analysis.bif.func_def" #include "CPP-load.bif.func_def" +#include "mmdb.bif.func_def" // clang-format on extern RETSIGTYPE sig_handler(int signo); @@ -1049,6 +1051,7 @@ void init_primary_bifs() { #include "CPP-load.bif.func_init" #include "communityid.bif.func_init" +#include "mmdb.bif.func_init" #include "option.bif.func_init" #include "packet_analysis.bif.func_init" #include "reporter.bif.func_init" diff --git a/src/mmdb.bif b/src/mmdb.bif new file mode 100644 index 0000000000..be47a05233 --- /dev/null +++ b/src/mmdb.bif @@ -0,0 +1,518 @@ +%%{ +#ifdef USE_GEOIP +#include + +extern "C" { +#include +#include +#include +#include +#include +} + +static int mmdb_msg_count = 0; +static constexpr int mmdb_msg_limit = 20; +static double mmdb_msg_suppression_time = 0; +static constexpr double mmdb_msg_suppression_duration = 300; + +static void report_mmdb_msg(const char* format, ...) + { + if ( zeek::run_state::network_time > mmdb_msg_suppression_time + mmdb_msg_suppression_duration ) + { + mmdb_msg_count = 0; + mmdb_msg_suppression_time = zeek::run_state::network_time; + } + + if ( mmdb_msg_count >= mmdb_msg_limit ) + return; + + ++mmdb_msg_count; + + va_list al; + va_start(al, format); + std::string msg = zeek::util::vfmt(format, al); + va_end(al); + + zeek::reporter->Info("%s", msg.data()); + } + +class MMDB { +public: + MMDB(const char* filename, struct stat info); + + ~MMDB(); + + MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); + bool StaleDB(); + const char* Filename(); + +private: + MMDB_s mmdb; + struct stat file_info; + bool lookup_error; + double last_check; +}; + +MMDB::MMDB(const char* filename, struct stat info) + : file_info(info), lookup_error{false}, + last_check{zeek::run_state::network_time} + { + int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); + + if ( MMDB_SUCCESS != status ) + { + throw std::runtime_error(MMDB_strerror(status)); + } + } + +MMDB::~MMDB() + { + MMDB_close(&mmdb); + } + +MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) + { + int mmdb_error; + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); + + if ( MMDB_SUCCESS != mmdb_error ) + { + lookup_error = true; + throw std::runtime_error(MMDB_strerror(mmdb_error)); + } + + return result; + } + +// Check to see if the Maxmind DB should be closed and reopened. This will +// happen if there was a lookup error or if the mmap'd file has been replaced +// by an external process. +bool MMDB::StaleDB() + { + struct stat buf; + + if ( lookup_error ) + return true; + + + static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); + + if ( mmdb_stale_check_interval < 0.0 ) + return false; + + if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) + return false; + + last_check = zeek::run_state::network_time; + + if ( 0 != stat(mmdb.filename, &buf) ) + return true; + + if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) + { + report_mmdb_msg("%s change detected for MaxMind DB [%s]", + buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", + mmdb.filename); + return true; + } + + return false; + } + +const char* MMDB::Filename() + { + return mmdb.filename; + } + +std::unique_ptr mmdb_loc; +std::unique_ptr mmdb_asn; +static bool did_mmdb_loc_db_error = false; +static bool did_mmdb_asn_db_error = false; + +static bool mmdb_open(const char* filename, bool asn) + { + struct stat buf; + + if ( 0 != stat(filename, &buf) ) + { + return false; + } + + try + { + if ( asn ) + { + mmdb_asn.reset(new MMDB(filename, buf)); + } + else + { + mmdb_loc.reset(new MMDB(filename, buf)); + } + } + + catch ( const std::exception& e ) + { + if ( asn ) + did_mmdb_asn_db_error = false; + else + did_mmdb_loc_db_error = false; + + report_mmdb_msg("Failed to open MaxMind DB: %s [%s]", filename, + e.what()); + return false; + } + + return true; + } + +static bool mmdb_open_loc(const char* filename) + { + return mmdb_open(filename, false); + } + +static bool mmdb_open_asn(const char* filename) + { + return mmdb_open(filename, true); + } + +static void mmdb_check_loc() + { + if ( mmdb_loc && mmdb_loc->StaleDB() ) + { + report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); + did_mmdb_loc_db_error = false; + mmdb_loc.reset(); + } + } + +static void mmdb_check_asn() + { + if ( mmdb_asn && mmdb_asn->StaleDB() ) + { + report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); + did_mmdb_asn_db_error = false; + mmdb_asn.reset(); + } + } + +static bool mmdb_lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result, + bool asn) + { + struct sockaddr_storage ss = {0}; + + if ( IPv4 == addr.GetFamily() ) + { + struct sockaddr_in* sa = (struct sockaddr_in*)&ss; + sa->sin_family = AF_INET; + addr.CopyIPv4(&sa->sin_addr); + } + + else + { + struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; + sa->sin6_family = AF_INET6; + addr.CopyIPv6(&sa->sin6_addr); + } + + try + { + result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss) + : mmdb_loc->Lookup((struct sockaddr*)&ss); + } + + catch ( const std::exception& e ) + { + report_mmdb_msg("MaxMind DB lookup location error [%s]", e.what()); + return false; + } + + return result.found_entry; + } + +static bool mmdb_lookup_loc(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) + { + return mmdb_lookup(addr, result, false); + } + +static bool mmdb_lookup_asn(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) + { + return mmdb_lookup(addr, result, true); + } + +static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, + int data_type ) + { + switch (status) + { + case MMDB_SUCCESS: + if ( entry_data->has_data ) + { + switch (data_type) + { + case MMDB_DATA_TYPE_UTF8_STRING: + return zeek::make_intrusive( + entry_data->data_size, entry_data->utf8_string); + break; + + case MMDB_DATA_TYPE_DOUBLE: + return zeek::make_intrusive(entry_data->double_value); + break; + + case MMDB_DATA_TYPE_UINT32: + return zeek::val_mgr->Count(entry_data->uint32); + + default: + break; + } + } + break; + + case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: + // key doesn't exist, nothing to do + break; + + default: + report_mmdb_msg("MaxMind DB error [%s]", MMDB_strerror(status)); + break; + } + + return nullptr; + } + +static bool mmdb_try_open_loc () + { + // City database is always preferred over Country database. + const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); + std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + + if ( ! mmdb_dir.empty() ) + { + auto d = mmdb_dir + "/GeoLite2-City.mmdb"; + + if ( mmdb_open_loc(d.data()) ) + return true; + + d = mmdb_dir + "/GeoLite2-Country.mmdb"; + + if ( mmdb_open_loc(d.data()) ) + return true;; + } + + return mmdb_open_loc("/usr/share/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/usr/share/GeoIP/GeoLite2-Country.mmdb") + || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-Country.mmdb") + || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-Country.mmdb") + || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-Country.mmdb"); + } + +static bool mmdb_try_open_asn () + { + const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); + std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + + if ( ! mmdb_dir.empty() ) + { + auto d = mmdb_dir + "/GeoLite2-ASN.mmdb"; + + if ( mmdb_open_asn(d.data()) ) + return true; + } + + return mmdb_open_asn("/usr/share/GeoIP/GeoLite2-ASN.mmdb") + || mmdb_open_asn("/var/lib/GeoIP/GeoLite2-ASN.mmdb") + || mmdb_open_asn("/usr/local/share/GeoIP/GeoLite2-ASN.mmdb") + || mmdb_open_asn("/usr/local/var/GeoIP/GeoLite2-ASN.mmdb"); + } + +#endif +%%} + +## Initializes MMDB for later use of lookup_location. +## Requires Zeek to be built with ``libmaxminddb``. +## +## f: The filename of the MaxMind City or Country DB. +## +## Returns: A boolean indicating whether the db was successfully opened. +## +## .. zeek:see:: lookup_autonomous_system +function mmdb_open_location_db%(f: string%) : bool + %{ +#ifdef USE_GEOIP + return zeek::val_mgr->Bool(mmdb_open_loc(f->CheckString())); +#else + return zeek::val_mgr->False(); +#endif + %} + +## Initializes MMDB for later use of lookup_autonomous_system. +## Requires Zeek to be built with ``libmaxminddb``. +## +## f: The filename of the MaxMind ASN DB. +## +## Returns: A boolean indicating whether the db was successfully opened. +## +## .. zeek:see:: lookup_autonomous_system +function mmdb_open_asn_db%(f: string%) : bool + %{ +#ifdef USE_GEOIP + return zeek::val_mgr->Bool(mmdb_open_asn(f->CheckString())); +#else + return zeek::val_mgr->False(); +#endif + %} + +## Performs a geo-lookup of an IP address. +## Requires Zeek to be built with ``libmaxminddb``. +## +## a: The IP address to lookup. +## +## Returns: A record with country, region, city, latitude, and longitude. +## +## .. zeek:see:: lookup_autonomous_system +function lookup_location%(a: addr%) : geo_location + %{ + static auto geo_location = zeek::id::find_type("geo_location"); + auto location = zeek::make_intrusive(geo_location); + +#ifdef USE_GEOIP + mmdb_check_loc(); + if ( ! mmdb_loc ) + { + if ( ! mmdb_try_open_loc() ) + { + if ( ! did_mmdb_loc_db_error ) + { + did_mmdb_loc_db_error = true; + zeek::emit_builtin_error("Failed to open GeoIP location database"); + } + + return location; + } + } + + MMDB_lookup_result_s result; + + if ( mmdb_lookup_loc(a->AsAddr(), result) ) + { + MMDB_entry_data_s entry_data; + int status; + + // Get Country ISO Code + status = MMDB_get_value(&result.entry, &entry_data, + "country", "iso_code", nullptr); + location->Assign(0, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UTF8_STRING)); + + // Get Major Subdivision ISO Code + status = MMDB_get_value(&result.entry, &entry_data, + "subdivisions", "0", "iso_code", nullptr); + location->Assign(1, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UTF8_STRING)); + + // Get City English Name + status = MMDB_get_value(&result.entry, &entry_data, + "city", "names", "en", nullptr); + location->Assign(2, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UTF8_STRING)); + + // Get Location Latitude + status = MMDB_get_value(&result.entry, &entry_data, + "location", "latitude", nullptr); + location->Assign(3, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_DOUBLE)); + + // Get Location Longitude + status = MMDB_get_value(&result.entry, &entry_data, + "location", "longitude", nullptr); + location->Assign(4, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_DOUBLE)); + + return location; + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) + { + zeek::emit_builtin_error("Zeek was not configured for GeoIP support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have MMDB support if we weren't + // able to initialize it or it didn't return any information for + // the address. + + return std::move(location); + %} + +## Performs an lookup of AS number & organization of an IP address. +## Requires Zeek to be built with ``libmaxminddb``. +## +## a: The IP address to lookup. +## +## Returns: A record with autonomous system number and organization that contains *a*. +## +## .. zeek:see:: lookup_location +function lookup_autonomous_system%(a: addr%) : geo_autonomous_system + %{ + static auto geo_autonomous_system = zeek::id::find_type("geo_autonomous_system"); + auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); + +#ifdef USE_GEOIP + mmdb_check_asn(); + if ( ! mmdb_asn ) + { + if ( ! mmdb_try_open_asn() ) + { + if ( ! did_mmdb_asn_db_error ) + { + did_mmdb_asn_db_error = true; + zeek::emit_builtin_error("Failed to open GeoIP ASN database"); + } + + return autonomous_system; + } + } + + MMDB_lookup_result_s result; + + if ( mmdb_lookup_asn(a->AsAddr(), result) ) + { + MMDB_entry_data_s entry_data; + int status; + + // Get Autonomous System Number + status = MMDB_get_value(&result.entry, &entry_data, + "autonomous_system_number", nullptr); + autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UINT32)); + + // Get Autonomous System Organization + status = MMDB_get_value(&result.entry, &entry_data, + "autonomous_system_organization", nullptr); + autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UTF8_STRING)); + + return autonomous_system; + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) + { + zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have GeoIP support, if we weren't + // able to initialize it or it didn't return any information for + // the address. + return std::move(autonomous_system); + %} diff --git a/src/zeek.bif b/src/zeek.bif index dd04941170..770524ebda 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -3980,525 +3980,6 @@ function lookup_hostname%(host: string%) : addr_set return nullptr; %} -%%{ -#ifdef USE_GEOIP -#include - -extern "C" { -#include -#include -#include -#include -#include -} - -static int mmdb_msg_count = 0; -static constexpr int mmdb_msg_limit = 20; -static double mmdb_msg_suppression_time = 0; -static constexpr double mmdb_msg_suppression_duration = 300; - -static void report_mmdb_msg(const char* format, ...) - { - if ( zeek::run_state::network_time > mmdb_msg_suppression_time + mmdb_msg_suppression_duration ) - { - mmdb_msg_count = 0; - mmdb_msg_suppression_time = zeek::run_state::network_time; - } - - if ( mmdb_msg_count >= mmdb_msg_limit ) - return; - - ++mmdb_msg_count; - - va_list al; - va_start(al, format); - std::string msg = zeek::util::vfmt(format, al); - va_end(al); - - zeek::reporter->Info("%s", msg.data()); - } - -class MMDB { -public: - MMDB(const char* filename, struct stat info); - - ~MMDB(); - - MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); - bool StaleDB(); - const char* Filename(); - -private: - MMDB_s mmdb; - struct stat file_info; - bool lookup_error; - double last_check; -}; - -MMDB::MMDB(const char* filename, struct stat info) - : file_info(info), lookup_error{false}, - last_check{zeek::run_state::network_time} - { - int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); - - if ( MMDB_SUCCESS != status ) - { - throw std::runtime_error(MMDB_strerror(status)); - } - } - -MMDB::~MMDB() - { - MMDB_close(&mmdb); - } - -MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) - { - int mmdb_error; - MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); - - if ( MMDB_SUCCESS != mmdb_error ) - { - lookup_error = true; - throw std::runtime_error(MMDB_strerror(mmdb_error)); - } - - return result; - } - -// Check to see if the Maxmind DB should be closed and reopened. This will -// happen if there was a lookup error or if the mmap'd file has been replaced -// by an external process. -bool MMDB::StaleDB() - { - struct stat buf; - - if ( lookup_error ) - return true; - - - static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); - - if ( mmdb_stale_check_interval < 0.0 ) - return false; - - if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) - return false; - - last_check = zeek::run_state::network_time; - - if ( 0 != stat(mmdb.filename, &buf) ) - return true; - - if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) - { - report_mmdb_msg("%s change detected for MaxMind DB [%s]", - buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", - mmdb.filename); - return true; - } - - return false; - } - -const char* MMDB::Filename() - { - return mmdb.filename; - } - -std::unique_ptr mmdb_loc; -std::unique_ptr mmdb_asn; -static bool did_mmdb_loc_db_error = false; -static bool did_mmdb_asn_db_error = false; - -static bool mmdb_open(const char* filename, bool asn) - { - struct stat buf; - - if ( 0 != stat(filename, &buf) ) - { - return false; - } - - try - { - if ( asn ) - { - mmdb_asn.reset(new MMDB(filename, buf)); - } - else - { - mmdb_loc.reset(new MMDB(filename, buf)); - } - } - - catch ( const std::exception& e ) - { - if ( asn ) - did_mmdb_asn_db_error = false; - else - did_mmdb_loc_db_error = false; - - report_mmdb_msg("Failed to open MaxMind DB: %s [%s]", filename, - e.what()); - return false; - } - - return true; - } - -static bool mmdb_open_loc(const char* filename) - { - return mmdb_open(filename, false); - } - -static bool mmdb_open_asn(const char* filename) - { - return mmdb_open(filename, true); - } - -static void mmdb_check_loc() - { - if ( mmdb_loc && mmdb_loc->StaleDB() ) - { - report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); - did_mmdb_loc_db_error = false; - mmdb_loc.reset(); - } - } - -static void mmdb_check_asn() - { - if ( mmdb_asn && mmdb_asn->StaleDB() ) - { - report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); - did_mmdb_asn_db_error = false; - mmdb_asn.reset(); - } - } - -static bool mmdb_lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result, - bool asn) - { - struct sockaddr_storage ss = {0}; - - if ( IPv4 == addr.GetFamily() ) - { - struct sockaddr_in* sa = (struct sockaddr_in*)&ss; - sa->sin_family = AF_INET; - addr.CopyIPv4(&sa->sin_addr); - } - - else - { - struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; - sa->sin6_family = AF_INET6; - addr.CopyIPv6(&sa->sin6_addr); - } - - try - { - result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss) - : mmdb_loc->Lookup((struct sockaddr*)&ss); - } - - catch ( const std::exception& e ) - { - report_mmdb_msg("MaxMind DB lookup location error [%s]", e.what()); - return false; - } - - return result.found_entry; - } - -static bool mmdb_lookup_loc(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) - { - return mmdb_lookup(addr, result, false); - } - -static bool mmdb_lookup_asn(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) - { - return mmdb_lookup(addr, result, true); - } - -static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, - int data_type ) - { - switch (status) - { - case MMDB_SUCCESS: - if ( entry_data->has_data ) - { - switch (data_type) - { - case MMDB_DATA_TYPE_UTF8_STRING: - return zeek::make_intrusive( - entry_data->data_size, entry_data->utf8_string); - break; - - case MMDB_DATA_TYPE_DOUBLE: - return zeek::make_intrusive(entry_data->double_value); - break; - - case MMDB_DATA_TYPE_UINT32: - return zeek::val_mgr->Count(entry_data->uint32); - - default: - break; - } - } - break; - - case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: - // key doesn't exist, nothing to do - break; - - default: - report_mmdb_msg("MaxMind DB error [%s]", MMDB_strerror(status)); - break; - } - - return nullptr; - } - -static bool mmdb_try_open_loc () - { - // City database is always preferred over Country database. - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); - - if ( ! mmdb_dir.empty() ) - { - auto d = mmdb_dir + "/GeoLite2-City.mmdb"; - - if ( mmdb_open_loc(d.data()) ) - return true; - - d = mmdb_dir + "/GeoLite2-Country.mmdb"; - - if ( mmdb_open_loc(d.data()) ) - return true;; - } - - return mmdb_open_loc("/usr/share/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/share/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-Country.mmdb"); - } - -static bool mmdb_try_open_asn () - { - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); - - if ( ! mmdb_dir.empty() ) - { - auto d = mmdb_dir + "/GeoLite2-ASN.mmdb"; - - if ( mmdb_open_asn(d.data()) ) - return true; - } - - return mmdb_open_asn("/usr/share/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/var/lib/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/usr/local/share/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/usr/local/var/GeoIP/GeoLite2-ASN.mmdb"); - } - -#endif -%%} - -## Initializes MMDB for later use of lookup_location. -## Requires Zeek to be built with ``libmaxminddb``. -## -## f: The filename of the MaxMind City or Country DB. -## -## Returns: A boolean indicating whether the db was successfully opened. -## -## .. zeek:see:: lookup_autonomous_system -function mmdb_open_location_db%(f: string%) : bool - %{ -#ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_loc(f->CheckString())); -#else - return zeek::val_mgr->False(); -#endif - %} - -## Initializes MMDB for later use of lookup_autonomous_system. -## Requires Zeek to be built with ``libmaxminddb``. -## -## f: The filename of the MaxMind ASN DB. -## -## Returns: A boolean indicating whether the db was successfully opened. -## -## .. zeek:see:: lookup_autonomous_system -function mmdb_open_asn_db%(f: string%) : bool - %{ -#ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_asn(f->CheckString())); -#else - return zeek::val_mgr->False(); -#endif - %} - -## Performs a geo-lookup of an IP address. -## Requires Zeek to be built with ``libmaxminddb``. -## -## a: The IP address to lookup. -## -## Returns: A record with country, region, city, latitude, and longitude. -## -## .. zeek:see:: lookup_autonomous_system -function lookup_location%(a: addr%) : geo_location - %{ - static auto geo_location = zeek::id::find_type("geo_location"); - auto location = zeek::make_intrusive(geo_location); - -#ifdef USE_GEOIP - mmdb_check_loc(); - if ( ! mmdb_loc ) - { - if ( ! mmdb_try_open_loc() ) - { - if ( ! did_mmdb_loc_db_error ) - { - did_mmdb_loc_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP location database"); - } - - return location; - } - } - - MMDB_lookup_result_s result; - - if ( mmdb_lookup_loc(a->AsAddr(), result) ) - { - MMDB_entry_data_s entry_data; - int status; - - // Get Country ISO Code - status = MMDB_get_value(&result.entry, &entry_data, - "country", "iso_code", nullptr); - location->Assign(0, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get Major Subdivision ISO Code - status = MMDB_get_value(&result.entry, &entry_data, - "subdivisions", "0", "iso_code", nullptr); - location->Assign(1, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get City English Name - status = MMDB_get_value(&result.entry, &entry_data, - "city", "names", "en", nullptr); - location->Assign(2, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get Location Latitude - status = MMDB_get_value(&result.entry, &entry_data, - "location", "latitude", nullptr); - location->Assign(3, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_DOUBLE)); - - // Get Location Longitude - status = MMDB_get_value(&result.entry, &entry_data, - "location", "longitude", nullptr); - location->Assign(4, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_DOUBLE)); - - return location; - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - zeek::emit_builtin_error("Zeek was not configured for GeoIP support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have MMDB support if we weren't - // able to initialize it or it didn't return any information for - // the address. - - return std::move(location); - %} - -## Performs an lookup of AS number & organization of an IP address. -## Requires Zeek to be built with ``libmaxminddb``. -## -## a: The IP address to lookup. -## -## Returns: A record with autonomous system number and organization that contains *a*. -## -## .. zeek:see:: lookup_location -function lookup_autonomous_system%(a: addr%) : geo_autonomous_system - %{ - static auto geo_autonomous_system = zeek::id::find_type("geo_autonomous_system"); - auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); - -#ifdef USE_GEOIP - mmdb_check_asn(); - if ( ! mmdb_asn ) - { - if ( ! mmdb_try_open_asn() ) - { - if ( ! did_mmdb_asn_db_error ) - { - did_mmdb_asn_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP ASN database"); - } - - return autonomous_system; - } - } - - MMDB_lookup_result_s result; - - if ( mmdb_lookup_asn(a->AsAddr(), result) ) - { - MMDB_entry_data_s entry_data; - int status; - - // Get Autonomous System Number - status = MMDB_get_value(&result.entry, &entry_data, - "autonomous_system_number", nullptr); - autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UINT32)); - - // Get Autonomous System Organization - status = MMDB_get_value(&result.entry, &entry_data, - "autonomous_system_organization", nullptr); - autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - return autonomous_system; - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have GeoIP support, if we weren't - // able to initialize it or it didn't return any information for - // the address. - return std::move(autonomous_system); - %} - ## Calculates distance between two geographic locations using the haversine ## formula. Latitudes and longitudes must be given in degrees, where southern ## hemisphere latitudes are negative and western hemisphere longitudes are From 4e45a3462bd00d6a4e82617fc05999fa4b2fcfa1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 10 Jan 2024 10:11:30 -0800 Subject: [PATCH 04/11] Update btest baselines to reflect introduction of mmdb.bif --- .../canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 1 + testing/btest/Baseline/plugins.hooks/output | 12 ++++++++++++ 3 files changed, 14 insertions(+) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index f7ca106dc0..0bdbda72a4 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -20,6 +20,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/supervisor.bif.zeek build/scripts/base/bif/packet_analysis.bif.zeek build/scripts/base/bif/CPP-load.bif.zeek + build/scripts/base/bif/mmdb.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek build/scripts/base/bif/event.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index ce766b6a49..29feb83dfb 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -20,6 +20,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/supervisor.bif.zeek build/scripts/base/bif/packet_analysis.bif.zeek build/scripts/base/bif/CPP-load.bif.zeek + build/scripts/base/bif/mmdb.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek build/scripts/base/bif/event.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 5bebc930f3..33c34f0daf 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -493,6 +493,7 @@ 0.000000 MetaHookPost LoadFile(0, ./main, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./main.zeek, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./patterns, <...>/patterns.zeek) -> -1 @@ -580,6 +581,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/main, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/main.zeek, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/mpls, <...>/mpls) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/nflog, <...>/nflog) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3) -> -1 @@ -778,6 +780,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./main, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./main.zeek, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./patterns, <...>/patterns.zeek) -> (-1, ) @@ -865,6 +868,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/main, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/mpls, <...>/mpls) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/nflog, <...>/nflog) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3) -> (-1, ) @@ -1411,6 +1415,7 @@ 0.000000 MetaHookPre LoadFile(0, ./main, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, ./main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./patterns, <...>/patterns.zeek) @@ -1498,6 +1503,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/main, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/mpls, <...>/mpls) 0.000000 MetaHookPre LoadFile(0, base<...>/nflog, <...>/nflog) 0.000000 MetaHookPre LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3) @@ -1696,6 +1702,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./main, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./patterns, <...>/patterns.zeek) @@ -1783,6 +1790,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/main, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/mpls, <...>/mpls) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/nflog, <...>/nflog) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3) @@ -2337,6 +2345,7 @@ 0.000000 | HookLoadFile ./main <...>/main.zeek 0.000000 | HookLoadFile ./main.zeek <...>/main.zeek 0.000000 | HookLoadFile ./messaging.bif.zeek <...>/messaging.bif.zeek +0.000000 | HookLoadFile ./mmdb.bif.zeek <...>/mmdb.bif.zeek 0.000000 | HookLoadFile ./office <...>/office.sig 0.000000 | HookLoadFile ./option.bif.zeek <...>/option.bif.zeek 0.000000 | HookLoadFile ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek @@ -2427,6 +2436,7 @@ 0.000000 | HookLoadFile base<...>/main <...>/main.zeek 0.000000 | HookLoadFile base<...>/main.zeek <...>/main.zeek 0.000000 | HookLoadFile base<...>/messaging.bif <...>/messaging.bif.zeek +0.000000 | HookLoadFile base<...>/mmdb.bif <...>/mmdb.bif.zeek 0.000000 | HookLoadFile base<...>/mpls <...>/mpls 0.000000 | HookLoadFile base<...>/nflog <...>/nflog 0.000000 | HookLoadFile base<...>/novell_802_3 <...>/novell_802_3 @@ -2622,6 +2632,7 @@ 0.000000 | HookLoadFileExtended ./main <...>/main.zeek 0.000000 | HookLoadFileExtended ./main.zeek <...>/main.zeek 0.000000 | HookLoadFileExtended ./messaging.bif.zeek <...>/messaging.bif.zeek +0.000000 | HookLoadFileExtended ./mmdb.bif.zeek <...>/mmdb.bif.zeek 0.000000 | HookLoadFileExtended ./office <...>/office.sig 0.000000 | HookLoadFileExtended ./option.bif.zeek <...>/option.bif.zeek 0.000000 | HookLoadFileExtended ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek @@ -2712,6 +2723,7 @@ 0.000000 | HookLoadFileExtended base<...>/main <...>/main.zeek 0.000000 | HookLoadFileExtended base<...>/main.zeek <...>/main.zeek 0.000000 | HookLoadFileExtended base<...>/messaging.bif <...>/messaging.bif.zeek +0.000000 | HookLoadFileExtended base<...>/mmdb.bif <...>/mmdb.bif.zeek 0.000000 | HookLoadFileExtended base<...>/mpls <...>/mpls 0.000000 | HookLoadFileExtended base<...>/nflog <...>/nflog 0.000000 | HookLoadFileExtended base<...>/novell_802_3 <...>/novell_802_3 From ba98ddc4f250537beb5e85df85d5dfd23e7cee1a Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 5 Jan 2024 17:34:07 -0800 Subject: [PATCH 05/11] Adapt MMDB BiF code to new script-layer variables --- src/mmdb.bif | 58 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/src/mmdb.bif b/src/mmdb.bif index be47a05233..4861a1f550 100644 --- a/src/mmdb.bif +++ b/src/mmdb.bif @@ -285,27 +285,43 @@ static bool mmdb_try_open_loc () const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + const auto& mmdb_city_db_val = zeek::detail::global_scope()->Find("mmdb_city_db")->GetVal(); + std::string mmdb_city_db = mmdb_city_db_val->AsString()->CheckString(); + + const auto& mmdb_country_db_val = zeek::detail::global_scope()->Find("mmdb_country_db")->GetVal(); + std::string mmdb_country_db = mmdb_country_db_val->AsString()->CheckString(); + if ( ! mmdb_dir.empty() ) { - auto d = mmdb_dir + "/GeoLite2-City.mmdb"; + auto d = mmdb_dir + "/" + mmdb_city_db; if ( mmdb_open_loc(d.data()) ) return true; - d = mmdb_dir + "/GeoLite2-Country.mmdb"; + d = mmdb_dir + "/" + mmdb_country_db; if ( mmdb_open_loc(d.data()) ) - return true;; + return true; } - return mmdb_open_loc("/usr/share/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-City.mmdb") - || mmdb_open_loc("/usr/share/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-Country.mmdb") - || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-Country.mmdb"); + const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); + auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); + + for ( unsigned int i = 0; i < vv->Size(); ++i ) + { + auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_city_db; + if ( mmdb_open_loc(d.data()) ) + return true; + } + + for ( unsigned int i = 0; i < vv->Size(); ++i ) + { + auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_country_db; + if ( mmdb_open_loc(d.data()) ) + return true; + } + + return false; } static bool mmdb_try_open_asn () @@ -313,18 +329,28 @@ static bool mmdb_try_open_asn () const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + const auto& mmdb_asn_db_val = zeek::detail::global_scope()->Find("mmdb_asn_db")->GetVal(); + std::string mmdb_asn_db = mmdb_asn_db_val->AsString()->CheckString(); + if ( ! mmdb_dir.empty() ) { - auto d = mmdb_dir + "/GeoLite2-ASN.mmdb"; + auto d = mmdb_dir + "/" + mmdb_asn_db; if ( mmdb_open_asn(d.data()) ) return true; } - return mmdb_open_asn("/usr/share/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/var/lib/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/usr/local/share/GeoIP/GeoLite2-ASN.mmdb") - || mmdb_open_asn("/usr/local/var/GeoIP/GeoLite2-ASN.mmdb"); + const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); + auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); + + for ( unsigned int i = 0; i < vv->Size(); ++i ) + { + auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_asn_db; + if ( mmdb_open_asn(d.data()) ) + return true; + } + + return false; } #endif From 20841ac689da7f3973f57be39122cfd33d75f8b1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 5 Jan 2024 18:41:44 -0800 Subject: [PATCH 06/11] Fix mmdb.temporary-error testcase when MMDBs are installed on system The test would previously fail in settings where the user has Maxmind DBs installed in the hardwired system locations, because the fallback logic still picked those up. --- .../btest/Baseline/core.mmdb.temporary-error/reporter.log | 8 ++++---- testing/btest/core/mmdb/temporary-error.zeek | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log index 8dcd9fb5c1..6c9ef5749e 100644 --- a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log +++ b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log @@ -3,15 +3,15 @@ ts level message location 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 100 +1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 101 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 101 +1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 102 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 100 +1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 101 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 101 +1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 102 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 diff --git a/testing/btest/core/mmdb/temporary-error.zeek b/testing/btest/core/mmdb/temporary-error.zeek index 4af1b05fa4..78dea4af2a 100644 --- a/testing/btest/core/mmdb/temporary-error.zeek +++ b/testing/btest/core/mmdb/temporary-error.zeek @@ -13,6 +13,7 @@ @load base/frameworks/reporter redef mmdb_dir = "./mmdb"; +redef mmdb_dir_fallbacks = vector(); # Clear out fallbacks to avoid influence on tests global pkt = 0; From a06053ce75ab30c49b98a2327be0f110562ed8c4 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Tue, 9 Jan 2024 21:18:35 -0800 Subject: [PATCH 07/11] Move MMDB logic out of mmdb.bif and into MMDB.cc/h. This does not change the implementation except for some light renaming where things are now naturally scoped within MMDB.cc. --- src/CMakeLists.txt | 1 + src/MMDB.cc | 412 +++++++++++++++++++++++++++++++++++++ src/MMDB.h | 38 ++++ src/mmdb.bif | 499 +-------------------------------------------- 4 files changed, 456 insertions(+), 494 deletions(-) create mode 100644 src/MMDB.cc create mode 100644 src/MMDB.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1208c07b76..8b1007416f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -329,6 +329,7 @@ set(MAIN_SRCS IP.cc IPAddr.cc List.cc + MMDB.cc Reporter.cc NFA.cc NetVar.cc diff --git a/src/MMDB.cc b/src/MMDB.cc new file mode 100644 index 0000000000..42a6f04726 --- /dev/null +++ b/src/MMDB.cc @@ -0,0 +1,412 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/MMDB.h" + +#include +#include +#include +#include +#include + +#include "zeek/Func.h" +#include "zeek/IPAddr.h" +#include "zeek/ZeekString.h" + +namespace zeek { + +#ifdef USE_GEOIP + +static int msg_count = 0; +static double msg_suppression_time = 0; +static bool did_loc_db_error = false; +static bool did_asn_db_error = false; +static constexpr int msg_limit = 20; +static constexpr double msg_suppression_duration = 300; + +static std::unique_ptr mmdb_loc; +static std::unique_ptr mmdb_asn; + +static void report_msg(const char* format, ...) { + if ( zeek::run_state::network_time > msg_suppression_time + msg_suppression_duration ) { + msg_count = 0; + msg_suppression_time = zeek::run_state::network_time; + } + + if ( msg_count >= msg_limit ) + return; + + ++msg_count; + + va_list al; + va_start(al, format); + std::string msg = zeek::util::vfmt(format, al); + va_end(al); + + zeek::reporter->Info("%s", msg.data()); +} + +MMDB::MMDB(const char* filename, struct stat info) + : file_info(info), lookup_error{false}, last_check{zeek::run_state::network_time} { + int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); + + if ( MMDB_SUCCESS != status ) { + throw std::runtime_error(MMDB_strerror(status)); + } +} + +MMDB::~MMDB() { MMDB_close(&mmdb); } + +MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) { + int mmdb_error; + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); + + if ( MMDB_SUCCESS != mmdb_error ) { + lookup_error = true; + throw std::runtime_error(MMDB_strerror(mmdb_error)); + } + + return result; +} + +// Check to see if the Maxmind DB should be closed and reopened. This will +// happen if there was a lookup error or if the mmap'd file has been replaced +// by an external process. +bool MMDB::StaleDB() { + struct stat buf; + + if ( lookup_error ) + return true; + + static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); + + if ( mmdb_stale_check_interval < 0.0 ) + return false; + + if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) + return false; + + last_check = zeek::run_state::network_time; + + if ( 0 != stat(mmdb.filename, &buf) ) + return true; + + if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) { + report_msg("%s change detected for MaxMind DB [%s]", + buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", mmdb.filename); + return true; + } + + return false; +} + +const char* MMDB::Filename() { return mmdb.filename; } + +static bool mmdb_open(const char* filename, bool asn) { + struct stat buf; + + if ( 0 != stat(filename, &buf) ) { + return false; + } + + try { + if ( asn ) { + mmdb_asn.reset(new MMDB(filename, buf)); + } + else { + mmdb_loc.reset(new MMDB(filename, buf)); + } + } + + catch ( const std::exception& e ) { + if ( asn ) + did_asn_db_error = false; + else + did_loc_db_error = false; + + report_msg("Failed to open MaxMind DB: %s [%s]", filename, e.what()); + return false; + } + + return true; +} + +static bool mmdb_open_loc(const char* filename) { return mmdb_open(filename, false); } + +static bool mmdb_open_asn(const char* filename) { return mmdb_open(filename, true); } + +static void mmdb_check_loc() { + if ( mmdb_loc && mmdb_loc->StaleDB() ) { + report_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); + did_loc_db_error = false; + mmdb_loc.reset(); + } +} + +static void mmdb_check_asn() { + if ( mmdb_asn && mmdb_asn->StaleDB() ) { + report_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); + did_asn_db_error = false; + mmdb_asn.reset(); + } +} + +static bool mmdb_lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result, bool asn) { + struct sockaddr_storage ss = {0}; + + if ( IPv4 == addr.GetFamily() ) { + struct sockaddr_in* sa = (struct sockaddr_in*)&ss; + sa->sin_family = AF_INET; + addr.CopyIPv4(&sa->sin_addr); + } + + else { + struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; + sa->sin6_family = AF_INET6; + addr.CopyIPv6(&sa->sin6_addr); + } + + try { + result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss) : mmdb_loc->Lookup((struct sockaddr*)&ss); + } + + catch ( const std::exception& e ) { + report_msg("MaxMind DB lookup location error [%s]", e.what()); + return false; + } + + return result.found_entry; +} + +static bool mmdb_lookup_loc(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { + return mmdb_lookup(addr, result, false); +} + +static bool mmdb_lookup_asn(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { + return mmdb_lookup(addr, result, true); +} + +static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, int data_type) { + switch ( status ) { + case MMDB_SUCCESS: + if ( entry_data->has_data ) { + switch ( data_type ) { + case MMDB_DATA_TYPE_UTF8_STRING: + return zeek::make_intrusive(entry_data->data_size, entry_data->utf8_string); + break; + + case MMDB_DATA_TYPE_DOUBLE: + return zeek::make_intrusive(entry_data->double_value); + break; + + case MMDB_DATA_TYPE_UINT32: return zeek::val_mgr->Count(entry_data->uint32); + + default: break; + } + } + break; + + case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: + // key doesn't exist, nothing to do + break; + + default: report_msg("MaxMind DB error [%s]", MMDB_strerror(status)); break; + } + + return nullptr; +} + +static bool mmdb_try_open_loc() { + // City database is always preferred over Country database. + const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); + std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + + const auto& mmdb_city_db_val = zeek::detail::global_scope()->Find("mmdb_city_db")->GetVal(); + std::string mmdb_city_db = mmdb_city_db_val->AsString()->CheckString(); + + const auto& mmdb_country_db_val = zeek::detail::global_scope()->Find("mmdb_country_db")->GetVal(); + std::string mmdb_country_db = mmdb_country_db_val->AsString()->CheckString(); + + if ( ! mmdb_dir.empty() ) { + auto d = mmdb_dir + "/" + mmdb_city_db; + + if ( mmdb_open_loc(d.data()) ) + return true; + + d = mmdb_dir + "/" + mmdb_country_db; + + if ( mmdb_open_loc(d.data()) ) + return true; + } + + const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); + auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); + + for ( unsigned int i = 0; i < vv->Size(); ++i ) { + auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_city_db; + if ( mmdb_open_loc(d.data()) ) + return true; + } + + for ( unsigned int i = 0; i < vv->Size(); ++i ) { + auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_country_db; + if ( mmdb_open_loc(d.data()) ) + return true; + } + + return false; +} + +static bool mmdb_try_open_asn() { + const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); + std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + + const auto& mmdb_asn_db_val = zeek::detail::global_scope()->Find("mmdb_asn_db")->GetVal(); + std::string mmdb_asn_db = mmdb_asn_db_val->AsString()->CheckString(); + + if ( ! mmdb_dir.empty() ) { + auto d = mmdb_dir + "/" + mmdb_asn_db; + + if ( mmdb_open_asn(d.data()) ) + return true; + } + + const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); + auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); + + for ( unsigned int i = 0; i < vv->Size(); ++i ) { + auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_asn_db; + if ( mmdb_open_loc(d.data()) ) + return true; + } + + return false; +} +#endif // USE_GEOIP + +ValPtr mmdb_open_location_db(StringVal* filename) { +#ifdef USE_GEOIP + return zeek::val_mgr->Bool(mmdb_open_loc(filename->CheckString())); +#else + return zeek::val_mgr->False(); +#endif +} + +ValPtr mmdb_open_asn_db(StringVal* filename) { +#ifdef USE_GEOIP + return zeek::val_mgr->Bool(mmdb_open_asn(filename->CheckString())); +#else + return zeek::val_mgr->False(); +#endif +} + +RecordValPtr mmdb_lookup_location(AddrVal* addr) { + static auto geo_location = zeek::id::find_type("geo_location"); + auto location = zeek::make_intrusive(geo_location); + +#ifdef USE_GEOIP + mmdb_check_loc(); + if ( ! mmdb_loc ) { + if ( ! mmdb_try_open_loc() ) { + if ( ! did_loc_db_error ) { + did_loc_db_error = true; + zeek::emit_builtin_error("Failed to open GeoIP location database"); + } + + return location; + } + } + + MMDB_lookup_result_s result; + + if ( mmdb_lookup_loc(addr->AsAddr(), result) ) { + MMDB_entry_data_s entry_data; + int status; + + // Get Country ISO Code + status = MMDB_get_value(&result.entry, &entry_data, "country", "iso_code", nullptr); + location->Assign(0, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + // Get Major Subdivision ISO Code + status = MMDB_get_value(&result.entry, &entry_data, "subdivisions", "0", "iso_code", nullptr); + location->Assign(1, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + // Get City English Name + status = MMDB_get_value(&result.entry, &entry_data, "city", "names", "en", nullptr); + location->Assign(2, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + // Get Location Latitude + status = MMDB_get_value(&result.entry, &entry_data, "location", "latitude", nullptr); + location->Assign(3, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_DOUBLE)); + + // Get Location Longitude + status = MMDB_get_value(&result.entry, &entry_data, "location", "longitude", nullptr); + location->Assign(4, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_DOUBLE)); + + return location; + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) { + zeek::emit_builtin_error("Zeek was not configured for GeoIP support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have MMDB support if we weren't + // able to initialize it or it didn't return any information for + // the address. + return location; +} + +RecordValPtr mmdb_lookup_autonomous_system(AddrVal* addr) { + static auto geo_autonomous_system = zeek::id::find_type("geo_autonomous_system"); + auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); + +#ifdef USE_GEOIP + mmdb_check_asn(); + if ( ! mmdb_asn ) { + if ( ! mmdb_try_open_asn() ) { + if ( ! did_asn_db_error ) { + did_asn_db_error = true; + zeek::emit_builtin_error("Failed to open GeoIP ASN database"); + } + + return autonomous_system; + } + } + + MMDB_lookup_result_s result; + + if ( mmdb_lookup_asn(addr->AsAddr(), result) ) { + MMDB_entry_data_s entry_data; + int status; + + // Get Autonomous System Number + status = MMDB_get_value(&result.entry, &entry_data, "autonomous_system_number", nullptr); + autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UINT32)); + + // Get Autonomous System Organization + status = MMDB_get_value(&result.entry, &entry_data, "autonomous_system_organization", nullptr); + autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UTF8_STRING)); + + return autonomous_system; + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) { + zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have GeoIP support, if we weren't + // able to initialize it or it didn't return any information for + // the address. + return autonomous_system; +} + +} // namespace zeek diff --git a/src/MMDB.h b/src/MMDB.h new file mode 100644 index 0000000000..23732b3e2b --- /dev/null +++ b/src/MMDB.h @@ -0,0 +1,38 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/Val.h" + +namespace zeek { + +#ifdef USE_GEOIP + +#include + +class MMDB { +public: + MMDB(const char* filename, struct stat info); + + ~MMDB(); + + MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); + bool StaleDB(); + const char* Filename(); + +private: + MMDB_s mmdb; + struct stat file_info; + bool lookup_error; + double last_check; +}; + +#endif // USE_GEOIP + +ValPtr mmdb_open_location_db(zeek::StringVal* filename); +ValPtr mmdb_open_asn_db(zeek::StringVal* filename); + +RecordValPtr mmdb_lookup_location(zeek::AddrVal* addr); +RecordValPtr mmdb_lookup_autonomous_system(zeek::AddrVal* addr); + +} // namespace zeek diff --git a/src/mmdb.bif b/src/mmdb.bif index 4861a1f550..8aa3100aa4 100644 --- a/src/mmdb.bif +++ b/src/mmdb.bif @@ -1,359 +1,5 @@ %%{ -#ifdef USE_GEOIP -#include - -extern "C" { -#include -#include -#include -#include -#include -} - -static int mmdb_msg_count = 0; -static constexpr int mmdb_msg_limit = 20; -static double mmdb_msg_suppression_time = 0; -static constexpr double mmdb_msg_suppression_duration = 300; - -static void report_mmdb_msg(const char* format, ...) - { - if ( zeek::run_state::network_time > mmdb_msg_suppression_time + mmdb_msg_suppression_duration ) - { - mmdb_msg_count = 0; - mmdb_msg_suppression_time = zeek::run_state::network_time; - } - - if ( mmdb_msg_count >= mmdb_msg_limit ) - return; - - ++mmdb_msg_count; - - va_list al; - va_start(al, format); - std::string msg = zeek::util::vfmt(format, al); - va_end(al); - - zeek::reporter->Info("%s", msg.data()); - } - -class MMDB { -public: - MMDB(const char* filename, struct stat info); - - ~MMDB(); - - MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); - bool StaleDB(); - const char* Filename(); - -private: - MMDB_s mmdb; - struct stat file_info; - bool lookup_error; - double last_check; -}; - -MMDB::MMDB(const char* filename, struct stat info) - : file_info(info), lookup_error{false}, - last_check{zeek::run_state::network_time} - { - int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); - - if ( MMDB_SUCCESS != status ) - { - throw std::runtime_error(MMDB_strerror(status)); - } - } - -MMDB::~MMDB() - { - MMDB_close(&mmdb); - } - -MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) - { - int mmdb_error; - MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); - - if ( MMDB_SUCCESS != mmdb_error ) - { - lookup_error = true; - throw std::runtime_error(MMDB_strerror(mmdb_error)); - } - - return result; - } - -// Check to see if the Maxmind DB should be closed and reopened. This will -// happen if there was a lookup error or if the mmap'd file has been replaced -// by an external process. -bool MMDB::StaleDB() - { - struct stat buf; - - if ( lookup_error ) - return true; - - - static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); - - if ( mmdb_stale_check_interval < 0.0 ) - return false; - - if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) - return false; - - last_check = zeek::run_state::network_time; - - if ( 0 != stat(mmdb.filename, &buf) ) - return true; - - if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) - { - report_mmdb_msg("%s change detected for MaxMind DB [%s]", - buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", - mmdb.filename); - return true; - } - - return false; - } - -const char* MMDB::Filename() - { - return mmdb.filename; - } - -std::unique_ptr mmdb_loc; -std::unique_ptr mmdb_asn; -static bool did_mmdb_loc_db_error = false; -static bool did_mmdb_asn_db_error = false; - -static bool mmdb_open(const char* filename, bool asn) - { - struct stat buf; - - if ( 0 != stat(filename, &buf) ) - { - return false; - } - - try - { - if ( asn ) - { - mmdb_asn.reset(new MMDB(filename, buf)); - } - else - { - mmdb_loc.reset(new MMDB(filename, buf)); - } - } - - catch ( const std::exception& e ) - { - if ( asn ) - did_mmdb_asn_db_error = false; - else - did_mmdb_loc_db_error = false; - - report_mmdb_msg("Failed to open MaxMind DB: %s [%s]", filename, - e.what()); - return false; - } - - return true; - } - -static bool mmdb_open_loc(const char* filename) - { - return mmdb_open(filename, false); - } - -static bool mmdb_open_asn(const char* filename) - { - return mmdb_open(filename, true); - } - -static void mmdb_check_loc() - { - if ( mmdb_loc && mmdb_loc->StaleDB() ) - { - report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); - did_mmdb_loc_db_error = false; - mmdb_loc.reset(); - } - } - -static void mmdb_check_asn() - { - if ( mmdb_asn && mmdb_asn->StaleDB() ) - { - report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); - did_mmdb_asn_db_error = false; - mmdb_asn.reset(); - } - } - -static bool mmdb_lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result, - bool asn) - { - struct sockaddr_storage ss = {0}; - - if ( IPv4 == addr.GetFamily() ) - { - struct sockaddr_in* sa = (struct sockaddr_in*)&ss; - sa->sin_family = AF_INET; - addr.CopyIPv4(&sa->sin_addr); - } - - else - { - struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; - sa->sin6_family = AF_INET6; - addr.CopyIPv6(&sa->sin6_addr); - } - - try - { - result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss) - : mmdb_loc->Lookup((struct sockaddr*)&ss); - } - - catch ( const std::exception& e ) - { - report_mmdb_msg("MaxMind DB lookup location error [%s]", e.what()); - return false; - } - - return result.found_entry; - } - -static bool mmdb_lookup_loc(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) - { - return mmdb_lookup(addr, result, false); - } - -static bool mmdb_lookup_asn(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) - { - return mmdb_lookup(addr, result, true); - } - -static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, - int data_type ) - { - switch (status) - { - case MMDB_SUCCESS: - if ( entry_data->has_data ) - { - switch (data_type) - { - case MMDB_DATA_TYPE_UTF8_STRING: - return zeek::make_intrusive( - entry_data->data_size, entry_data->utf8_string); - break; - - case MMDB_DATA_TYPE_DOUBLE: - return zeek::make_intrusive(entry_data->double_value); - break; - - case MMDB_DATA_TYPE_UINT32: - return zeek::val_mgr->Count(entry_data->uint32); - - default: - break; - } - } - break; - - case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: - // key doesn't exist, nothing to do - break; - - default: - report_mmdb_msg("MaxMind DB error [%s]", MMDB_strerror(status)); - break; - } - - return nullptr; - } - -static bool mmdb_try_open_loc () - { - // City database is always preferred over Country database. - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); - - const auto& mmdb_city_db_val = zeek::detail::global_scope()->Find("mmdb_city_db")->GetVal(); - std::string mmdb_city_db = mmdb_city_db_val->AsString()->CheckString(); - - const auto& mmdb_country_db_val = zeek::detail::global_scope()->Find("mmdb_country_db")->GetVal(); - std::string mmdb_country_db = mmdb_country_db_val->AsString()->CheckString(); - - if ( ! mmdb_dir.empty() ) - { - auto d = mmdb_dir + "/" + mmdb_city_db; - - if ( mmdb_open_loc(d.data()) ) - return true; - - d = mmdb_dir + "/" + mmdb_country_db; - - if ( mmdb_open_loc(d.data()) ) - return true; - } - - const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); - auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); - - for ( unsigned int i = 0; i < vv->Size(); ++i ) - { - auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_city_db; - if ( mmdb_open_loc(d.data()) ) - return true; - } - - for ( unsigned int i = 0; i < vv->Size(); ++i ) - { - auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_country_db; - if ( mmdb_open_loc(d.data()) ) - return true; - } - - return false; - } - -static bool mmdb_try_open_asn () - { - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); - - const auto& mmdb_asn_db_val = zeek::detail::global_scope()->Find("mmdb_asn_db")->GetVal(); - std::string mmdb_asn_db = mmdb_asn_db_val->AsString()->CheckString(); - - if ( ! mmdb_dir.empty() ) - { - auto d = mmdb_dir + "/" + mmdb_asn_db; - - if ( mmdb_open_asn(d.data()) ) - return true; - } - - const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); - auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); - - for ( unsigned int i = 0; i < vv->Size(); ++i ) - { - auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_asn_db; - if ( mmdb_open_asn(d.data()) ) - return true; - } - - return false; - } - -#endif +#include %%} ## Initializes MMDB for later use of lookup_location. @@ -366,11 +12,7 @@ static bool mmdb_try_open_asn () ## .. zeek:see:: lookup_autonomous_system function mmdb_open_location_db%(f: string%) : bool %{ -#ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_loc(f->CheckString())); -#else - return zeek::val_mgr->False(); -#endif + return zeek::mmdb_open_location_db(f); %} ## Initializes MMDB for later use of lookup_autonomous_system. @@ -383,11 +25,7 @@ function mmdb_open_location_db%(f: string%) : bool ## .. zeek:see:: lookup_autonomous_system function mmdb_open_asn_db%(f: string%) : bool %{ -#ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_asn(f->CheckString())); -#else - return zeek::val_mgr->False(); -#endif + return zeek::mmdb_open_asn_db(f); %} ## Performs a geo-lookup of an IP address. @@ -400,80 +38,7 @@ function mmdb_open_asn_db%(f: string%) : bool ## .. zeek:see:: lookup_autonomous_system function lookup_location%(a: addr%) : geo_location %{ - static auto geo_location = zeek::id::find_type("geo_location"); - auto location = zeek::make_intrusive(geo_location); - -#ifdef USE_GEOIP - mmdb_check_loc(); - if ( ! mmdb_loc ) - { - if ( ! mmdb_try_open_loc() ) - { - if ( ! did_mmdb_loc_db_error ) - { - did_mmdb_loc_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP location database"); - } - - return location; - } - } - - MMDB_lookup_result_s result; - - if ( mmdb_lookup_loc(a->AsAddr(), result) ) - { - MMDB_entry_data_s entry_data; - int status; - - // Get Country ISO Code - status = MMDB_get_value(&result.entry, &entry_data, - "country", "iso_code", nullptr); - location->Assign(0, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get Major Subdivision ISO Code - status = MMDB_get_value(&result.entry, &entry_data, - "subdivisions", "0", "iso_code", nullptr); - location->Assign(1, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get City English Name - status = MMDB_get_value(&result.entry, &entry_data, - "city", "names", "en", nullptr); - location->Assign(2, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - // Get Location Latitude - status = MMDB_get_value(&result.entry, &entry_data, - "location", "latitude", nullptr); - location->Assign(3, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_DOUBLE)); - - // Get Location Longitude - status = MMDB_get_value(&result.entry, &entry_data, - "location", "longitude", nullptr); - location->Assign(4, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_DOUBLE)); - - return location; - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - zeek::emit_builtin_error("Zeek was not configured for GeoIP support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have MMDB support if we weren't - // able to initialize it or it didn't return any information for - // the address. - - return std::move(location); + return zeek::mmdb_lookup_location(a); %} ## Performs an lookup of AS number & organization of an IP address. @@ -486,59 +51,5 @@ function lookup_location%(a: addr%) : geo_location ## .. zeek:see:: lookup_location function lookup_autonomous_system%(a: addr%) : geo_autonomous_system %{ - static auto geo_autonomous_system = zeek::id::find_type("geo_autonomous_system"); - auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); - -#ifdef USE_GEOIP - mmdb_check_asn(); - if ( ! mmdb_asn ) - { - if ( ! mmdb_try_open_asn() ) - { - if ( ! did_mmdb_asn_db_error ) - { - did_mmdb_asn_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP ASN database"); - } - - return autonomous_system; - } - } - - MMDB_lookup_result_s result; - - if ( mmdb_lookup_asn(a->AsAddr(), result) ) - { - MMDB_entry_data_s entry_data; - int status; - - // Get Autonomous System Number - status = MMDB_get_value(&result.entry, &entry_data, - "autonomous_system_number", nullptr); - autonomous_system->Assign(0, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UINT32)); - - // Get Autonomous System Organization - status = MMDB_get_value(&result.entry, &entry_data, - "autonomous_system_organization", nullptr); - autonomous_system->Assign(1, mmdb_getvalue(&entry_data, status, - MMDB_DATA_TYPE_UTF8_STRING)); - - return autonomous_system; - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - zeek::emit_builtin_error("Zeek was not configured for GeoIP ASN support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have GeoIP support, if we weren't - // able to initialize it or it didn't return any information for - // the address. - return std::move(autonomous_system); + return zeek::mmdb_lookup_autonomous_system(a); %} From 07499cd2e5928b5fd09564c4b400bb1f200b1524 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Tue, 9 Jan 2024 22:00:29 -0800 Subject: [PATCH 08/11] Simplify MMDB code by moving more lookup functionality into MMDB class --- src/MMDB.cc | 63 ++++++++++++++++++++++------------------------------- src/MMDB.h | 4 +++- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/MMDB.cc b/src/MMDB.cc index 42a6f04726..0e704eddbc 100644 --- a/src/MMDB.cc +++ b/src/MMDB.cc @@ -56,6 +56,30 @@ MMDB::MMDB(const char* filename, struct stat info) MMDB::~MMDB() { MMDB_close(&mmdb); } +bool MMDB::Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { + struct sockaddr_storage ss = {0}; + + if ( IPv4 == addr.GetFamily() ) { + struct sockaddr_in* sa = (struct sockaddr_in*)&ss; + sa->sin_family = AF_INET; + addr.CopyIPv4(&sa->sin_addr); + } + else { + struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; + sa->sin6_family = AF_INET6; + addr.CopyIPv6(&sa->sin6_addr); + } + + try { + result = Lookup((struct sockaddr*)&ss); + } catch ( const std::exception& e ) { + report_msg("MaxMind DB lookup location error [%s]", e.what()); + return false; + } + + return result.found_entry; +} + MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) { int mmdb_error; MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); @@ -150,41 +174,6 @@ static void mmdb_check_asn() { } } -static bool mmdb_lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result, bool asn) { - struct sockaddr_storage ss = {0}; - - if ( IPv4 == addr.GetFamily() ) { - struct sockaddr_in* sa = (struct sockaddr_in*)&ss; - sa->sin_family = AF_INET; - addr.CopyIPv4(&sa->sin_addr); - } - - else { - struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; - sa->sin6_family = AF_INET6; - addr.CopyIPv6(&sa->sin6_addr); - } - - try { - result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss) : mmdb_loc->Lookup((struct sockaddr*)&ss); - } - - catch ( const std::exception& e ) { - report_msg("MaxMind DB lookup location error [%s]", e.what()); - return false; - } - - return result.found_entry; -} - -static bool mmdb_lookup_loc(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { - return mmdb_lookup(addr, result, false); -} - -static bool mmdb_lookup_asn(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { - return mmdb_lookup(addr, result, true); -} - static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, int data_type) { switch ( status ) { case MMDB_SUCCESS: @@ -318,7 +307,7 @@ RecordValPtr mmdb_lookup_location(AddrVal* addr) { MMDB_lookup_result_s result; - if ( mmdb_lookup_loc(addr->AsAddr(), result) ) { + if ( mmdb_loc->Lookup(addr->AsAddr(), result) ) { MMDB_entry_data_s entry_data; int status; @@ -379,7 +368,7 @@ RecordValPtr mmdb_lookup_autonomous_system(AddrVal* addr) { MMDB_lookup_result_s result; - if ( mmdb_lookup_asn(addr->AsAddr(), result) ) { + if ( mmdb_asn->Lookup(addr->AsAddr(), result) ) { MMDB_entry_data_s entry_data; int status; diff --git a/src/MMDB.h b/src/MMDB.h index 23732b3e2b..7f20fbffcc 100644 --- a/src/MMDB.h +++ b/src/MMDB.h @@ -16,11 +16,13 @@ public: ~MMDB(); - MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); + bool Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result); bool StaleDB(); const char* Filename(); private: + MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); + MMDB_s mmdb; struct stat file_info; bool lookup_error; From 2e3270d7ecdc20b6ae9cad75c69b8a2ae8bf96d5 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 10 Jan 2024 12:19:50 -0800 Subject: [PATCH 09/11] Add btest to verify behavior of re-opened MMDBs opened directly via BIFs The mmdb_open_location_db() and mmdb_open_asn_db() BiFs were untested, and Zeek has a bug that makes any DBs opened that way fall back to looking up DBs via the existing script-level config mechanism (via mmdb_dir), which is at least unexpected and might well be unconfigured if somebody uses the direct BiFs. --- .../Baseline/core.mmdb.explicit-open/out | 17 +++++++ .../core.mmdb.explicit-open/reporter.log | 11 ++++ testing/btest/core/mmdb/explicit-open.zeek | 50 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 testing/btest/Baseline/core.mmdb.explicit-open/out create mode 100644 testing/btest/Baseline/core.mmdb.explicit-open/reporter.log create mode 100644 testing/btest/core/mmdb/explicit-open.zeek diff --git a/testing/btest/Baseline/core.mmdb.explicit-open/out b/testing/btest/Baseline/core.mmdb.explicit-open/out new file mode 100644 index 0000000000..5641085ef3 --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.explicit-open/out @@ -0,0 +1,17 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299466805.0, 1, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299466805.0, 1, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470395.0, 2, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470395.0, 2, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470395.0, 2, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470395.0, 2, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470405.0, 3, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470405.0, 3, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299473995.0, 4, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299473995.0, 4, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299473995.0, 4, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299473995.0, 4, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] diff --git a/testing/btest/Baseline/core.mmdb.explicit-open/reporter.log b/testing/btest/Baseline/core.mmdb.explicit-open/reporter.log new file mode 100644 index 0000000000..91b9edb7f8 --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.explicit-open/reporter.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts level message location +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO received termination signal (empty) diff --git a/testing/btest/core/mmdb/explicit-open.zeek b/testing/btest/core/mmdb/explicit-open.zeek new file mode 100644 index 0000000000..9b8c6ddebb --- /dev/null +++ b/testing/btest/core/mmdb/explicit-open.zeek @@ -0,0 +1,50 @@ +# @TEST-DOC: verifies that the explicit BiFs for loading MMDBs work, including when re-opening. +# +# Like other MMDB tests, this uses a pcap to use each packet as a driver to +# touch the DBs involved upon each packet, triggering DB reloads. +# +# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h +# +# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb +# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log + +@load base/frameworks/reporter + +global pkt = 0; +global asn_fn = "./mmdb/GeoLite2-ASN.mmdb"; +global city_fn = "./mmdb/GeoLite2-City.mmdb"; + +function timestamp(n: count): string + { + assert n <= 60; + return fmt("2020-01-01T00:%s:00", n); + } + +event new_packet(c: connection, p: pkt_hdr) + { + ++pkt; + + print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1); + print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1); + print network_time(), pkt, 131.243.0.1, "asn", lookup_autonomous_system(131.243.0.1); + print network_time(), pkt, 131.243.0.1, "location", lookup_location(131.243.0.1); + + # Increment MMDBs' modification time, triggering a re-open. + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(asn_fn)), "") ) + exit(1); + + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(city_fn)), "") ) + exit(1); + + if ( pkt == 4 ) + terminate(); + } + +event zeek_init() + { + assert mmdb_open_asn_db(asn_fn); + assert mmdb_open_location_db(city_fn); + } From e8f0f727cde774f15bcd5992e004adffe3e0f2e9 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 10 Jan 2024 12:22:24 -0800 Subject: [PATCH 10/11] Fix MMDB code to re-open explicitly opened DBs correctly The filename from which a DB first gets opened (either via an explicitly specified filename, or via the path sequence now configurable at the script layer) is now "sticky", meaning re-opening won't switch to a different file. This was easiest by moving most state into the MMDB class itself. The previous approach of tracking the two DB instances via a smart pointer and blowing the pointed-to objects away as needed is now instead one of two objects fixed over the lifetime of Zeek, able to open/close/reopen their underlying Maxmind DBs. The MMDB class now only has one Lookup() method since there was no need to break them apart -- it saves the return of a MMDB_lookup_result_s over the stack and there's no need for throwing an exception. --- src/MMDB.cc | 294 ++++++++++++++++++++++------------------------------ src/MMDB.h | 63 +++++++++-- 2 files changed, 183 insertions(+), 174 deletions(-) diff --git a/src/MMDB.cc b/src/MMDB.cc index 0e704eddbc..71abf77135 100644 --- a/src/MMDB.cc +++ b/src/MMDB.cc @@ -18,13 +18,11 @@ namespace zeek { static int msg_count = 0; static double msg_suppression_time = 0; -static bool did_loc_db_error = false; -static bool did_asn_db_error = false; static constexpr int msg_limit = 20; static constexpr double msg_suppression_duration = 300; -static std::unique_ptr mmdb_loc; -static std::unique_ptr mmdb_asn; +LocDB mmdb_loc; +AsnDB mmdb_asn; static void report_msg(const char* format, ...) { if ( zeek::run_state::network_time > msg_suppression_time + msg_suppression_duration ) { @@ -45,135 +43,6 @@ static void report_msg(const char* format, ...) { zeek::reporter->Info("%s", msg.data()); } -MMDB::MMDB(const char* filename, struct stat info) - : file_info(info), lookup_error{false}, last_check{zeek::run_state::network_time} { - int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); - - if ( MMDB_SUCCESS != status ) { - throw std::runtime_error(MMDB_strerror(status)); - } -} - -MMDB::~MMDB() { MMDB_close(&mmdb); } - -bool MMDB::Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { - struct sockaddr_storage ss = {0}; - - if ( IPv4 == addr.GetFamily() ) { - struct sockaddr_in* sa = (struct sockaddr_in*)&ss; - sa->sin_family = AF_INET; - addr.CopyIPv4(&sa->sin_addr); - } - else { - struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; - sa->sin6_family = AF_INET6; - addr.CopyIPv6(&sa->sin6_addr); - } - - try { - result = Lookup((struct sockaddr*)&ss); - } catch ( const std::exception& e ) { - report_msg("MaxMind DB lookup location error [%s]", e.what()); - return false; - } - - return result.found_entry; -} - -MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) { - int mmdb_error; - MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); - - if ( MMDB_SUCCESS != mmdb_error ) { - lookup_error = true; - throw std::runtime_error(MMDB_strerror(mmdb_error)); - } - - return result; -} - -// Check to see if the Maxmind DB should be closed and reopened. This will -// happen if there was a lookup error or if the mmap'd file has been replaced -// by an external process. -bool MMDB::StaleDB() { - struct stat buf; - - if ( lookup_error ) - return true; - - static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); - - if ( mmdb_stale_check_interval < 0.0 ) - return false; - - if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) - return false; - - last_check = zeek::run_state::network_time; - - if ( 0 != stat(mmdb.filename, &buf) ) - return true; - - if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) { - report_msg("%s change detected for MaxMind DB [%s]", - buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", mmdb.filename); - return true; - } - - return false; -} - -const char* MMDB::Filename() { return mmdb.filename; } - -static bool mmdb_open(const char* filename, bool asn) { - struct stat buf; - - if ( 0 != stat(filename, &buf) ) { - return false; - } - - try { - if ( asn ) { - mmdb_asn.reset(new MMDB(filename, buf)); - } - else { - mmdb_loc.reset(new MMDB(filename, buf)); - } - } - - catch ( const std::exception& e ) { - if ( asn ) - did_asn_db_error = false; - else - did_loc_db_error = false; - - report_msg("Failed to open MaxMind DB: %s [%s]", filename, e.what()); - return false; - } - - return true; -} - -static bool mmdb_open_loc(const char* filename) { return mmdb_open(filename, false); } - -static bool mmdb_open_asn(const char* filename) { return mmdb_open(filename, true); } - -static void mmdb_check_loc() { - if ( mmdb_loc && mmdb_loc->StaleDB() ) { - report_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); - did_loc_db_error = false; - mmdb_loc.reset(); - } -} - -static void mmdb_check_asn() { - if ( mmdb_asn && mmdb_asn->StaleDB() ) { - report_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); - did_asn_db_error = false; - mmdb_asn.reset(); - } -} - static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, int data_type) { switch ( status ) { case MMDB_SUCCESS: @@ -204,7 +73,115 @@ static zeek::ValPtr mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, int return nullptr; } -static bool mmdb_try_open_loc() { +MMDB::MMDB() : mmdb{}, file_info{}, reported_error{false}, last_check{zeek::run_state::network_time} {} + +MMDB::~MMDB() { Close(); } + +bool MMDB::OpenFile(const char* a_filename) { + filename = a_filename; + Close(); + + if ( 0 != stat(a_filename, &file_info) ) { + return false; + } + + int status = MMDB_open(a_filename, MMDB_MODE_MMAP, &mmdb); + + if ( MMDB_SUCCESS != status ) { + memset(&mmdb, 0, sizeof(mmdb)); + report_msg("Failed to open MaxMind DB: %s [%s]", filename.data(), MMDB_strerror(status)); + return false; + } + + return true; +} + +void MMDB::Close() { + if ( IsOpen() ) { + MMDB_close(&mmdb); + memset(&mmdb, 0, sizeof(mmdb)); + reported_error = false; + } +} + +bool MMDB::EnsureLoaded() { + bool res = true; + + if ( filename.empty() ) + res = OpenFromScriptConfig(); + else if ( ! IsOpen() ) + res = OpenFile(filename.data()); + else if ( IsStaleDB() ) { + report_msg("Closing stale MaxMind DB [%s]", filename.data()); + if ( ! OpenFile(filename.data()) ) + res = false; + } + + if ( ! res && ! reported_error ) { + reported_error = true; + zeek::emit_builtin_error(zeek::util::fmt("Failed to open %s", Description())); + } + + return res; +} + +bool MMDB::Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result) { + struct sockaddr_storage ss = {0}; + + if ( IPv4 == addr.GetFamily() ) { + struct sockaddr_in* sa = (struct sockaddr_in*)&ss; + sa->sin_family = AF_INET; + addr.CopyIPv4(&sa->sin_addr); + } + else { + struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; + sa->sin6_family = AF_INET6; + addr.CopyIPv6(&sa->sin6_addr); + } + + int mmdb_error; + result = MMDB_lookup_sockaddr(&mmdb, (struct sockaddr*)&ss, &mmdb_error); + + if ( MMDB_SUCCESS != mmdb_error ) { + report_msg("MaxMind DB lookup location error [%s]", MMDB_strerror(mmdb_error)); + Close(); + return false; + } + + return true; +} + +// Check to see if the Maxmind DB should be closed and reopened. This will +// happen if there was a lookup error or if the mmap'd file has been replaced +// by an external process. +bool MMDB::IsStaleDB() { + if ( ! IsOpen() ) + return false; + + static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); + + if ( mmdb_stale_check_interval < 0.0 ) + return false; + + if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) + return false; + + last_check = zeek::run_state::network_time; + struct stat buf; + + if ( 0 != stat(mmdb.filename, &buf) ) + return true; + + if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) { + report_msg("%s change detected for MaxMind DB [%s]", + buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", mmdb.filename); + return true; + } + + return false; +} + +bool LocDB::OpenFromScriptConfig() { // City database is always preferred over Country database. const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); @@ -218,12 +195,12 @@ static bool mmdb_try_open_loc() { if ( ! mmdb_dir.empty() ) { auto d = mmdb_dir + "/" + mmdb_city_db; - if ( mmdb_open_loc(d.data()) ) + if ( OpenFile(d.data()) ) return true; d = mmdb_dir + "/" + mmdb_country_db; - if ( mmdb_open_loc(d.data()) ) + if ( OpenFile(d.data()) ) return true; } @@ -232,20 +209,19 @@ static bool mmdb_try_open_loc() { for ( unsigned int i = 0; i < vv->Size(); ++i ) { auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_city_db; - if ( mmdb_open_loc(d.data()) ) + if ( OpenFile(d.data()) ) return true; } for ( unsigned int i = 0; i < vv->Size(); ++i ) { auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_country_db; - if ( mmdb_open_loc(d.data()) ) + if ( OpenFile(d.data()) ) return true; } return false; } - -static bool mmdb_try_open_asn() { +bool AsnDB::OpenFromScriptConfig() { const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); @@ -255,7 +231,7 @@ static bool mmdb_try_open_asn() { if ( ! mmdb_dir.empty() ) { auto d = mmdb_dir + "/" + mmdb_asn_db; - if ( mmdb_open_asn(d.data()) ) + if ( OpenFile(d.data()) ) return true; } @@ -264,7 +240,7 @@ static bool mmdb_try_open_asn() { for ( unsigned int i = 0; i < vv->Size(); ++i ) { auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_asn_db; - if ( mmdb_open_loc(d.data()) ) + if ( OpenFile(d.data()) ) return true; } @@ -274,7 +250,7 @@ static bool mmdb_try_open_asn() { ValPtr mmdb_open_location_db(StringVal* filename) { #ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_loc(filename->CheckString())); + return zeek::val_mgr->Bool(mmdb_loc.OpenFile(filename->CheckString())); #else return zeek::val_mgr->False(); #endif @@ -282,7 +258,7 @@ ValPtr mmdb_open_location_db(StringVal* filename) { ValPtr mmdb_open_asn_db(StringVal* filename) { #ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_open_asn(filename->CheckString())); + return zeek::val_mgr->Bool(mmdb_asn.OpenFile(filename->CheckString())); #else return zeek::val_mgr->False(); #endif @@ -293,21 +269,12 @@ RecordValPtr mmdb_lookup_location(AddrVal* addr) { auto location = zeek::make_intrusive(geo_location); #ifdef USE_GEOIP - mmdb_check_loc(); - if ( ! mmdb_loc ) { - if ( ! mmdb_try_open_loc() ) { - if ( ! did_loc_db_error ) { - did_loc_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP location database"); - } - - return location; - } - } + if ( ! mmdb_loc.EnsureLoaded() ) + return location; MMDB_lookup_result_s result; - if ( mmdb_loc->Lookup(addr->AsAddr(), result) ) { + if ( mmdb_loc.Lookup(addr->AsAddr(), result) ) { MMDB_entry_data_s entry_data; int status; @@ -354,21 +321,12 @@ RecordValPtr mmdb_lookup_autonomous_system(AddrVal* addr) { auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); #ifdef USE_GEOIP - mmdb_check_asn(); - if ( ! mmdb_asn ) { - if ( ! mmdb_try_open_asn() ) { - if ( ! did_asn_db_error ) { - did_asn_db_error = true; - zeek::emit_builtin_error("Failed to open GeoIP ASN database"); - } - - return autonomous_system; - } - } + if ( ! mmdb_asn.EnsureLoaded() ) + return autonomous_system; MMDB_lookup_result_s result; - if ( mmdb_asn->Lookup(addr->AsAddr(), result) ) { + if ( mmdb_asn.Lookup(addr->AsAddr(), result) ) { MMDB_entry_data_s entry_data; int status; diff --git a/src/MMDB.h b/src/MMDB.h index 7f20fbffcc..cbf63057eb 100644 --- a/src/MMDB.h +++ b/src/MMDB.h @@ -10,25 +10,76 @@ namespace zeek { #include +// The MMDB class encapsulates a low-level libmaxmind MMDB_s structure. It +// tracks whether that DB is currently loaded, and can open it from a file in +// two ways: (1) via explicit specification of a filename, (2) by determining +// the configuration from configuration settings in the script layer (mmdb_dir +// etc). This configuration depends on whether this is a geolocation DB or an +// ASN one, so details are left to derived classes below that specialize. +// +// The class tracks the inode and modification time of a DB file to detect +// "stale" DBs, which get reloaded (from the same location in the file system) +// upon the first lookup that detects staleness. class MMDB { public: - MMDB(const char* filename, struct stat info); + MMDB(); + virtual ~MMDB(); - ~MMDB(); + // Implements the logic to determine a file system path for the DB from + // script-layer configuration settings, and opens the DB from there. Returns + // true if successful, false otherwise. + virtual bool OpenFromScriptConfig() = 0; + // Helper string to identify the type of DB, useful in error messages. + virtual const char* Description() = 0; + + // Opens the DB at the given location, closing and cleaning up any currently + // opened DB if there is one. Returns true if successful, false otherwise. + bool OpenFile(const char* filename); + + // Closes a currently opened DB, releasing its state. Safe to call on a + // closed DB. + void Close(); + + // Predicate; returns true if the DB is currently opened. + bool IsOpen() const { return mmdb.filename != nullptr; } + + // Ensures that upon return the underlying DB file is loaded. When no + // filename is configured for the DB (i.e. OpenFile() has never been called + // on it), this triggers the script-level configuration lookup via + // OpenFromScriptConfig(). When a filename is available but it's not + // currently loaded, it does so. Finally, if there's a loaded DB but it's + // found to be stale, it gets reloaded. When the load operation succeeds, or + // the DB was already loaded and not stale, this returns true, and false if + // anything went wrong. + bool EnsureLoaded(); + + // Looks up a given IP address in the DB, storing the result in the provided + // result structure. bool Lookup(const zeek::IPAddr& addr, MMDB_lookup_result_s& result); - bool StaleDB(); - const char* Filename(); private: - MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); + bool IsStaleDB(); + std::string filename; MMDB_s mmdb; struct stat file_info; - bool lookup_error; + bool reported_error; // to ensure we emit builtin errors during opening only once. double last_check; }; +class LocDB : public MMDB { +public: + bool OpenFromScriptConfig(); + const char* Description() { return "GeoIP location database"; } +}; + +class AsnDB : public MMDB { +public: + bool OpenFromScriptConfig(); + const char* Description() { return "GeoIP ASN database"; } +}; + #endif // USE_GEOIP ValPtr mmdb_open_location_db(zeek::StringVal* filename); From dbad072f0638b524b9b1d16b6e93944dec8941c4 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 11 Jan 2024 17:13:41 -0800 Subject: [PATCH 11/11] Modernize various C++/Zeek-isms in the MMDB code. --- src/MMDB.cc | 74 +++++++++++++++++++++++++--------------------------- src/MMDB.h | 16 ++++++------ src/mmdb.bif | 8 +++--- 3 files changed, 48 insertions(+), 50 deletions(-) diff --git a/src/MMDB.cc b/src/MMDB.cc index 71abf77135..45b6b0052b 100644 --- a/src/MMDB.cc +++ b/src/MMDB.cc @@ -77,15 +77,15 @@ MMDB::MMDB() : mmdb{}, file_info{}, reported_error{false}, last_check{zeek::run_ MMDB::~MMDB() { Close(); } -bool MMDB::OpenFile(const char* a_filename) { +bool MMDB::OpenFile(const std::string& a_filename) { filename = a_filename; Close(); - if ( 0 != stat(a_filename, &file_info) ) { + if ( 0 != stat(filename.data(), &file_info) ) { return false; } - int status = MMDB_open(a_filename, MMDB_MODE_MMAP, &mmdb); + int status = MMDB_open(a_filename.data(), MMDB_MODE_MMAP, &mmdb); if ( MMDB_SUCCESS != status ) { memset(&mmdb, 0, sizeof(mmdb)); @@ -110,16 +110,16 @@ bool MMDB::EnsureLoaded() { if ( filename.empty() ) res = OpenFromScriptConfig(); else if ( ! IsOpen() ) - res = OpenFile(filename.data()); + res = OpenFile(filename); else if ( IsStaleDB() ) { report_msg("Closing stale MaxMind DB [%s]", filename.data()); - if ( ! OpenFile(filename.data()) ) + if ( ! OpenFile(filename) ) res = false; } if ( ! res && ! reported_error ) { reported_error = true; - zeek::emit_builtin_error(zeek::util::fmt("Failed to open %s", Description())); + zeek::emit_builtin_error(zeek::util::fmt("Failed to open %s", Description().data())); } return res; @@ -183,64 +183,62 @@ bool MMDB::IsStaleDB() { bool LocDB::OpenFromScriptConfig() { // City database is always preferred over Country database. - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + const auto& mmdb_dir_val = zeek::id::find_val("mmdb_dir"); + std::string mmdb_dir{mmdb_dir_val->ToStdStringView()}; - const auto& mmdb_city_db_val = zeek::detail::global_scope()->Find("mmdb_city_db")->GetVal(); - std::string mmdb_city_db = mmdb_city_db_val->AsString()->CheckString(); + const auto& mmdb_city_db_val = zeek::id::find_val("mmdb_city_db"); + std::string mmdb_city_db{mmdb_city_db_val->ToStdStringView()}; - const auto& mmdb_country_db_val = zeek::detail::global_scope()->Find("mmdb_country_db")->GetVal(); - std::string mmdb_country_db = mmdb_country_db_val->AsString()->CheckString(); + const auto& mmdb_country_db_val = zeek::id::find_val("mmdb_country_db"); + std::string mmdb_country_db{mmdb_country_db_val->ToStdStringView()}; if ( ! mmdb_dir.empty() ) { auto d = mmdb_dir + "/" + mmdb_city_db; - if ( OpenFile(d.data()) ) + if ( OpenFile(d) ) return true; d = mmdb_dir + "/" + mmdb_country_db; - if ( OpenFile(d.data()) ) + if ( OpenFile(d) ) return true; } - const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); - auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); + const auto& mmdb_dir_fallbacks_val = zeek::id::find_val("mmdb_dir_fallbacks"); - for ( unsigned int i = 0; i < vv->Size(); ++i ) { - auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_city_db; - if ( OpenFile(d.data()) ) + for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) { + auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_city_db; + if ( OpenFile(d) ) return true; } - for ( unsigned int i = 0; i < vv->Size(); ++i ) { - auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_country_db; - if ( OpenFile(d.data()) ) + for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) { + auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_country_db; + if ( OpenFile(d) ) return true; } return false; } bool AsnDB::OpenFromScriptConfig() { - const auto& mmdb_dir_val = zeek::detail::global_scope()->Find("mmdb_dir")->GetVal(); - std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + const auto& mmdb_dir_val = zeek::id::find_val("mmdb_dir"); + std::string mmdb_dir{mmdb_dir_val->ToStdStringView()}; - const auto& mmdb_asn_db_val = zeek::detail::global_scope()->Find("mmdb_asn_db")->GetVal(); - std::string mmdb_asn_db = mmdb_asn_db_val->AsString()->CheckString(); + const auto& mmdb_asn_db_val = zeek::id::find_val("mmdb_asn_db"); + std::string mmdb_asn_db{mmdb_asn_db_val->ToStdStringView()}; if ( ! mmdb_dir.empty() ) { auto d = mmdb_dir + "/" + mmdb_asn_db; - if ( OpenFile(d.data()) ) + if ( OpenFile(d) ) return true; } - const auto& mmdb_dir_fallbacks_val = zeek::detail::global_scope()->Find("mmdb_dir_fallbacks")->GetVal(); - auto* vv = mmdb_dir_fallbacks_val->AsVectorVal(); + const auto& mmdb_dir_fallbacks_val = zeek::id::find_val("mmdb_dir_fallbacks"); - for ( unsigned int i = 0; i < vv->Size(); ++i ) { - auto d = std::string(vv->StringAt(i)->CheckString()) + "/" + mmdb_asn_db; - if ( OpenFile(d.data()) ) + for ( unsigned int i = 0; i < mmdb_dir_fallbacks_val->Size(); ++i ) { + auto d = mmdb_dir_fallbacks_val->StringValAt(i)->ToStdString() + "/" + mmdb_asn_db; + if ( OpenFile(d) ) return true; } @@ -248,23 +246,23 @@ bool AsnDB::OpenFromScriptConfig() { } #endif // USE_GEOIP -ValPtr mmdb_open_location_db(StringVal* filename) { +ValPtr mmdb_open_location_db(const StringValPtr& filename) { #ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_loc.OpenFile(filename->CheckString())); + return zeek::val_mgr->Bool(mmdb_loc.OpenFile(filename->ToStdString())); #else return zeek::val_mgr->False(); #endif } -ValPtr mmdb_open_asn_db(StringVal* filename) { +ValPtr mmdb_open_asn_db(const StringValPtr& filename) { #ifdef USE_GEOIP - return zeek::val_mgr->Bool(mmdb_asn.OpenFile(filename->CheckString())); + return zeek::val_mgr->Bool(mmdb_asn.OpenFile(filename->ToStdString())); #else return zeek::val_mgr->False(); #endif } -RecordValPtr mmdb_lookup_location(AddrVal* addr) { +RecordValPtr mmdb_lookup_location(const AddrValPtr& addr) { static auto geo_location = zeek::id::find_type("geo_location"); auto location = zeek::make_intrusive(geo_location); @@ -316,7 +314,7 @@ RecordValPtr mmdb_lookup_location(AddrVal* addr) { return location; } -RecordValPtr mmdb_lookup_autonomous_system(AddrVal* addr) { +RecordValPtr mmdb_lookup_autonomous_system(const AddrValPtr& addr) { static auto geo_autonomous_system = zeek::id::find_type("geo_autonomous_system"); auto autonomous_system = zeek::make_intrusive(geo_autonomous_system); diff --git a/src/MMDB.h b/src/MMDB.h index cbf63057eb..0d3af59638 100644 --- a/src/MMDB.h +++ b/src/MMDB.h @@ -31,11 +31,11 @@ public: virtual bool OpenFromScriptConfig() = 0; // Helper string to identify the type of DB, useful in error messages. - virtual const char* Description() = 0; + virtual std::string_view Description() = 0; // Opens the DB at the given location, closing and cleaning up any currently // opened DB if there is one. Returns true if successful, false otherwise. - bool OpenFile(const char* filename); + bool OpenFile(const std::string& filename); // Closes a currently opened DB, releasing its state. Safe to call on a // closed DB. @@ -71,21 +71,21 @@ private: class LocDB : public MMDB { public: bool OpenFromScriptConfig(); - const char* Description() { return "GeoIP location database"; } + std::string_view Description() { return "GeoIP location database"; } }; class AsnDB : public MMDB { public: bool OpenFromScriptConfig(); - const char* Description() { return "GeoIP ASN database"; } + std::string_view Description() { return "GeoIP ASN database"; } }; #endif // USE_GEOIP -ValPtr mmdb_open_location_db(zeek::StringVal* filename); -ValPtr mmdb_open_asn_db(zeek::StringVal* filename); +ValPtr mmdb_open_location_db(const StringValPtr& filename); +ValPtr mmdb_open_asn_db(const StringValPtr& filename); -RecordValPtr mmdb_lookup_location(zeek::AddrVal* addr); -RecordValPtr mmdb_lookup_autonomous_system(zeek::AddrVal* addr); +RecordValPtr mmdb_lookup_location(const AddrValPtr& addr); +RecordValPtr mmdb_lookup_autonomous_system(const AddrValPtr& addr); } // namespace zeek diff --git a/src/mmdb.bif b/src/mmdb.bif index 8aa3100aa4..76ff974e07 100644 --- a/src/mmdb.bif +++ b/src/mmdb.bif @@ -12,7 +12,7 @@ ## .. zeek:see:: lookup_autonomous_system function mmdb_open_location_db%(f: string%) : bool %{ - return zeek::mmdb_open_location_db(f); + return zeek::mmdb_open_location_db(StringValPtr(NewRef(), f)); %} ## Initializes MMDB for later use of lookup_autonomous_system. @@ -25,7 +25,7 @@ function mmdb_open_location_db%(f: string%) : bool ## .. zeek:see:: lookup_autonomous_system function mmdb_open_asn_db%(f: string%) : bool %{ - return zeek::mmdb_open_asn_db(f); + return zeek::mmdb_open_asn_db(StringValPtr(NewRef(), f)); %} ## Performs a geo-lookup of an IP address. @@ -38,7 +38,7 @@ function mmdb_open_asn_db%(f: string%) : bool ## .. zeek:see:: lookup_autonomous_system function lookup_location%(a: addr%) : geo_location %{ - return zeek::mmdb_lookup_location(a); + return zeek::mmdb_lookup_location(AddrValPtr(NewRef(), a)); %} ## Performs an lookup of AS number & organization of an IP address. @@ -51,5 +51,5 @@ function lookup_location%(a: addr%) : geo_location ## .. zeek:see:: lookup_location function lookup_autonomous_system%(a: addr%) : geo_autonomous_system %{ - return zeek::mmdb_lookup_autonomous_system(a); + return zeek::mmdb_lookup_autonomous_system(AddrValPtr(NewRef(), a)); %}