diff --git a/scripts/base/frameworks/spicy/init-framework.zeek b/scripts/base/frameworks/spicy/init-framework.zeek index 886b0f4170..ae3a3b8e65 100644 --- a/scripts/base/frameworks/spicy/init-framework.zeek +++ b/scripts/base/frameworks/spicy/init-framework.zeek @@ -1,5 +1,3 @@ -@load base/misc/version - # doc-common-start module Spicy; diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index cbc6c74da4..ba909f2191 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -1080,6 +1080,11 @@ type geo_autonomous_system: record { ## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support. const mmdb_dir: string = "" &redef; +## Sets the interval for MaxMind DB file staleness checks. When Zeek detects a +## change in inode or modification time, the database is re-opened. Setting +## a negative interval disables staleness checks. +const mmdb_stale_check_interval: interval = 5min &redef; + ## Computed entropy values. The record captures a number of measures that are ## computed in parallel. See `A Pseudorandom Number Sequence Test Program ## `_ for more information, Zeek uses the same diff --git a/src/zeek.bif b/src/zeek.bif index c97e6e7d9c..bf849a8ec9 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -4004,7 +4004,7 @@ static void report_mmdb_msg(const char* format, ...) va_list al; va_start(al, format); - std::string msg = zeek::util::fmt(format, al); + std::string msg = zeek::util::vfmt(format, al); va_end(al); zeek::reporter->Info("%s", msg.data()); @@ -4024,12 +4024,12 @@ private: MMDB_s mmdb; struct stat file_info; bool lookup_error; - std::chrono::time_point last_check; + double last_check; }; MMDB::MMDB(const char* filename, struct stat info) : file_info(info), lookup_error{false}, - last_check{std::chrono::steady_clock::now()} + last_check{zeek::run_state::network_time} { int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); @@ -4064,26 +4064,29 @@ MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) bool MMDB::StaleDB() { struct stat buf; - using Clock = std::chrono::steady_clock; - std::chrono::time_point now = Clock::now(); if ( lookup_error ) return true; - // Only perform stat once per 5 minutes. - using Min = std::chrono::minutes; - if ( std::chrono::duration_cast(now - last_check).count() < 5 ) + + static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval(); + + if ( mmdb_stale_check_interval < 0.0 ) return false; - last_check = now; + if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval ) + return false; + + last_check = zeek::run_state::network_time; if ( 0 != stat(mmdb.filename, &buf) ) return true; if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) { - report_mmdb_msg("Inode change detected for MaxMind DB [%s]", - mmdb.filename); + report_mmdb_msg("%s change detected for MaxMind DB [%s]", + buf.st_ino != file_info.st_ino ? "Inode" : "Modification time", + mmdb.filename); return true; } @@ -4152,7 +4155,7 @@ static void mmdb_check_loc() { report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); did_mmdb_loc_db_error = false; - mmdb_loc.release(); + mmdb_loc.reset(); } } @@ -4162,7 +4165,7 @@ static void mmdb_check_asn() { report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); did_mmdb_asn_db_error = false; - mmdb_asn.release(); + mmdb_asn.reset(); } } diff --git a/testing/btest/Baseline/broker.remote_event/recv.recv.out b/testing/btest/Baseline/broker.remote_event/recv.recv.out index dacc55938b..c13ab4ea41 100644 --- a/testing/btest/Baseline/broker.remote_event/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event/recv.recv.out @@ -10,4 +10,4 @@ is_remote should be T, and is, T receiver got ping: my-message, 4 is_remote should be T, and is, T receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/broker.remote_event_any/recv.recv.out b/testing/btest/Baseline/broker.remote_event_any/recv.recv.out index dacc55938b..c13ab4ea41 100644 --- a/testing/btest/Baseline/broker.remote_event_any/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event_any/recv.recv.out @@ -10,4 +10,4 @@ is_remote should be T, and is, T receiver got ping: my-message, 4 is_remote should be T, and is, T receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out b/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out index b25425641f..7a0dc495f2 100644 --- a/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out @@ -5,4 +5,4 @@ receiver got ping: my-message, 2 receiver got ping: my-message, 3 receiver got ping: my-message, 4 receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/core.mmdb.reopen/out b/testing/btest/Baseline/core.mmdb.reopen/out new file mode 100644 index 0000000000..5641085ef3 --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.reopen/out @@ -0,0 +1,17 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299466805.0, 1, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299466805.0, 1, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470395.0, 2, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470395.0, 2, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470395.0, 2, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470395.0, 2, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299470405.0, 3, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470405.0, 3, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299473995.0, 4, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299473995.0, 4, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +1299473995.0, 4, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299473995.0, 4, 131.243.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] diff --git a/testing/btest/Baseline/core.mmdb.reopen/reporter.log b/testing/btest/Baseline/core.mmdb.reopen/reporter.log new file mode 100644 index 0000000000..91b9edb7f8 --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.reopen/reporter.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts level message location +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty) +1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty) +1299473995.000000 Reporter::INFO received termination signal (empty) diff --git a/testing/btest/Baseline/core.mmdb.temporary-error/out b/testing/btest/Baseline/core.mmdb.temporary-error/out new file mode 100644 index 0000000000..7359472613 --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.temporary-error/out @@ -0,0 +1,20 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +start +1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +corrupting db +1299470395.0, 2, 128.3.0.1, asn, [number=, organization=] +1299470395.0, 2, 128.3.0.1, location, [country_code=, region=, city=, latitude=, longitude=] +restoring backup db +1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +unlinking +1299473995.0, 4, 128.3.0.1, asn, [number=, organization=] +1299473995.0, 4, 128.3.0.1, location, [country_code=, region=, city=, latitude=, longitude=] +restoring backup db +1299474005.0, 5, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299474005.0, 5, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +unlinking and restoring +1299477595.0, 6, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory] +1299477595.0, 6, 128.3.0.1, location, [country_code=US, region=, city=Berkeley, latitude=37.751, longitude=-97.822] +done diff --git a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log new file mode 100644 index 0000000000..5ec6a3e3ec --- /dev/null +++ b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log @@ -0,0 +1,19 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts level message location +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 +1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] , line 1 +1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83 +1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 +1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 +1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] , line 1 +1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84 +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 +1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83 +1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 +1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84 +1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 +1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 +1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 +1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 +1299477605.000000 Reporter::INFO received termination signal , line 1 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 5540e6d45b..531a6aa1d9 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -266,10 +266,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek scripts/base/frameworks/spicy/init-framework.zeek - scripts/base/misc/version.zeek - scripts/base/frameworks/reporter/__load__.zeek - scripts/base/frameworks/reporter/main.zeek - scripts/base/utils/strings.zeek build/scripts/builtin-plugins/__load__.zeek scripts/policy/misc/loaded-scripts.zeek scripts/base/utils/paths.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 2c597084a7..8ac7c7dcc4 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -266,10 +266,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek scripts/base/frameworks/spicy/init-framework.zeek - scripts/base/misc/version.zeek - scripts/base/frameworks/reporter/__load__.zeek - scripts/base/frameworks/reporter/main.zeek - scripts/base/utils/strings.zeek scripts/base/init-default.zeek scripts/base/utils/active-http.zeek scripts/base/utils/exec.zeek @@ -277,6 +273,8 @@ scripts/base/init-default.zeek scripts/base/utils/backtrace.zeek scripts/base/utils/conn-ids.zeek scripts/base/utils/dir.zeek + scripts/base/frameworks/reporter/__load__.zeek + scripts/base/frameworks/reporter/main.zeek scripts/base/utils/paths.zeek scripts/base/utils/directions-and-hosts.zeek scripts/base/utils/email.zeek @@ -284,6 +282,7 @@ scripts/base/init-default.zeek scripts/base/utils/geoip-distance.zeek scripts/base/utils/numbers.zeek scripts/base/utils/queue.zeek + scripts/base/utils/strings.zeek scripts/base/utils/thresholds.zeek scripts/base/utils/time.zeek scripts/base/utils/urls.zeek @@ -347,6 +346,7 @@ scripts/base/init-default.zeek scripts/base/frameworks/netcontrol/non-cluster.zeek scripts/base/frameworks/telemetry/__load__.zeek scripts/base/frameworks/telemetry/main.zeek + scripts/base/misc/version.zeek scripts/base/frameworks/spicy/__load__.zeek scripts/base/frameworks/spicy/main.zeek scripts/base/protocols/conn/__load__.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr index 9422ad94ac..e97cb1bad7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr @@ -1,3 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension -XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension diff --git a/testing/btest/Files/mmdb/.gitignore b/testing/btest/Files/mmdb/.gitignore new file mode 100644 index 0000000000..b4daca1937 --- /dev/null +++ b/testing/btest/Files/mmdb/.gitignore @@ -0,0 +1 @@ +testmmdb diff --git a/testing/btest/Files/mmdb/GeoLite2-ASN.mmdb b/testing/btest/Files/mmdb/GeoLite2-ASN.mmdb new file mode 100644 index 0000000000..65ade5917e Binary files /dev/null and b/testing/btest/Files/mmdb/GeoLite2-ASN.mmdb differ diff --git a/testing/btest/Files/mmdb/GeoLite2-City.mmdb b/testing/btest/Files/mmdb/GeoLite2-City.mmdb new file mode 100644 index 0000000000..2f375e3bd6 Binary files /dev/null and b/testing/btest/Files/mmdb/GeoLite2-City.mmdb differ diff --git a/testing/btest/Files/mmdb/README b/testing/btest/Files/mmdb/README new file mode 100644 index 0000000000..269a36c045 --- /dev/null +++ b/testing/btest/Files/mmdb/README @@ -0,0 +1,13 @@ +These .mmdb databases were created with the mmdbwriter from MaxMind [1] for +testing purposes. See the main.go file. They only contain information about +LBL's network ranges: + + 128.3.0.0/16 + 131.243.0.0/16 + +Rebuild with: + + go build + ./testmmdb + +[1] https://github.com/maxmind/mmdbwriter diff --git a/testing/btest/Files/mmdb/go.mod b/testing/btest/Files/mmdb/go.mod new file mode 100644 index 0000000000..5aa1c9679c --- /dev/null +++ b/testing/btest/Files/mmdb/go.mod @@ -0,0 +1,11 @@ +module testmmdb + +go 1.21.0 + +require github.com/maxmind/mmdbwriter v1.0.0 + +require ( + github.com/oschwald/maxminddb-golang v1.12.0 // indirect + go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect + golang.org/x/sys v0.10.0 // indirect +) diff --git a/testing/btest/Files/mmdb/go.sum b/testing/btest/Files/mmdb/go.sum new file mode 100644 index 0000000000..a7e4aa9667 --- /dev/null +++ b/testing/btest/Files/mmdb/go.sum @@ -0,0 +1,16 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw= +github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8= +github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= +github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw= +go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/testing/btest/Files/mmdb/main.go b/testing/btest/Files/mmdb/main.go new file mode 100644 index 0000000000..ecad3ea8db --- /dev/null +++ b/testing/btest/Files/mmdb/main.go @@ -0,0 +1,71 @@ +// Create test MaxMind DB database files containing information about +// just LBL's IPv4 ranges for testing. +package main + +import ( + "log" + "net" + "os" + + "github.com/maxmind/mmdbwriter" + "github.com/maxmind/mmdbwriter/mmdbtype" +) + +func writeDb(fname, name string, record mmdbtype.Map, nets ...*net.IPNet) { + writer, err := mmdbwriter.New( + mmdbwriter.Options{ + DatabaseType: name, + }, + ) + if err != nil { + log.Fatal(err) + } + + for _, n := range nets { + if err = writer.Insert(n, record); err != nil { + log.Fatal(err) + } + } + + fh, err := os.Create(fname) + if err != nil { + log.Fatal(err) + } + defer fh.Close() + + _, err = writer.WriteTo(fh) + if err != nil { + log.Fatal(err) + } +} + +func main() { + _, net1, _ := net.ParseCIDR("128.3.0.0/16") + _, net2, _ := net.ParseCIDR("131.243.0.0/16") + + // The ASN record. + asn_record := mmdbtype.Map{} + asn_record["autonomous_system_number"] = mmdbtype.Uint32(16) + asn_record["autonomous_system_organization"] = mmdbtype.String("Lawrence Berkeley National Laboratory") + writeDb("GeoLite2-ASN.mmdb", "My-ASN-DB", asn_record, net1, net2) + + // The Location record. + loc_record := mmdbtype.Map{ + "country": mmdbtype.Map{ + "iso_code": mmdbtype.String("US"), + "names": mmdbtype.Map{ + "en": mmdbtype.String("United States"), + }, + }, + "location": mmdbtype.Map{ + "latitude": mmdbtype.Float64(37.75100), + "longitude": mmdbtype.Float64(-97.822000), + }, + "city": mmdbtype.Map{ + "names": mmdbtype.Map{ + "en": mmdbtype.String("Berkeley"), + }, + }, + } + writeDb("GeoLite2-City.mmdb", "My-City-DB", loc_record, net1, net2) +} diff --git a/testing/btest/core/mmdb/reopen.zeek b/testing/btest/core/mmdb/reopen.zeek new file mode 100644 index 0000000000..d9c120c8dc --- /dev/null +++ b/testing/btest/core/mmdb/reopen.zeek @@ -0,0 +1,37 @@ +# @TEST-DOC: Change the modification time of the mmdb database on every packet. This triggers reopening of the MMDB database. +# +# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h +# +# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb +# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log + +@load base/frameworks/reporter + +redef mmdb_dir = "./mmdb"; + +global pkt = 0; + +event new_packet(c: connection, p: pkt_hdr) + { + ++pkt; + # Set MMDB's modification time to current network time. + local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); + local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); + + if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), asn_fn), "") ) + exit(1); + + if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), city_fn), "") ) + exit(1); + + print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1); + print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1); + print network_time(), pkt, 131.243.0.1, "asn", lookup_autonomous_system(131.243.0.1); + print network_time(), pkt, 131.243.0.1, "location", lookup_location(131.243.0.1); + + if ( pkt == 4 ) + terminate(); + } diff --git a/testing/btest/core/mmdb/temporary-error.zeek b/testing/btest/core/mmdb/temporary-error.zeek new file mode 100644 index 0000000000..0a5102535c --- /dev/null +++ b/testing/btest/core/mmdb/temporary-error.zeek @@ -0,0 +1,85 @@ +# @TEST-DOC: Test a few error and recovery cases (corrupted, removed and restored MMDB databases). +# +# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h +# +# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb +# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb-backup +# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT mmdb_dir=./mmdb >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log + +@load base/frameworks/reporter + +redef mmdb_dir = "./mmdb"; + +global pkt = 0; + +event new_packet(c: connection, p: pkt_hdr) + { + ++pkt; + + local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); + local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); + + local asn_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-ASN.mmdb"); + local city_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-City.mmdb"); + + if ( pkt == 1 ) + { + print "start"; + } + if ( pkt == 2 ) + { + print "corrupting db"; + if ( ! piped_exec(fmt("truncate --size=8 %s", asn_fn), "") ) + exit(1); + + if ( ! piped_exec(fmt("truncate --size=8 %s", city_fn), "") ) + exit(1); + } + else if ( pkt == 4 ) + { + print "unlinking"; + if ( ! piped_exec(fmt("rm %s", asn_fn), "") ) + exit(1); + + if ( ! piped_exec(fmt("rm %s", city_fn), "") ) + exit(1); + } + else if ( pkt == 6 ) + { + # This should provoke an inode change. + print "unlinking and restoring"; + if ( ! piped_exec(fmt("mv %s %s.tmp; cp %s.tmp %s", asn_fn, asn_fn, asn_fn, asn_fn), "") ) + exit(1); + + if ( ! piped_exec(fmt("mv %s %s.tmp; cp %s.tmp %s", city_fn, city_fn, city_fn, city_fn), "") ) + exit(1); + } + else if ( pkt == 7 ) + { + print "done"; + terminate(); + return; + } + else if ( pkt == 3 || pkt == 5 ) + { + print "restoring backup db"; + if ( ! piped_exec(fmt("cp %s %s", asn_fn_backup, asn_fn), "") ) + exit(1); + + if ( ! piped_exec(fmt("cp %s %s", city_fn_backup, city_fn), "") ) + exit(1); + } + + # Set MMDB's modification time to current network time for predictability. + if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", asn_fn, network_time(), asn_fn), "") ) + exit(1); + + if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", city_fn, network_time(), city_fn), "") ) + exit(1); + + print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1); + print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1); + } diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek b/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek index e547833052..87a2caecbc 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek @@ -1,4 +1,3 @@ -# @TEST-REQUIRES: $SCRIPTS/have-spicy # @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT # @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff .stderr