mirror of
https://github.com/zeek/zeek.git
synced 2025-10-13 03:58:20 +00:00
Merge remote-tracking branch 'origin/topic/awelzel/mmdb-fix-and-tests'
* origin/topic/awelzel/mmdb-fix-and-tests:
base/frameworks/spicy: Do not load base/misc/version
btest/core/mmdb: Basic lookup_autonomous_system / lookup_location tests
btest/files: Add mmdb testing databases and generator code
zeek.bif: Switch mmdb stale check to network_time
Fix mmdb pointer destruction
(cherry picked from commit 44b67894ac)
This commit is contained in:
Arne Welzel
Tim Wojtulewicz
parent
4d80668f1f
commit
65f8222b92
23 changed files with 329 additions and 28 deletions
|
|
@ -1,5 +1,3 @@
|
|||
@load base/misc/version
|
||||
|
||||
# doc-common-start
|
||||
module Spicy;
|
||||
|
||||
|
|
|
|||
|
|
@ -1080,6 +1080,11 @@ type geo_autonomous_system: record {
|
|||
## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support.
|
||||
const mmdb_dir: string = "" &redef;
|
||||
|
||||
## Sets the interval for MaxMind DB file staleness checks. When Zeek detects a
|
||||
## change in inode or modification time, the database is re-opened. Setting
|
||||
## a negative interval disables staleness checks.
|
||||
const mmdb_stale_check_interval: interval = 5min &redef;
|
||||
|
||||
## Computed entropy values. The record captures a number of measures that are
|
||||
## computed in parallel. See `A Pseudorandom Number Sequence Test Program
|
||||
## <http://www.fourmilab.ch/random>`_ for more information, Zeek uses the same
|
||||
|
|
|
|||
27
src/zeek.bif
27
src/zeek.bif
|
|
@ -4004,7 +4004,7 @@ static void report_mmdb_msg(const char* format, ...)
|
|||
|
||||
va_list al;
|
||||
va_start(al, format);
|
||||
std::string msg = zeek::util::fmt(format, al);
|
||||
std::string msg = zeek::util::vfmt(format, al);
|
||||
va_end(al);
|
||||
|
||||
zeek::reporter->Info("%s", msg.data());
|
||||
|
|
@ -4024,12 +4024,12 @@ private:
|
|||
MMDB_s mmdb;
|
||||
struct stat file_info;
|
||||
bool lookup_error;
|
||||
std::chrono::time_point<std::chrono::steady_clock> last_check;
|
||||
double last_check;
|
||||
};
|
||||
|
||||
MMDB::MMDB(const char* filename, struct stat info)
|
||||
: file_info(info), lookup_error{false},
|
||||
last_check{std::chrono::steady_clock::now()}
|
||||
last_check{zeek::run_state::network_time}
|
||||
{
|
||||
int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb);
|
||||
|
||||
|
|
@ -4064,25 +4064,28 @@ MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa)
|
|||
bool MMDB::StaleDB()
|
||||
{
|
||||
struct stat buf;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
std::chrono::time_point<Clock> now = Clock::now();
|
||||
|
||||
if ( lookup_error )
|
||||
return true;
|
||||
|
||||
// Only perform stat once per 5 minutes.
|
||||
using Min = std::chrono::minutes;
|
||||
if ( std::chrono::duration_cast<Min>(now - last_check).count() < 5 )
|
||||
|
||||
static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval();
|
||||
|
||||
if ( mmdb_stale_check_interval < 0.0 )
|
||||
return false;
|
||||
|
||||
last_check = now;
|
||||
if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval )
|
||||
return false;
|
||||
|
||||
last_check = zeek::run_state::network_time;
|
||||
|
||||
if ( 0 != stat(mmdb.filename, &buf) )
|
||||
return true;
|
||||
|
||||
if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime )
|
||||
{
|
||||
report_mmdb_msg("Inode change detected for MaxMind DB [%s]",
|
||||
report_mmdb_msg("%s change detected for MaxMind DB [%s]",
|
||||
buf.st_ino != file_info.st_ino ? "Inode" : "Modification time",
|
||||
mmdb.filename);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -4152,7 +4155,7 @@ static void mmdb_check_loc()
|
|||
{
|
||||
report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename());
|
||||
did_mmdb_loc_db_error = false;
|
||||
mmdb_loc.release();
|
||||
mmdb_loc.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4162,7 +4165,7 @@ static void mmdb_check_asn()
|
|||
{
|
||||
report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename());
|
||||
did_mmdb_asn_db_error = false;
|
||||
mmdb_asn.release();
|
||||
mmdb_asn.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,4 +10,4 @@ is_remote should be T, and is, T
|
|||
receiver got ping: my-message, 4
|
||||
is_remote should be T, and is, T
|
||||
receiver got ping: my-message, 5
|
||||
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]
|
||||
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
|
||||
|
|
|
|||
|
|
@ -10,4 +10,4 @@ is_remote should be T, and is, T
|
|||
receiver got ping: my-message, 4
|
||||
is_remote should be T, and is, T
|
||||
receiver got ping: my-message, 5
|
||||
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]
|
||||
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
|
||||
|
|
|
|||
|
|
@ -5,4 +5,4 @@ receiver got ping: my-message, 2
|
|||
receiver got ping: my-message, 3
|
||||
receiver got ping: my-message, 4
|
||||
receiver got ping: my-message, 5
|
||||
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]
|
||||
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
|
||||
|
|
|
|||
17
testing/btest/Baseline/core.mmdb.reopen/out
Normal file
17
testing/btest/Baseline/core.mmdb.reopen/out
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
1299466805.0, 1, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299466805.0, 1, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
1299470395.0, 2, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299470395.0, 2, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
1299470395.0, 2, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299470395.0, 2, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
1299470405.0, 3, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299470405.0, 3, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
1299473995.0, 4, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299473995.0, 4, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
1299473995.0, 4, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299473995.0, 4, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
11
testing/btest/Baseline/core.mmdb.reopen/reporter.log
Normal file
11
testing/btest/Baseline/core.mmdb.reopen/reporter.log
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
ts level message location
|
||||
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
|
||||
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
|
||||
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
|
||||
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
|
||||
1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
|
||||
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
|
||||
1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
|
||||
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
|
||||
1299473995.000000 Reporter::INFO received termination signal (empty)
|
||||
20
testing/btest/Baseline/core.mmdb.temporary-error/out
Normal file
20
testing/btest/Baseline/core.mmdb.temporary-error/out
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
start
|
||||
1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
corrupting db
|
||||
1299470395.0, 2, 128.3.0.1, asn, [number=<uninitialized>, organization=<uninitialized>]
|
||||
1299470395.0, 2, 128.3.0.1, location, [country_code=<uninitialized>, region=<uninitialized>, city=<uninitialized>, latitude=<uninitialized>, longitude=<uninitialized>]
|
||||
restoring backup db
|
||||
1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
unlinking
|
||||
1299473995.0, 4, 128.3.0.1, asn, [number=<uninitialized>, organization=<uninitialized>]
|
||||
1299473995.0, 4, 128.3.0.1, location, [country_code=<uninitialized>, region=<uninitialized>, city=<uninitialized>, latitude=<uninitialized>, longitude=<uninitialized>]
|
||||
restoring backup db
|
||||
1299474005.0, 5, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299474005.0, 5, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
unlinking and restoring
|
||||
1299477595.0, 6, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
|
||||
1299477595.0, 6, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
|
||||
done
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
ts level message location
|
||||
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
|
||||
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
|
||||
1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] <params>, line 1
|
||||
1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83
|
||||
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
|
||||
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
|
||||
1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] <params>, line 1
|
||||
1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84
|
||||
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
|
||||
1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83
|
||||
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
|
||||
1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84
|
||||
1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
|
||||
1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
|
||||
1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
|
||||
1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
|
||||
1299477605.000000 Reporter::INFO received termination signal <params>, line 1
|
||||
|
|
@ -266,10 +266,6 @@ scripts/base/init-frameworks-and-bifs.zeek
|
|||
build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
|
||||
build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
|
||||
scripts/base/frameworks/spicy/init-framework.zeek
|
||||
scripts/base/misc/version.zeek
|
||||
scripts/base/frameworks/reporter/__load__.zeek
|
||||
scripts/base/frameworks/reporter/main.zeek
|
||||
scripts/base/utils/strings.zeek
|
||||
build/scripts/builtin-plugins/__load__.zeek
|
||||
scripts/policy/misc/loaded-scripts.zeek
|
||||
scripts/base/utils/paths.zeek
|
||||
|
|
|
|||
|
|
@ -266,10 +266,6 @@ scripts/base/init-frameworks-and-bifs.zeek
|
|||
build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
|
||||
build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
|
||||
scripts/base/frameworks/spicy/init-framework.zeek
|
||||
scripts/base/misc/version.zeek
|
||||
scripts/base/frameworks/reporter/__load__.zeek
|
||||
scripts/base/frameworks/reporter/main.zeek
|
||||
scripts/base/utils/strings.zeek
|
||||
scripts/base/init-default.zeek
|
||||
scripts/base/utils/active-http.zeek
|
||||
scripts/base/utils/exec.zeek
|
||||
|
|
@ -277,6 +273,8 @@ scripts/base/init-default.zeek
|
|||
scripts/base/utils/backtrace.zeek
|
||||
scripts/base/utils/conn-ids.zeek
|
||||
scripts/base/utils/dir.zeek
|
||||
scripts/base/frameworks/reporter/__load__.zeek
|
||||
scripts/base/frameworks/reporter/main.zeek
|
||||
scripts/base/utils/paths.zeek
|
||||
scripts/base/utils/directions-and-hosts.zeek
|
||||
scripts/base/utils/email.zeek
|
||||
|
|
@ -284,6 +282,7 @@ scripts/base/init-default.zeek
|
|||
scripts/base/utils/geoip-distance.zeek
|
||||
scripts/base/utils/numbers.zeek
|
||||
scripts/base/utils/queue.zeek
|
||||
scripts/base/utils/strings.zeek
|
||||
scripts/base/utils/thresholds.zeek
|
||||
scripts/base/utils/time.zeek
|
||||
scripts/base/utils/urls.zeek
|
||||
|
|
@ -347,6 +346,7 @@ scripts/base/init-default.zeek
|
|||
scripts/base/frameworks/netcontrol/non-cluster.zeek
|
||||
scripts/base/frameworks/telemetry/__load__.zeek
|
||||
scripts/base/frameworks/telemetry/main.zeek
|
||||
scripts/base/misc/version.zeek
|
||||
scripts/base/frameworks/spicy/__load__.zeek
|
||||
scripts/base/frameworks/spicy/main.zeek
|
||||
scripts/base/protocols/conn/__load__.zeek
|
||||
|
|
|
|||
|
|
@ -1,3 +1,2 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension
|
||||
XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension
|
||||
|
|
|
|||
1
testing/btest/Files/mmdb/.gitignore
vendored
Normal file
1
testing/btest/Files/mmdb/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
testmmdb
|
||||
BIN
testing/btest/Files/mmdb/GeoLite2-ASN.mmdb
Normal file
BIN
testing/btest/Files/mmdb/GeoLite2-ASN.mmdb
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 2.8 KiB |
BIN
testing/btest/Files/mmdb/GeoLite2-City.mmdb
Normal file
BIN
testing/btest/Files/mmdb/GeoLite2-City.mmdb
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 2.9 KiB |
13
testing/btest/Files/mmdb/README
Normal file
13
testing/btest/Files/mmdb/README
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
These .mmdb databases were created with the mmdbwriter from MaxMind [1] for
|
||||
testing purposes. See the main.go file. They only contain information about
|
||||
LBL's network ranges:
|
||||
|
||||
128.3.0.0/16
|
||||
131.243.0.0/16
|
||||
|
||||
Rebuild with:
|
||||
|
||||
go build
|
||||
./testmmdb
|
||||
|
||||
[1] https://github.com/maxmind/mmdbwriter
|
||||
11
testing/btest/Files/mmdb/go.mod
Normal file
11
testing/btest/Files/mmdb/go.mod
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
module testmmdb
|
||||
|
||||
go 1.21.0
|
||||
|
||||
require github.com/maxmind/mmdbwriter v1.0.0
|
||||
|
||||
require (
|
||||
github.com/oschwald/maxminddb-golang v1.12.0 // indirect
|
||||
go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
)
|
||||
16
testing/btest/Files/mmdb/go.sum
Normal file
16
testing/btest/Files/mmdb/go.sum
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw=
|
||||
github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8=
|
||||
github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
|
||||
github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw=
|
||||
go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc=
|
||||
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
71
testing/btest/Files/mmdb/main.go
Normal file
71
testing/btest/Files/mmdb/main.go
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
// Create test MaxMind DB database files containing information about
|
||||
// just LBL's IPv4 ranges for testing.
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
|
||||
"github.com/maxmind/mmdbwriter"
|
||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||
)
|
||||
|
||||
func writeDb(fname, name string, record mmdbtype.Map, nets ...*net.IPNet) {
|
||||
writer, err := mmdbwriter.New(
|
||||
mmdbwriter.Options{
|
||||
DatabaseType: name,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, n := range nets {
|
||||
if err = writer.Insert(n, record); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
fh, err := os.Create(fname)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer fh.Close()
|
||||
|
||||
_, err = writer.WriteTo(fh)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
_, net1, _ := net.ParseCIDR("128.3.0.0/16")
|
||||
_, net2, _ := net.ParseCIDR("131.243.0.0/16")
|
||||
|
||||
// The ASN record.
|
||||
asn_record := mmdbtype.Map{}
|
||||
asn_record["autonomous_system_number"] = mmdbtype.Uint32(16)
|
||||
asn_record["autonomous_system_organization"] = mmdbtype.String("Lawrence Berkeley National Laboratory")
|
||||
writeDb("GeoLite2-ASN.mmdb", "My-ASN-DB", asn_record, net1, net2)
|
||||
|
||||
// The Location record.
|
||||
loc_record := mmdbtype.Map{
|
||||
"country": mmdbtype.Map{
|
||||
"iso_code": mmdbtype.String("US"),
|
||||
"names": mmdbtype.Map{
|
||||
"en": mmdbtype.String("United States"),
|
||||
},
|
||||
},
|
||||
"location": mmdbtype.Map{
|
||||
"latitude": mmdbtype.Float64(37.75100),
|
||||
"longitude": mmdbtype.Float64(-97.822000),
|
||||
},
|
||||
"city": mmdbtype.Map{
|
||||
"names": mmdbtype.Map{
|
||||
"en": mmdbtype.String("Berkeley"),
|
||||
},
|
||||
},
|
||||
}
|
||||
writeDb("GeoLite2-City.mmdb", "My-City-DB", loc_record, net1, net2)
|
||||
}
|
||||
37
testing/btest/core/mmdb/reopen.zeek
Normal file
37
testing/btest/core/mmdb/reopen.zeek
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# @TEST-DOC: Change the modification time of the mmdb database on every packet. This triggers reopening of the MMDB database.
|
||||
#
|
||||
# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h
|
||||
#
|
||||
# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb
|
||||
# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT >out
|
||||
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
|
||||
# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log
|
||||
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
|
||||
|
||||
@load base/frameworks/reporter
|
||||
|
||||
redef mmdb_dir = "./mmdb";
|
||||
|
||||
global pkt = 0;
|
||||
|
||||
event new_packet(c: connection, p: pkt_hdr)
|
||||
{
|
||||
++pkt;
|
||||
# Set MMDB's modification time to current network time.
|
||||
local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb");
|
||||
local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb");
|
||||
|
||||
if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), asn_fn), "") )
|
||||
exit(1);
|
||||
|
||||
if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), city_fn), "") )
|
||||
exit(1);
|
||||
|
||||
print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1);
|
||||
print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1);
|
||||
print network_time(), pkt, 131.243.0.1, "asn", lookup_autonomous_system(131.243.0.1);
|
||||
print network_time(), pkt, 131.243.0.1, "location", lookup_location(131.243.0.1);
|
||||
|
||||
if ( pkt == 4 )
|
||||
terminate();
|
||||
}
|
||||
85
testing/btest/core/mmdb/temporary-error.zeek
Normal file
85
testing/btest/core/mmdb/temporary-error.zeek
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
# @TEST-DOC: Test a few error and recovery cases (corrupted, removed and restored MMDB databases).
|
||||
#
|
||||
# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h
|
||||
#
|
||||
# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb
|
||||
# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb-backup
|
||||
# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT mmdb_dir=./mmdb >out
|
||||
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
|
||||
# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log
|
||||
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
|
||||
|
||||
@load base/frameworks/reporter
|
||||
|
||||
redef mmdb_dir = "./mmdb";
|
||||
|
||||
global pkt = 0;
|
||||
|
||||
event new_packet(c: connection, p: pkt_hdr)
|
||||
{
|
||||
++pkt;
|
||||
|
||||
local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb");
|
||||
local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb");
|
||||
|
||||
local asn_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-ASN.mmdb");
|
||||
local city_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-City.mmdb");
|
||||
|
||||
if ( pkt == 1 )
|
||||
{
|
||||
print "start";
|
||||
}
|
||||
if ( pkt == 2 )
|
||||
{
|
||||
print "corrupting db";
|
||||
if ( ! piped_exec(fmt("truncate --size=8 %s", asn_fn), "") )
|
||||
exit(1);
|
||||
|
||||
if ( ! piped_exec(fmt("truncate --size=8 %s", city_fn), "") )
|
||||
exit(1);
|
||||
}
|
||||
else if ( pkt == 4 )
|
||||
{
|
||||
print "unlinking";
|
||||
if ( ! piped_exec(fmt("rm %s", asn_fn), "") )
|
||||
exit(1);
|
||||
|
||||
if ( ! piped_exec(fmt("rm %s", city_fn), "") )
|
||||
exit(1);
|
||||
}
|
||||
else if ( pkt == 6 )
|
||||
{
|
||||
# This should provoke an inode change.
|
||||
print "unlinking and restoring";
|
||||
if ( ! piped_exec(fmt("mv %s %s.tmp; cp %s.tmp %s", asn_fn, asn_fn, asn_fn, asn_fn), "") )
|
||||
exit(1);
|
||||
|
||||
if ( ! piped_exec(fmt("mv %s %s.tmp; cp %s.tmp %s", city_fn, city_fn, city_fn, city_fn), "") )
|
||||
exit(1);
|
||||
}
|
||||
else if ( pkt == 7 )
|
||||
{
|
||||
print "done";
|
||||
terminate();
|
||||
return;
|
||||
}
|
||||
else if ( pkt == 3 || pkt == 5 )
|
||||
{
|
||||
print "restoring backup db";
|
||||
if ( ! piped_exec(fmt("cp %s %s", asn_fn_backup, asn_fn), "") )
|
||||
exit(1);
|
||||
|
||||
if ( ! piped_exec(fmt("cp %s %s", city_fn_backup, city_fn), "") )
|
||||
exit(1);
|
||||
}
|
||||
|
||||
# Set MMDB's modification time to current network time for predictability.
|
||||
if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", asn_fn, network_time(), asn_fn), "") )
|
||||
exit(1);
|
||||
|
||||
if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", city_fn, network_time(), city_fn), "") )
|
||||
exit(1);
|
||||
|
||||
print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1);
|
||||
print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1);
|
||||
}
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
# @TEST-REQUIRES: $SCRIPTS/have-spicy
|
||||
# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT
|
||||
# @TEST-EXEC: btest-diff conn.log
|
||||
# @TEST-EXEC: btest-diff .stderr
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue