Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-13 12:08:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/topic/awelzel/mmdb-fix-and-tests'

Browse source 
* origin/topic/awelzel/mmdb-fix-and-tests:
  base/frameworks/spicy: Do not load base/misc/version
  btest/core/mmdb: Basic lookup_autonomous_system / lookup_location tests
  btest/files: Add mmdb testing databases and generator code
  zeek.bif: Switch mmdb stale check to network_time
  Fix mmdb pointer destruction

(cherry picked from commit 44b67894ac)
This commit is contained in:
Arne Welzel 2023-10-24 13:57:03 +02:00 • committed by Tim Wojtulewicz
parent 4d80668f1f
commit 65f8222b92
23 changed files with 329 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/base/frameworks/spicy/init-framework.zeek
View file

@ -1,5 +1,3 @@
@load base/misc/version
# doc-common-start # doc-common-start
module Spicy; module Spicy;

5
scripts/base/init-bare.zeek
View file

@ -1080,6 +1080,11 @@ type geo_autonomous_system: record {
## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support. ## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support.
const mmdb_dir: string = "" &redef; const mmdb_dir: string = "" &redef;
## Sets the interval for MaxMind DB file staleness checks. When Zeek detects a
## change in inode or modification time, the database is re-opened. Setting
## a negative interval disables staleness checks.
const mmdb_stale_check_interval: interval = 5min &redef;
## Computed entropy values. The record captures a number of measures that are ## Computed entropy values. The record captures a number of measures that are
## computed in parallel. See `A Pseudorandom Number Sequence Test Program ## computed in parallel. See `A Pseudorandom Number Sequence Test Program
## <http://www.fourmilab.ch/random>`_ for more information, Zeek uses the same ## <http://www.fourmilab.ch/random>`_ for more information, Zeek uses the same

27
src/zeek.bif
View file

@ -4004,7 +4004,7 @@ static void report_mmdb_msg(const char* format, ...)
va_list al; va_list al;
va_start(al, format); va_start(al, format);
std::string msg = zeek::util::fmt(format, al); std::string msg = zeek::util::vfmt(format, al);
va_end(al); va_end(al);
zeek::reporter->Info("%s", msg.data()); zeek::reporter->Info("%s", msg.data());
@ -4024,12 +4024,12 @@ private:
MMDB_s mmdb; MMDB_s mmdb;
struct stat file_info; struct stat file_info;
bool lookup_error; bool lookup_error;
std::chrono::time_point<std::chrono::steady_clock> last_check; double last_check;
}; };
MMDB::MMDB(const char* filename, struct stat info) MMDB::MMDB(const char* filename, struct stat info)
: file_info(info), lookup_error{false}, : file_info(info), lookup_error{false},
last_check{std::chrono::steady_clock::now()} last_check{zeek::run_state::network_time}
{ {
int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb);
@ -4064,25 +4064,28 @@ MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa)
bool MMDB::StaleDB() bool MMDB::StaleDB()
{ {
struct stat buf; struct stat buf;
using Clock = std::chrono::steady_clock;
std::chrono::time_point<Clock> now = Clock::now();
if ( lookup_error ) if ( lookup_error )
return true; return true;
// Only perform stat once per 5 minutes.
using Min = std::chrono::minutes; static double mmdb_stale_check_interval = zeek::id::find_val("mmdb_stale_check_interval")->AsInterval();
if ( std::chrono::duration_cast<Min>(now - last_check).count() < 5 )
if ( mmdb_stale_check_interval < 0.0 )
return false; return false;
last_check = now; if ( zeek::run_state::network_time - last_check < mmdb_stale_check_interval )
return false;
last_check = zeek::run_state::network_time;
if ( 0 != stat(mmdb.filename, &buf) ) if ( 0 != stat(mmdb.filename, &buf) )
return true; return true;
if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime )
{ {
report_mmdb_msg("Inode change detected for MaxMind DB [%s]", report_mmdb_msg("%s change detected for MaxMind DB [%s]",
buf.st_ino != file_info.st_ino ? "Inode" : "Modification time",
mmdb.filename); mmdb.filename);
return true; return true;
} }
@ -4152,7 +4155,7 @@ static void mmdb_check_loc()
{ {
report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_loc->Filename());
did_mmdb_loc_db_error = false; did_mmdb_loc_db_error = false;
mmdb_loc.release(); mmdb_loc.reset();
} }
} }
@ -4162,7 +4165,7 @@ static void mmdb_check_asn()
{ {
report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); report_mmdb_msg("Closing stale MaxMind DB [%s]", mmdb_asn->Filename());
did_mmdb_asn_db_error = false; did_mmdb_asn_db_error = false;
mmdb_asn.release(); mmdb_asn.reset();
} }
} }

2
testing/btest/Baseline/broker.remote_event/recv.recv.out
View file

@ -10,4 +10,4 @@ is_remote should be T, and is, T
receiver got ping: my-message, 4 receiver got ping: my-message, 4
is_remote should be T, and is, T is_remote should be T, and is, T
receiver got ping: my-message, 5 receiver got ping: my-message, 5
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] [num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]

2
testing/btest/Baseline/broker.remote_event_any/recv.recv.out
View file

@ -10,4 +10,4 @@ is_remote should be T, and is, T
receiver got ping: my-message, 4 receiver got ping: my-message, 4
is_remote should be T, and is, T is_remote should be T, and is, T
receiver got ping: my-message, 5 receiver got ping: my-message, 5
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] [num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]

2
testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out
View file

@ -5,4 +5,4 @@ receiver got ping: my-message, 2
receiver got ping: my-message, 3 receiver got ping: my-message, 3
receiver got ping: my-message, 4 receiver got ping: my-message, 4
receiver got ping: my-message, 5 receiver got ping: my-message, 5
[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] [num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]

17
testing/btest/Baseline/core.mmdb.reopen/out Normal file
View file

@ -0,0 +1,17 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299466805.0, 1, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299466805.0, 1, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470395.0, 2, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470395.0, 2, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470395.0, 2, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470395.0, 2, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470405.0, 3, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470405.0, 3, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299473995.0, 4, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299473995.0, 4, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299473995.0, 4, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299473995.0, 4, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]

11
testing/btest/Baseline/core.mmdb.reopen/reporter.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
ts level message location
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299473995.000000 Reporter::INFO received termination signal (empty)

20
testing/btest/Baseline/core.mmdb.temporary-error/out Normal file
View file

@ -0,0 +1,20 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
start
1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
corrupting db
1299470395.0, 2, 128.3.0.1, asn, [number=<uninitialized>, organization=<uninitialized>]
1299470395.0, 2, 128.3.0.1, location, [country_code=<uninitialized>, region=<uninitialized>, city=<uninitialized>, latitude=<uninitialized>, longitude=<uninitialized>]
restoring backup db
1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
unlinking
1299473995.0, 4, 128.3.0.1, asn, [number=<uninitialized>, organization=<uninitialized>]
1299473995.0, 4, 128.3.0.1, location, [country_code=<uninitialized>, region=<uninitialized>, city=<uninitialized>, latitude=<uninitialized>, longitude=<uninitialized>]
restoring backup db
1299474005.0, 5, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299474005.0, 5, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
unlinking and restoring
1299477595.0, 6, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299477595.0, 6, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
done

19
testing/btest/Baseline/core.mmdb.temporary-error/reporter.log Normal file
View file

@ -0,0 +1,19 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
ts level message location
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] <params>, line 1
1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] <params>, line 1
1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84
1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299477605.000000 Reporter::INFO received termination signal <params>, line 1

4
testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
View file

@ -266,10 +266,6 @@ scripts/base/init-frameworks-and-bifs.zeek
build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
scripts/base/frameworks/spicy/init-framework.zeek scripts/base/frameworks/spicy/init-framework.zeek
scripts/base/misc/version.zeek
scripts/base/frameworks/reporter/__load__.zeek
scripts/base/frameworks/reporter/main.zeek
scripts/base/utils/strings.zeek
build/scripts/builtin-plugins/__load__.zeek build/scripts/builtin-plugins/__load__.zeek
scripts/policy/misc/loaded-scripts.zeek scripts/policy/misc/loaded-scripts.zeek
scripts/base/utils/paths.zeek scripts/base/utils/paths.zeek

8
testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
View file

@ -266,10 +266,6 @@ scripts/base/init-frameworks-and-bifs.zeek
build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
scripts/base/frameworks/spicy/init-framework.zeek scripts/base/frameworks/spicy/init-framework.zeek
scripts/base/misc/version.zeek
scripts/base/frameworks/reporter/__load__.zeek
scripts/base/frameworks/reporter/main.zeek
scripts/base/utils/strings.zeek
scripts/base/init-default.zeek scripts/base/init-default.zeek
scripts/base/utils/active-http.zeek scripts/base/utils/active-http.zeek
scripts/base/utils/exec.zeek scripts/base/utils/exec.zeek
@ -277,6 +273,8 @@ scripts/base/init-default.zeek
scripts/base/utils/backtrace.zeek scripts/base/utils/backtrace.zeek
scripts/base/utils/conn-ids.zeek scripts/base/utils/conn-ids.zeek
scripts/base/utils/dir.zeek scripts/base/utils/dir.zeek
scripts/base/frameworks/reporter/__load__.zeek
scripts/base/frameworks/reporter/main.zeek
scripts/base/utils/paths.zeek scripts/base/utils/paths.zeek
scripts/base/utils/directions-and-hosts.zeek scripts/base/utils/directions-and-hosts.zeek
scripts/base/utils/email.zeek scripts/base/utils/email.zeek
@ -284,6 +282,7 @@ scripts/base/init-default.zeek
scripts/base/utils/geoip-distance.zeek scripts/base/utils/geoip-distance.zeek
scripts/base/utils/numbers.zeek scripts/base/utils/numbers.zeek
scripts/base/utils/queue.zeek scripts/base/utils/queue.zeek
scripts/base/utils/strings.zeek
scripts/base/utils/thresholds.zeek scripts/base/utils/thresholds.zeek
scripts/base/utils/time.zeek scripts/base/utils/time.zeek
scripts/base/utils/urls.zeek scripts/base/utils/urls.zeek
@ -347,6 +346,7 @@ scripts/base/init-default.zeek
scripts/base/frameworks/netcontrol/non-cluster.zeek scripts/base/frameworks/netcontrol/non-cluster.zeek
scripts/base/frameworks/telemetry/__load__.zeek scripts/base/frameworks/telemetry/__load__.zeek
scripts/base/frameworks/telemetry/main.zeek scripts/base/frameworks/telemetry/main.zeek
scripts/base/misc/version.zeek
scripts/base/frameworks/spicy/__load__.zeek scripts/base/frameworks/spicy/__load__.zeek
scripts/base/frameworks/spicy/main.zeek scripts/base/frameworks/spicy/main.zeek
scripts/base/protocols/conn/__load__.zeek scripts/base/protocols/conn/__load__.zeek

1
testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr
View file

@ -1,3 +1,2 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension
XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension

1
testing/btest/Files/mmdb/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
testmmdb

BIN
testing/btest/Files/mmdb/GeoLite2-ASN.mmdb Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.8 KiB

BIN
testing/btest/Files/mmdb/GeoLite2-City.mmdb Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.9 KiB

13
testing/btest/Files/mmdb/README Normal file
View file

@ -0,0 +1,13 @@
These .mmdb databases were created with the mmdbwriter from MaxMind [1] for
testing purposes. See the main.go file. They only contain information about
LBL's network ranges:
128.3.0.0/16
131.243.0.0/16
Rebuild with:
go build
./testmmdb
[1] https://github.com/maxmind/mmdbwriter

11
testing/btest/Files/mmdb/go.mod Normal file
View file

@ -0,0 +1,11 @@
module testmmdb
go 1.21.0
require github.com/maxmind/mmdbwriter v1.0.0
require (
github.com/oschwald/maxminddb-golang v1.12.0 // indirect
go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect
golang.org/x/sys v0.10.0 // indirect
)

16
testing/btest/Files/mmdb/go.sum Normal file
View file

@ -0,0 +1,16 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw=
github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8=
github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw=
go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

71
testing/btest/Files/mmdb/main.go Normal file
View file

@ -0,0 +1,71 @@
// Create test MaxMind DB database files containing information about
// just LBL's IPv4 ranges for testing.
package main
import (
"log"
"net"
"os"
"github.com/maxmind/mmdbwriter"
"github.com/maxmind/mmdbwriter/mmdbtype"
)
func writeDb(fname, name string, record mmdbtype.Map, nets ...*net.IPNet) {
writer, err := mmdbwriter.New(
mmdbwriter.Options{
DatabaseType: name,
},
)
if err != nil {
log.Fatal(err)
}
for _, n := range nets {
if err = writer.Insert(n, record); err != nil {
log.Fatal(err)
}
}
fh, err := os.Create(fname)
if err != nil {
log.Fatal(err)
}
defer fh.Close()
_, err = writer.WriteTo(fh)
if err != nil {
log.Fatal(err)
}
}
func main() {
_, net1, _ := net.ParseCIDR("128.3.0.0/16")
_, net2, _ := net.ParseCIDR("131.243.0.0/16")
// The ASN record.
asn_record := mmdbtype.Map{}
asn_record["autonomous_system_number"] = mmdbtype.Uint32(16)
asn_record["autonomous_system_organization"] = mmdbtype.String("Lawrence Berkeley National Laboratory")
writeDb("GeoLite2-ASN.mmdb", "My-ASN-DB", asn_record, net1, net2)
// The Location record.
loc_record := mmdbtype.Map{
"country": mmdbtype.Map{
"iso_code": mmdbtype.String("US"),
"names": mmdbtype.Map{
"en": mmdbtype.String("United States"),
},
},
"location": mmdbtype.Map{
"latitude": mmdbtype.Float64(37.75100),
"longitude": mmdbtype.Float64(-97.822000),
},
"city": mmdbtype.Map{
"names": mmdbtype.Map{
"en": mmdbtype.String("Berkeley"),
},
},
}
writeDb("GeoLite2-City.mmdb", "My-City-DB", loc_record, net1, net2)
}

37
testing/btest/core/mmdb/reopen.zeek Normal file
View file

@ -0,0 +1,37 @@
# @TEST-DOC: Change the modification time of the mmdb database on every packet. This triggers reopening of the MMDB database.
#
# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h
#
# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb
# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT >out
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
@load base/frameworks/reporter
redef mmdb_dir = "./mmdb";
global pkt = 0;
event new_packet(c: connection, p: pkt_hdr)
{
++pkt;
# Set MMDB's modification time to current network time.
local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb");
local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb");
if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), asn_fn), "") )
exit(1);
if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), city_fn), "") )
exit(1);
print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1);
print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1);
print network_time(), pkt, 131.243.0.1, "asn", lookup_autonomous_system(131.243.0.1);
print network_time(), pkt, 131.243.0.1, "location", lookup_location(131.243.0.1);
if ( pkt == 4 )
terminate();
}

85
testing/btest/core/mmdb/temporary-error.zeek Normal file
View file

@ -0,0 +1,85 @@
# @TEST-DOC: Test a few error and recovery cases (corrupted, removed and restored MMDB databases).
#
# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h
#
# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb
# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb-backup
# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT mmdb_dir=./mmdb >out
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
@load base/frameworks/reporter
redef mmdb_dir = "./mmdb";
global pkt = 0;
event new_packet(c: connection, p: pkt_hdr)
{
++pkt;
local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb");
local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb");
local asn_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-ASN.mmdb");
local city_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-City.mmdb");
if ( pkt == 1 )
{
print "start";
}
if ( pkt == 2 )
{
print "corrupting db";
if ( ! piped_exec(fmt("truncate --size=8 %s", asn_fn), "") )
exit(1);
if ( ! piped_exec(fmt("truncate --size=8 %s", city_fn), "") )
exit(1);
}
else if ( pkt == 4 )
{
print "unlinking";
if ( ! piped_exec(fmt("rm %s", asn_fn), "") )
exit(1);
if ( ! piped_exec(fmt("rm %s", city_fn), "") )
exit(1);
}
else if ( pkt == 6 )
{
# This should provoke an inode change.
print "unlinking and restoring";
if ( ! piped_exec(fmt("mv %s %s.tmp; cp %s.tmp %s", asn_fn, asn_fn, asn_fn, asn_fn), "") )
exit(1);
if ( ! piped_exec(fmt("mv %s %s.tmp; cp %s.tmp %s", city_fn, city_fn, city_fn, city_fn), "") )
exit(1);
}
else if ( pkt == 7 )
{
print "done";
terminate();
return;
}
else if ( pkt == 3 || pkt == 5 )
{
print "restoring backup db";
if ( ! piped_exec(fmt("cp %s %s", asn_fn_backup, asn_fn), "") )
exit(1);
if ( ! piped_exec(fmt("cp %s %s", city_fn_backup, city_fn), "") )
exit(1);
}
# Set MMDB's modification time to current network time for predictability.
if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", asn_fn, network_time(), asn_fn), "") )
exit(1);
if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", city_fn, network_time(), city_fn), "") )
exit(1);
print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1);
print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1);
}

1
testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek
View file

@ -1,4 +1,3 @@
# @TEST-REQUIRES: $SCRIPTS/have-spicy
# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT # @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT
# @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff conn.log
# @TEST-EXEC: btest-diff .stderr # @TEST-EXEC: btest-diff .stderr