Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-17 05:58:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/topic/christian/mmdb-configurability'

Browse source 
* origin/topic/christian/mmdb-configurability:
  Modernize various C++/Zeek-isms in the MMDB code.
  Fix MMDB code to re-open explicitly opened DBs correctly
  Add btest to verify behavior of re-opened MMDBs opened directly via BIFs
  Simplify MMDB code by moving more lookup functionality into MMDB class
  Move MMDB logic out of mmdb.bif and into MMDB.cc/h.
  Fix mmdb.temporary-error testcase when MMDBs are installed on system
  Adapt MMDB BiF code to new script-layer variables
  Update btest baselines to reflect introduction of mmdb.bif
  Move MaxMind/GeoIP BiF functionality into separate file
  Provide script-level configurability of MaxMind DB placement on disk
  Sort toplevel .bif list in CMakeLists
This commit is contained in:
Arne Welzel 2024-01-12 09:27:58 +01:00
commit ffffd88bef
17 changed files with 695 additions and 536 deletions
Download patch file Download diff file Expand all files Collapse all files

17
testing/btest/Baseline/core.mmdb.explicit-open/out Normal file
View file

@ -0,0 +1,17 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
1299466805.0, 1, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299466805.0, 1, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299466805.0, 1, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299466805.0, 1, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470395.0, 2, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470395.0, 2, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470395.0, 2, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470395.0, 2, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470405.0, 3, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470405.0, 3, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299470405.0, 3, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299470405.0, 3, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299473995.0, 4, 128.3.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299473995.0, 4, 128.3.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]
1299473995.0, 4, 131.243.0.1, asn, [number=16, organization=Lawrence Berkeley National Laboratory]
1299473995.0, 4, 131.243.0.1, location, [country_code=US, region=<uninitialized>, city=Berkeley, latitude=37.751, longitude=-97.822]

11
testing/btest/Baseline/core.mmdb.explicit-open/reporter.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
ts level message location
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] (empty)
1299473995.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] (empty)
1299473995.000000 Reporter::INFO received termination signal (empty)

8
testing/btest/Baseline/core.mmdb.temporary-error/reporter.log
View file

@ -3,15 +3,15 @@ ts level message location
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] <params>, line 1
1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 100
1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 101
1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] <params>, line 1
1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 101
1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 102
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 100
1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 101
1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1
1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 101
1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 102
1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] <params>, line 1
1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] <params>, line 1

1
testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
View file

@ -20,6 +20,7 @@ scripts/base/init-bare.zeek
build/scripts/base/bif/supervisor.bif.zeek
build/scripts/base/bif/packet_analysis.bif.zeek
build/scripts/base/bif/CPP-load.bif.zeek
build/scripts/base/bif/mmdb.bif.zeek
build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek
build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek
build/scripts/base/bif/event.bif.zeek

1
testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
View file

@ -20,6 +20,7 @@ scripts/base/init-bare.zeek
build/scripts/base/bif/supervisor.bif.zeek
build/scripts/base/bif/packet_analysis.bif.zeek
build/scripts/base/bif/CPP-load.bif.zeek
build/scripts/base/bif/mmdb.bif.zeek
build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek
build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek
build/scripts/base/bif/event.bif.zeek

12
testing/btest/Baseline/plugins.hooks/output
View file

@ -493,6 +493,7 @@
0.000000 MetaHookPost LoadFile(0, ./main, <...>/main.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./main.zeek, <...>/main.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./patterns, <...>/patterns.zeek) -> -1
@ -580,6 +581,7 @@
0.000000 MetaHookPost LoadFile(0, base<...>/main, <...>/main.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/main.zeek, <...>/main.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/mpls, <...>/mpls) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/nflog, <...>/nflog) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3) -> -1
@ -778,6 +780,7 @@
0.000000 MetaHookPost LoadFileExtended(0, ./main, <...>/main.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./main.zeek, <...>/main.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./patterns, <...>/patterns.zeek) -> (-1, <no content>)
@ -865,6 +868,7 @@
0.000000 MetaHookPost LoadFileExtended(0, base<...>/main, <...>/main.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/mpls, <...>/mpls) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/nflog, <...>/nflog) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3) -> (-1, <no content>)
@ -1411,6 +1415,7 @@
0.000000 MetaHookPre LoadFile(0, ./main, <...>/main.zeek)
0.000000 MetaHookPre LoadFile(0, ./main.zeek, <...>/main.zeek)
0.000000 MetaHookPre LoadFile(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./option.bif.zeek, <...>/option.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./patterns, <...>/patterns.zeek)
@ -1498,6 +1503,7 @@
0.000000 MetaHookPre LoadFile(0, base<...>/main, <...>/main.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/main.zeek, <...>/main.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/messaging.bif, <...>/messaging.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/mpls, <...>/mpls)
0.000000 MetaHookPre LoadFile(0, base<...>/nflog, <...>/nflog)
0.000000 MetaHookPre LoadFile(0, base<...>/novell_802_3, <...>/novell_802_3)
@ -1696,6 +1702,7 @@
0.000000 MetaHookPre LoadFileExtended(0, ./main, <...>/main.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./main.zeek, <...>/main.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./mmdb.bif.zeek, <...>/mmdb.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./patterns, <...>/patterns.zeek)
@ -1783,6 +1790,7 @@
0.000000 MetaHookPre LoadFileExtended(0, base<...>/main, <...>/main.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/main.zeek, <...>/main.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/mmdb.bif, <...>/mmdb.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/mpls, <...>/mpls)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/nflog, <...>/nflog)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/novell_802_3, <...>/novell_802_3)
@ -2337,6 +2345,7 @@
0.000000 | HookLoadFile ./main <...>/main.zeek
0.000000 | HookLoadFile ./main.zeek <...>/main.zeek
0.000000 | HookLoadFile ./messaging.bif.zeek <...>/messaging.bif.zeek
0.000000 | HookLoadFile ./mmdb.bif.zeek <...>/mmdb.bif.zeek
0.000000 | HookLoadFile ./office <...>/office.sig
0.000000 | HookLoadFile ./option.bif.zeek <...>/option.bif.zeek
0.000000 | HookLoadFile ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek
@ -2427,6 +2436,7 @@
0.000000 | HookLoadFile base<...>/main <...>/main.zeek
0.000000 | HookLoadFile base<...>/main.zeek <...>/main.zeek
0.000000 | HookLoadFile base<...>/messaging.bif <...>/messaging.bif.zeek
0.000000 | HookLoadFile base<...>/mmdb.bif <...>/mmdb.bif.zeek
0.000000 | HookLoadFile base<...>/mpls <...>/mpls
0.000000 | HookLoadFile base<...>/nflog <...>/nflog
0.000000 | HookLoadFile base<...>/novell_802_3 <...>/novell_802_3
@ -2622,6 +2632,7 @@
0.000000 | HookLoadFileExtended ./main <...>/main.zeek
0.000000 | HookLoadFileExtended ./main.zeek <...>/main.zeek
0.000000 | HookLoadFileExtended ./messaging.bif.zeek <...>/messaging.bif.zeek
0.000000 | HookLoadFileExtended ./mmdb.bif.zeek <...>/mmdb.bif.zeek
0.000000 | HookLoadFileExtended ./office <...>/office.sig
0.000000 | HookLoadFileExtended ./option.bif.zeek <...>/option.bif.zeek
0.000000 | HookLoadFileExtended ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek
@ -2712,6 +2723,7 @@
0.000000 | HookLoadFileExtended base<...>/main <...>/main.zeek
0.000000 | HookLoadFileExtended base<...>/main.zeek <...>/main.zeek
0.000000 | HookLoadFileExtended base<...>/messaging.bif <...>/messaging.bif.zeek
0.000000 | HookLoadFileExtended base<...>/mmdb.bif <...>/mmdb.bif.zeek
0.000000 | HookLoadFileExtended base<...>/mpls <...>/mpls
0.000000 | HookLoadFileExtended base<...>/nflog <...>/nflog
0.000000 | HookLoadFileExtended base<...>/novell_802_3 <...>/novell_802_3

50
testing/btest/core/mmdb/explicit-open.zeek Normal file
View file

@ -0,0 +1,50 @@
# @TEST-DOC: verifies that the explicit BiFs for loading MMDBs work, including when re-opening.
#
# Like other MMDB tests, this uses a pcap to use each packet as a driver to
# touch the DBs involved upon each packet, triggering DB reloads.
#
# @TEST-REQUIRES: grep -q "#define USE_GEOIP" $BUILD/zeek-config.h
#
# @TEST-EXEC: cp -R $FILES/mmdb ./mmdb
# @TEST-EXEC: zeek -b -r $TRACES/rotation.trace %INPUT >out
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
# @TEST-EXEC: zeek-cut -m < reporter.log > reporter.log.tmp && mv reporter.log.tmp reporter.log
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
@load base/frameworks/reporter
global pkt = 0;
global asn_fn = "./mmdb/GeoLite2-ASN.mmdb";
global city_fn = "./mmdb/GeoLite2-City.mmdb";
function timestamp(n: count): string
{
assert n <= 60;
return fmt("2020-01-01T00:%s:00", n);
}
event new_packet(c: connection, p: pkt_hdr)
{
++pkt;
print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1);
print network_time(), pkt, 128.3.0.1, "location", lookup_location(128.3.0.1);
print network_time(), pkt, 131.243.0.1, "asn", lookup_autonomous_system(131.243.0.1);
print network_time(), pkt, 131.243.0.1, "location", lookup_location(131.243.0.1);
# Increment MMDBs' modification time, triggering a re-open.
if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(asn_fn)), "") )
exit(1);
if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), safe_shell_quote(city_fn)), "") )
exit(1);
if ( pkt == 4 )
terminate();
}
event zeek_init()
{
assert mmdb_open_asn_db(asn_fn);
assert mmdb_open_location_db(city_fn);
}

1
testing/btest/core/mmdb/temporary-error.zeek
View file

@ -13,6 +13,7 @@
@load base/frameworks/reporter
redef mmdb_dir = "./mmdb";
redef mmdb_dir_fallbacks = vector(); # Clear out fallbacks to avoid influence on tests
global pkt = 0;