Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-13 12:08:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

broker/store: Extend SQLiteOptions around data safety and performance

Browse source 
Add configurability of synchronous and journal_mode for SQLite backed
Broker data stores. Setting these to synchronous=normal and journal_mode=wal
can significantly improve throughput at the cost of some durability in
the presence of power loss or OS crash. In the context of Zeek, this is
likely more than acceptable.

Additionally, add integrity_check and failure_mode options to support deleting
and re-opening a corrupted SQLite database at store creation.

Closes #2698
This commit is contained in:
Arne Welzel 2023-01-20 16:15:23 +01:00
parent b720f71e5e
commit f35cf228dc
11 changed files with 271 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
testing/btest/Baseline/broker.store.brokerstore-backend-sqlite-corrupt-delete/out Normal file
View file

@ -0,0 +1,6 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
store is open
populated 100 rows
store is open
populated 100 rows
100

4
testing/btest/Baseline/broker.store.brokerstore-backend-sqlite-corrupt-fail/out Normal file
View file

@ -0,0 +1,4 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
store is open
populated 100 rows
failed to open store

5
testing/btest/Baseline/broker.store.brokerstore-backend-sqlite-integrity-check/out Normal file
View file

@ -0,0 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
store is open
populated 100 rows
store is open
populated 100 rows

4
testing/btest/Baseline/broker.store.brokerstore-backend-sqlite-wal-mode/out Normal file
View file

@ -0,0 +1,4 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
populated 100 rows
wal
100

46
testing/btest/broker/store/brokerstore-backend-sqlite-corrupt-delete.zeek Normal file
View file

@ -0,0 +1,46 @@
# @TEST-DOC: Populate a database, corrupt it then observe Zeek's behavior deleting the database and reopening it.
# @TEST-REQUIRES: dd --version
# @TEST-REQUIRES: sqlite3 --version
# @TEST-REQUIRES: test -e /dev/zero
# @TEST-EXEC: zeek -b %INPUT >> out
# Evil
# @TEST-EXEC: dd if=/dev/zero of=path_to_db.sqlite seek=512 count=32 bs=1
# @TEST-EXEC: zeek -b %INPUT >> out
# This will find 100 rows, the previous DB was deleted.
# @TEST-EXEC: sqlite3 ./path_to_db.sqlite 'select count(*) from store' >> out;
#
# @TEST-EXEC: grep 'database disk image is malformed' .stderr
# @TEST-EXEC: btest-diff out
@load base/frameworks/broker/store
global test_store: opaque of Broker::Store;
global test_table: table[string] of count &broker_store="test_store_42";
event zeek_init()
{
test_store = Broker::create_master(
"test_store_42",
Broker::SQLITE,
Broker::BackendOptions(
$sqlite=Broker::SQLiteOptions(
$path="path_to_db.sqlite",
$failure_mode=Broker::SQLITE_FAILURE_MODE_DELETE,
),
),
);
if ( Broker::is_closed(test_store) ) {
print("failed to open store");
exit(1);
} else {
print("store is open");
}
local rows = 100;
local i = 0;
while ( ++i <= rows )
test_table[cat(|test_table|)] = i;
print fmt("populated %s rows", rows);
}

41
testing/btest/broker/store/brokerstore-backend-sqlite-corrupt-fail.zeek Normal file
View file

@ -0,0 +1,41 @@
# @TEST-DOC: Populate a database, corrupt it then observe Zeek's behavior not being able to open the database and store.
# @TEST-REQUIRES: dd --version
# @TEST-REQUIRES: test -e /dev/zero
# @TEST-EXEC: zeek -b %INPUT >> out
# Evil
# @TEST-EXEC: dd if=/dev/zero of=path_to_db.sqlite seek=512 count=32 bs=1
# @TEST-EXEC-FAIL: zeek -b %INPUT >> out
#
# @TEST-EXEC: grep 'database disk image is malformed' .stderr
# @TEST-EXEC: btest-diff out
@load base/frameworks/broker/store
global test_store: opaque of Broker::Store;
global test_table: table[string] of count &broker_store="test_store_42";
event zeek_init()
{
test_store = Broker::create_master(
"test_store_42",
Broker::SQLITE,
Broker::BackendOptions(
$sqlite=Broker::SQLiteOptions(
$path="path_to_db.sqlite",
),
),
);
if ( Broker::is_closed(test_store) ) {
print("failed to open store");
exit(1);
} else {
print("store is open");
}
local rows = 100;
local i = 0;
while ( ++i <= rows )
test_table[cat(|test_table|)] = i;
print fmt("populated %s rows", rows);
}

37
testing/btest/broker/store/brokerstore-backend-sqlite-integrity-check.zeek Normal file
View file

@ -0,0 +1,37 @@
# @TEST-DOC: Use SQLite backend option integrity_check, but not breaking anything.
# @TEST-EXEC: zeek -b %INPUT >> out
# @TEST-EXEC: zeek -b %INPUT >> out
# @TEST-EXEC: btest-diff out
@load base/frameworks/broker/store
global test_store: opaque of Broker::Store;
global test_table: table[string] of count &broker_store="test_store_42";
event zeek_init()
{
test_store = Broker::create_master(
"test_store_42",
Broker::SQLITE,
Broker::BackendOptions(
$sqlite=Broker::SQLiteOptions(
$path="path_to_db.sqlite",
$integrity_check=T,
),
),
);
if ( Broker::is_closed(test_store) ) {
print("failed to open store");
exit(1);
} else {
print("store is open");
}
local rows = 100;
local i = 0;
while ( ++i <= rows )
test_table[cat(|test_table|)] = i;
print fmt("populated %s rows", rows);
}

43
testing/btest/broker/store/brokerstore-backend-sqlite-wal-mode.zeek Normal file
View file

@ -0,0 +1,43 @@
# @TEST-DOC: Configure a broker store to be in WAL mode withou journal_mode NORMAL.
# @TEST-REQUIRES: sqlite3 --version
# @TEST-EXEC: zeek -b %INPUT > out 2>&1
#
# This is poking a bit at SQLite internals, but because WAL mode
# was flipped on, expect a wal and a shm file to exist.
# @TEST-EXEC: test -f path_to_db.sqlite || ls -lha >> out
# @TEST-EXEC: test -f path_to_db.sqlite-shm || ls -lha >> out
# @TEST-EXEC: test -f path_to_db.sqlite-wal || ls -lha >> out
# More poking, running sqlite3 should detect WAL mode, and the store
# table has 100 entries.
#
# @TEST-EXEC: sqlite3 ./path_to_db.sqlite 'PRAGMA journal_mode' >> out;
# @TEST-EXEC: sqlite3 ./path_to_db.sqlite 'select count(*) from store' >> out;
#
# @TEST-EXEC: btest-diff out
@load base/frameworks/broker/store
global test_store: opaque of Broker::Store;
global test_table: table[string] of count &broker_store="test_store_42";
event zeek_init()
{
test_store = Broker::create_master(
"test_store_42",
Broker::SQLITE,
Broker::BackendOptions(
$sqlite=Broker::SQLiteOptions(
$path="path_to_db.sqlite",
$synchronous=Broker::SQLITE_SYNCHRONOUS_NORMAL,
$journal_mode=Broker::SQLITE_JOURNAL_MODE_WAL,
),
),
);
local rows = 100;
local i = 0;
while ( ++i <= rows )
test_table[cat(|test_table|)] = i;
print fmt("populated %s rows", rows);
}