Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Test synchronous/journal mode options for SQLite log writer

Browse source 
Also adds some small tweaks and adds the new feature to NEWS.
This commit is contained in:
Johanna Amann 2024-11-26 12:26:38 +00:00
parent 3ca56f7e0f
commit d592942ccb
8 changed files with 54 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

5
NEWS
View file

@ -81,6 +81,11 @@ New Functionality
of Zeek's AST after ZAM optimizations ran. This hook executes right before
the ``zeek_init()`` event is enqueued.
* The SQLite logger now supports setting the value of the SQLite synchronous mode,
as well as of the journal mode. For example, WAL mode can be enabled by setting:
redef LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL;
Changed Functionality
---------------------

16
scripts/base/frameworks/logging/writers/sqlite.zeek
View file

@ -21,7 +21,8 @@ export {
## String to use for empty fields. This should be different from
## *unset_field* to make the output unambiguous.
const empty_field = Log::empty_field &redef;
## Values supported for SQLite's PRAGMA synchronous statement.
type SQLiteSynchronous: enum {
SQLITE_SYNCHRONOUS_DEFAULT,
SQLITE_SYNCHRONOUS_OFF,
@ -40,22 +41,19 @@ export {
SQLITE_JOURNAL_MODE_WAL,
SQLITE_JOURNAL_MODE_OFF,
};
## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous
## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous
## statement with the provided value after connecting to the SQLite database. See
## `SQLite's synchronous documentation <https://www.sqlite.org/pragma.html#pragma_synchronous>`_
## for more details around performance and data safety trade offs.
const synchronous = SQLITE_SYNCHRONOUS_DEFAULT &redef;
## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA
## journal_mode statement with the provided value after connecting to
## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA
## journal_mode statement with the provided value after connecting to
## the SQLite database.
## `SQLite's journal_mode documentation <https://www.sqlite.org/pragma.html#pragma_journal_mode>`_
## for more details around performance, data safety trade offs
## and interaction with the PRAGMA synchronous statement.
const journal_mode = SQLITE_JOURNAL_MODE_DEFAULT &redef;
}

11
src/logging/writers/sqlite/SQLite.cc
View file

@ -131,7 +131,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) )
return false;
char* errorMsg = 0;
char* errorMsg = nullptr;
int res;
switch ( synchronous ) {
case SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break;
@ -145,8 +145,9 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
break;
default: Error("Invalid LogSQLite::synchronous enum"); return false;
}
if ( res != SQLITE_OK ) {
Error(Fmt("Error setting synchronous pragma %s", errorMsg));
Error(Fmt("Error setting synchronous pragma: %s", errorMsg));
sqlite3_free(errorMsg);
return false;
}
@ -169,13 +170,13 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
case SQLITE_JOURNAL_MODE_OFF: res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); break;
default: Error("Invalid LogSQLite::journal_mode enum"); return false;
}
if ( res != SQLITE_OK ) {
Error(Fmt("Error setting journal_mode pragma %s", errorMsg));
Error(Fmt("Error setting journal_mode pragma: %s", errorMsg));
sqlite3_free(errorMsg);
return false;
}
string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n";
//"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here.
@ -211,7 +212,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
create += "\n);";
errorMsg = 0;
errorMsg = nullptr;
res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg);
if ( res != SQLITE_OK ) {
Error(Fmt("Error executing table creation statement: %s", errorMsg));

1
src/logging/writers/sqlite/SQLite.h
View file

@ -55,6 +55,7 @@ private:
SQLITE_SYNCHRONOUS_FULL,
SQLITE_SYNCHRONOUS_EXTRA,
};
enum SQLiteJournalMode {
SQLITE_JOURNAL_MODE_DEFAULT,
SQLITE_JOURNAL_MODE_DELETE,

2
src/logging/writers/sqlite/sqlite.bif
View file

@ -27,5 +27,3 @@ enum SQLiteJournalMode %{
const synchronous: SQLiteSynchronous;
const journal_mode: SQLiteJournalMode;

2
testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr Normal file
View file

@ -0,0 +1,2 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
end of stderr

5
testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results Normal file
View file

@ -0,0 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Should be delete
delete
Should be WAL
wal

28
testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek Normal file
View file

@ -0,0 +1,28 @@
# This test exercises the SQLIte pragmas for synchronous and jornal_mode.
# Sadly, most of these do not have a way to test that they succeeded. So
# we mostly do the inverse test - if there are no error messages in .stderr
# everything should be fine.
#
# We can test for WAL journaling mode, as this persists and can be queried from
# the sqlite file.
# @TEST-REQUIRES: which sqlite3
# @TEST-REQUIRES: has-writer Zeek::SQLiteWriter
# @TEST-GROUP: sqlite
#
# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_DELETE
# @TEST-EXEC: echo "Should be delete" > results
# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results
# @TEST-EXEC: rm http.sqlite
# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_OFF LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_TRUNCATE
# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_NORMAL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_PERSIST
# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_MEMORY
# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL
# @TEST-EXEC: echo "Should be WAL" >> results
# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results
# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_FULL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_OFF
# @TEST-EXEC: btest-diff results
# @TEST-EXEC: echo "end of stderr" >> .stderr # btest-diff does not like the canonifier to have an empty output
# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v tablename' btest-diff .stderr
@load base/protocols/http