diff --git a/CHANGES b/CHANGES index b9834f78a0..bbfa5a0828 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +7.1.0-dev.635 | 2024-11-27 08:29:19 +0000 + + * Support setting the synchronous flag and the journal_mode for the SQLite log writer. + + These options allow for a significant speed-up of SQLite logging, at the expense + of some data security. To change the default settings, redef the LogSQLite::synchronous + and LogSQLite::journal_mode consts. (Mymaqn) + 7.1.0-dev.629 | 2024-11-26 17:45:08 +0100 * ci/test.sh: Run doctest with TZ=UTC (Arne Welzel, Corelight) diff --git a/NEWS b/NEWS index 8b5382f528..fa36d249f1 100644 --- a/NEWS +++ b/NEWS @@ -81,6 +81,11 @@ New Functionality of Zeek's AST after ZAM optimizations ran. This hook executes right before the ``zeek_init()`` event is enqueued. +* The SQLite logger now supports setting the value of the SQLite synchronous mode, + as well as of the journal mode. For example, WAL mode can be enabled by setting: + + redef LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL; + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 90674fa1c9..8d196e74ce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.1.0-dev.629 +7.1.0-dev.635 diff --git a/scripts/base/frameworks/logging/writers/sqlite.zeek b/scripts/base/frameworks/logging/writers/sqlite.zeek index f7ebd9130c..19253ccb53 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.zeek +++ b/scripts/base/frameworks/logging/writers/sqlite.zeek @@ -21,5 +21,39 @@ export { ## String to use for empty fields. This should be different from ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; + + ## Values supported for SQLite's PRAGMA synchronous statement. + type SQLiteSynchronous: enum { + SQLITE_SYNCHRONOUS_DEFAULT, + SQLITE_SYNCHRONOUS_OFF, + SQLITE_SYNCHRONOUS_NORMAL, + SQLITE_SYNCHRONOUS_FULL, + SQLITE_SYNCHRONOUS_EXTRA, + }; + + ## Values supported for SQLite's PRAGMA journal_mode statement. + type SQLiteJournalMode: enum { + SQLITE_JOURNAL_MODE_DEFAULT, + SQLITE_JOURNAL_MODE_DELETE, + SQLITE_JOURNAL_MODE_TRUNCATE, + SQLITE_JOURNAL_MODE_PERSIST, + SQLITE_JOURNAL_MODE_MEMORY, + SQLITE_JOURNAL_MODE_WAL, + SQLITE_JOURNAL_MODE_OFF, + }; + + ## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous + ## statement with the provided value after connecting to the SQLite database. See + ## `SQLite's synchronous documentation `_ + ## for more details around performance and data safety trade offs. + const synchronous = SQLITE_SYNCHRONOUS_DEFAULT &redef; + + ## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA + ## journal_mode statement with the provided value after connecting to + ## the SQLite database. + ## `SQLite's journal_mode documentation `_ + ## for more details around performance, data safety trade offs + ## and interaction with the PRAGMA synchronous statement. + const journal_mode = SQLITE_JOURNAL_MODE_DEFAULT &redef; } diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index 21942d82ec..5186c14a5c 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -26,6 +26,9 @@ SQLite::SQLite(WriterFrontend* frontend) : WriterBackend(frontend), fields(), nu empty_field.assign((const char*)BifConst::LogSQLite::empty_field->Bytes(), BifConst::LogSQLite::empty_field->Len()); + synchronous = BifConst::LogSQLite::synchronous->AsInt(); + journal_mode = BifConst::LogSQLite::journal_mode->AsInt(); + threading::formatter::Ascii::SeparatorInfo sep_info(string(), set_separator, unset_field, empty_field); io = new threading::formatter::Ascii(this, sep_info); } @@ -128,6 +131,60 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) ) return false; + char* errorMsg = nullptr; + int res; + switch ( synchronous ) { + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break; + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_OFF: + res = sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_NORMAL: + res = sqlite3_exec(db, "PRAGMA synchronous=NORMAL;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_FULL: + res = sqlite3_exec(db, "PRAGMA synchronous=FULL;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_EXTRA: + res = sqlite3_exec(db, "PRAGMA synchronous=EXTRA;", NULL, NULL, &errorMsg); + break; + default: Error("Invalid LogSQLite::synchronous enum"); return false; + } + + if ( res != SQLITE_OK ) { + Error(Fmt("Error setting synchronous pragma: %s", errorMsg)); + sqlite3_free(errorMsg); + return false; + } + + switch ( journal_mode ) { + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_DEFAULT: res = SQLITE_OK; break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_DELETE: + res = sqlite3_exec(db, "PRAGMA journal_mode=DELETE;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_TRUNCATE: + res = sqlite3_exec(db, "PRAGMA journal_mode=TRUNCATE;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_PERSIST: + res = sqlite3_exec(db, "PRAGMA journal_mode=PERSIST;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_MEMORY: + res = sqlite3_exec(db, "PRAGMA journal_mode=MEMORY;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_WAL: + res = sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_OFF: + res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); + break; + default: Error("Invalid LogSQLite::journal_mode enum"); return false; + } + + if ( res != SQLITE_OK ) { + Error(Fmt("Error setting journal_mode pragma: %s", errorMsg)); + sqlite3_free(errorMsg); + return false; + } + string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n"; //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here. @@ -163,8 +220,8 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con create += "\n);"; - char* errorMsg = 0; - int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); + errorMsg = nullptr; + res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); if ( res != SQLITE_OK ) { Error(Fmt("Error executing table creation statement: %s", errorMsg)); sqlite3_free(errorMsg); diff --git a/src/logging/writers/sqlite/SQLite.h b/src/logging/writers/sqlite/SQLite.h index 15e1255730..5fd20bf45f 100644 --- a/src/logging/writers/sqlite/SQLite.h +++ b/src/logging/writers/sqlite/SQLite.h @@ -45,6 +45,9 @@ private: std::string unset_field; std::string empty_field; + int64_t synchronous; + int64_t journal_mode; + threading::formatter::Ascii* io; }; diff --git a/src/logging/writers/sqlite/sqlite.bif b/src/logging/writers/sqlite/sqlite.bif index 29b93f3a0c..4214c3f2d7 100644 --- a/src/logging/writers/sqlite/sqlite.bif +++ b/src/logging/writers/sqlite/sqlite.bif @@ -7,3 +7,23 @@ const set_separator: string; const empty_field: string; const unset_field: string; +enum SQLiteSynchronous %{ + SQLITE_SYNCHRONOUS_DEFAULT, + SQLITE_SYNCHRONOUS_OFF, + SQLITE_SYNCHRONOUS_NORMAL, + SQLITE_SYNCHRONOUS_FULL, + SQLITE_SYNCHRONOUS_EXTRA, +%} + +enum SQLiteJournalMode %{ + SQLITE_JOURNAL_MODE_DEFAULT, + SQLITE_JOURNAL_MODE_DELETE, + SQLITE_JOURNAL_MODE_TRUNCATE, + SQLITE_JOURNAL_MODE_PERSIST, + SQLITE_JOURNAL_MODE_MEMORY, + SQLITE_JOURNAL_MODE_WAL, + SQLITE_JOURNAL_MODE_OFF, +%} + +const synchronous: SQLiteSynchronous; +const journal_mode: SQLiteJournalMode; diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr new file mode 100644 index 0000000000..899e2ce169 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +end of stderr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results new file mode 100644 index 0000000000..9a6e979f75 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Should be delete +delete +Should be WAL +wal diff --git a/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek b/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek new file mode 100644 index 0000000000..985f3d1b80 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek @@ -0,0 +1,28 @@ +# This test exercises the SQLIte pragmas for synchronous and jornal_mode. +# Sadly, most of these do not have a way to test that they succeeded. So +# we mostly do the inverse test - if there are no error messages in .stderr +# everything should be fine. +# +# We can test for WAL journaling mode, as this persists and can be queried from +# the sqlite file. + +# @TEST-REQUIRES: which sqlite3 +# @TEST-REQUIRES: has-writer Zeek::SQLiteWriter +# @TEST-GROUP: sqlite +# +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_DELETE +# @TEST-EXEC: echo "Should be delete" > results +# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results +# @TEST-EXEC: rm http.sqlite +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_OFF LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_TRUNCATE +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_NORMAL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_PERSIST +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_MEMORY +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL +# @TEST-EXEC: echo "Should be WAL" >> results +# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_FULL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_OFF +# @TEST-EXEC: btest-diff results +# @TEST-EXEC: echo "end of stderr" >> .stderr # grep -v returns false on empty output +# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v tablename' btest-diff .stderr + +@load base/protocols/http