From 6e026ba31376aba4910aef5d223aa4c66cdc5357 Mon Sep 17 00:00:00 2001 From: Mymaqn Date: Thu, 24 Oct 2024 05:01:56 +0200 Subject: [PATCH 1/4] Support for synchronous and journal_mode --- .../frameworks/logging/writers/sqlite.zeek | 33 ++++++++++++ src/logging/writers/sqlite/SQLite.cc | 50 ++++++++++++++++++- src/logging/writers/sqlite/SQLite.h | 18 +++++++ src/logging/writers/sqlite/sqlite.bif | 20 ++++++++ 4 files changed, 119 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/logging/writers/sqlite.zeek b/scripts/base/frameworks/logging/writers/sqlite.zeek index f7ebd9130c..56c762cb61 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.zeek +++ b/scripts/base/frameworks/logging/writers/sqlite.zeek @@ -21,5 +21,38 @@ export { ## String to use for empty fields. This should be different from ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; + + type SQLiteSynchronous: enum { + SQLITE_SYNCHRONOUS_OFF, + SQLITE_SYNCHRONOUS_NORMAL, + SQLITE_SYNCHRONOUS_FULL, + SQLITE_SYNCHRONOUS_EXTRA, + }; + + ## Values supported for SQLite's PRAGMA journal_mode statement. + type SQLiteJournalMode: enum { + SQLITE_JOURNAL_MODE_DELETE, + SQLITE_JOURNAL_MODE_TRUNCATE, + SQLITE_JOURNAL_MODE_PERIST, + SQLITE_JOURNAL_MODE_MEMORY, + SQLITE_JOURNAL_MODE_WAL, + SQLITE_JOURNAL_MODE_OFF, + }; + + ## If set, runs the PRAGMA synchronous statement with the + ## provided value after connecting to the SQLite database. See + ## `SQLite's synchronous documentation `_ + ## for more details around performance and data safety trade offs. + + const synchronous = SQLITE_SYNCHRONOUS_FULL &redef; + + ## If set, runs the PRAGMA journal_mode statement with the + ## provided value after connecting to the SQLite database. See + ## `SQLite's journal_mode documentation `_ + ## for more details around performance, data safety trade offs + ## and interaction with the PRAGMA synchronous statement. + const journal_mode = SQLITE_JOURNAL_MODE_DELETE &redef; + + } diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index 21942d82ec..e7b1af313c 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -26,6 +26,9 @@ SQLite::SQLite(WriterFrontend* frontend) : WriterBackend(frontend), fields(), nu empty_field.assign((const char*)BifConst::LogSQLite::empty_field->Bytes(), BifConst::LogSQLite::empty_field->Len()); + synchronous = BifConst::LogSQLite::synchronous->AsInt(); + journal_mode = BifConst::LogSQLite::journal_mode->AsInt(); + threading::formatter::Ascii::SeparatorInfo sep_info(string(), set_separator, unset_field, empty_field); io = new threading::formatter::Ascii(this, sep_info); } @@ -128,6 +131,49 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) ) return false; + char* errorMsg = 0; + int res; + switch ( synchronous ) { + case SQLITE_SYNCHRONOUS_OFF: res = sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, &errorMsg); break; + case SQLITE_SYNCHRONOUS_NORMAL: + res = sqlite3_exec(db, "PRAGMA synchronous=NORMAL;", NULL, NULL, &errorMsg); + break; + case SQLITE_SYNCHRONOUS_FULL: res = sqlite3_exec(db, "PRAGMA synchronous=FULL;", NULL, NULL, &errorMsg); break; + case SQLITE_SYNCHRONOUS_EXTRA: + res = sqlite3_exec(db, "PRAGMA synchronous=EXTRA;", NULL, NULL, &errorMsg); + break; + default: Error("Invalid LogSQLite::synchronous enum"); return false; + } + if ( res != SQLITE_OK ) { + Error(Fmt("Error setting synchronous pragma %s", errorMsg)); + sqlite3_free(errorMsg); + return false; + } + + switch ( journal_mode ) { + case SQLITE_JOURNAL_MODE_DELETE: + res = sqlite3_exec(db, "PRAGMA journal_mode=DELETE;", NULL, NULL, &errorMsg); + break; + case SQLITE_JOURNAL_MODE_TRUNCATE: + res = sqlite3_exec(db, "PRAGMA journal_mode=TRUNCATE;", NULL, NULL, &errorMsg); + break; + case SQLITE_JOURNAL_MODE_PERSIST: + res = sqlite3_exec(db, "PRAGMA journal_mode=PERSIST;", NULL, NULL, &errorMsg); + break; + case SQLITE_JOURNAL_MODE_MEMORY: + res = sqlite3_exec(db, "PRAGMA journal_mode=MEMORY;", NULL, NULL, &errorMsg); + break; + case SQLITE_JOURNAL_MODE_WAL: res = sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, &errorMsg); break; + case SQLITE_JOURNAL_MODE_OFF: res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); break; + default: Error("Invalid LogSQLite::journal_mode enum"); return false; + } + if ( res != SQLITE_OK ) { + Error(Fmt("Error setting journal_mode pragma %s", errorMsg)); + sqlite3_free(errorMsg); + return false; + } + + string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n"; //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here. @@ -163,8 +209,8 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con create += "\n);"; - char* errorMsg = 0; - int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); + errorMsg = 0; + res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); if ( res != SQLITE_OK ) { Error(Fmt("Error executing table creation statement: %s", errorMsg)); sqlite3_free(errorMsg); diff --git a/src/logging/writers/sqlite/SQLite.h b/src/logging/writers/sqlite/SQLite.h index 15e1255730..61acdc28af 100644 --- a/src/logging/writers/sqlite/SQLite.h +++ b/src/logging/writers/sqlite/SQLite.h @@ -45,6 +45,24 @@ private: std::string unset_field; std::string empty_field; + int64_t synchronous; + int64_t journal_mode; + + enum SQLiteSynchronous { + SQLITE_SYNCHRONOUS_OFF, + SQLITE_SYNCHRONOUS_NORMAL, + SQLITE_SYNCHRONOUS_FULL, + SQLITE_SYNCHRONOUS_EXTRA, + }; + enum SQLiteJournalMode { + SQLITE_JOURNAL_MODE_DELETE, + SQLITE_JOURNAL_MODE_TRUNCATE, + SQLITE_JOURNAL_MODE_PERSIST, + SQLITE_JOURNAL_MODE_MEMORY, + SQLITE_JOURNAL_MODE_WAL, + SQLITE_JOURNAL_MODE_OFF, + }; + threading::formatter::Ascii* io; }; diff --git a/src/logging/writers/sqlite/sqlite.bif b/src/logging/writers/sqlite/sqlite.bif index 29b93f3a0c..7c68f2795e 100644 --- a/src/logging/writers/sqlite/sqlite.bif +++ b/src/logging/writers/sqlite/sqlite.bif @@ -7,3 +7,23 @@ const set_separator: string; const empty_field: string; const unset_field: string; +enum SQLiteSynchronous %{ + SQLITE_SYNCHRONOUS_OFF, + SQLITE_SYNCHRONOUS_NORMAL, + SQLITE_SYNCHRONOUS_FULL, + SQLITE_SYNCHRONOUS_EXTRA, +%} + +enum SQLiteJournalMode %{ + SQLITE_JOURNAL_MODE_DELETE, + SQLITE_JOURNAL_MODE_TRUNCATE, + SQLITE_JOURNAL_MODE_PERIST, + SQLITE_JOURNAL_MODE_MEMORY, + SQLITE_JOURNAL_MODE_WAL, + SQLITE_JOURNAL_MODE_OFF, +%} + +const synchronous: SQLiteSynchronous; +const journal_mode: SQLiteJournalMode; + + From 3ca56f7e0f74b82df83bc6d4bd87e32f9a7b1a0d Mon Sep 17 00:00:00 2001 From: Mymaqn Date: Wed, 30 Oct 2024 13:56:19 +0100 Subject: [PATCH 2/4] Added default options for synchronous and journal mode Added enum options SQLITE_SYNCHRONOUS_DEFAULT and SQLITE_JOURNAL_MODE_DEFAULT and changed the default to be these instead. --- .../base/frameworks/logging/writers/sqlite.zeek | 17 ++++++++++------- src/logging/writers/sqlite/SQLite.cc | 2 ++ src/logging/writers/sqlite/SQLite.h | 2 ++ src/logging/writers/sqlite/sqlite.bif | 4 +++- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/logging/writers/sqlite.zeek b/scripts/base/frameworks/logging/writers/sqlite.zeek index 56c762cb61..a813eb056a 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.zeek +++ b/scripts/base/frameworks/logging/writers/sqlite.zeek @@ -23,6 +23,7 @@ export { const empty_field = Log::empty_field &redef; type SQLiteSynchronous: enum { + SQLITE_SYNCHRONOUS_DEFAULT, SQLITE_SYNCHRONOUS_OFF, SQLITE_SYNCHRONOUS_NORMAL, SQLITE_SYNCHRONOUS_FULL, @@ -31,27 +32,29 @@ export { ## Values supported for SQLite's PRAGMA journal_mode statement. type SQLiteJournalMode: enum { + SQLITE_JOURNAL_MODE_DEFAULT, SQLITE_JOURNAL_MODE_DELETE, SQLITE_JOURNAL_MODE_TRUNCATE, - SQLITE_JOURNAL_MODE_PERIST, + SQLITE_JOURNAL_MODE_PERSIST, SQLITE_JOURNAL_MODE_MEMORY, SQLITE_JOURNAL_MODE_WAL, SQLITE_JOURNAL_MODE_OFF, }; - ## If set, runs the PRAGMA synchronous statement with the - ## provided value after connecting to the SQLite database. See + ## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous + ## statement with the provided value after connecting to the SQLite database. See ## `SQLite's synchronous documentation `_ ## for more details around performance and data safety trade offs. - const synchronous = SQLITE_SYNCHRONOUS_FULL &redef; + const synchronous = SQLITE_SYNCHRONOUS_DEFAULT &redef; - ## If set, runs the PRAGMA journal_mode statement with the - ## provided value after connecting to the SQLite database. See + ## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA + ## journal_mode statement with the provided value after connecting to + ## the SQLite database. ## `SQLite's journal_mode documentation `_ ## for more details around performance, data safety trade offs ## and interaction with the PRAGMA synchronous statement. - const journal_mode = SQLITE_JOURNAL_MODE_DELETE &redef; + const journal_mode = SQLITE_JOURNAL_MODE_DEFAULT &redef; } diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index e7b1af313c..f5f86fe415 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -134,6 +134,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con char* errorMsg = 0; int res; switch ( synchronous ) { + case SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break; case SQLITE_SYNCHRONOUS_OFF: res = sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, &errorMsg); break; case SQLITE_SYNCHRONOUS_NORMAL: res = sqlite3_exec(db, "PRAGMA synchronous=NORMAL;", NULL, NULL, &errorMsg); @@ -151,6 +152,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con } switch ( journal_mode ) { + case SQLITE_JOURNAL_MODE_DEFAULT: res = SQLITE_OK; break; case SQLITE_JOURNAL_MODE_DELETE: res = sqlite3_exec(db, "PRAGMA journal_mode=DELETE;", NULL, NULL, &errorMsg); break; diff --git a/src/logging/writers/sqlite/SQLite.h b/src/logging/writers/sqlite/SQLite.h index 61acdc28af..a7c2312f08 100644 --- a/src/logging/writers/sqlite/SQLite.h +++ b/src/logging/writers/sqlite/SQLite.h @@ -49,12 +49,14 @@ private: int64_t journal_mode; enum SQLiteSynchronous { + SQLITE_SYNCHRONOUS_DEFAULT, SQLITE_SYNCHRONOUS_OFF, SQLITE_SYNCHRONOUS_NORMAL, SQLITE_SYNCHRONOUS_FULL, SQLITE_SYNCHRONOUS_EXTRA, }; enum SQLiteJournalMode { + SQLITE_JOURNAL_MODE_DEFAULT, SQLITE_JOURNAL_MODE_DELETE, SQLITE_JOURNAL_MODE_TRUNCATE, SQLITE_JOURNAL_MODE_PERSIST, diff --git a/src/logging/writers/sqlite/sqlite.bif b/src/logging/writers/sqlite/sqlite.bif index 7c68f2795e..a95c4056e3 100644 --- a/src/logging/writers/sqlite/sqlite.bif +++ b/src/logging/writers/sqlite/sqlite.bif @@ -8,6 +8,7 @@ const empty_field: string; const unset_field: string; enum SQLiteSynchronous %{ + SQLITE_SYNCHRONOUS_DEFAULT, SQLITE_SYNCHRONOUS_OFF, SQLITE_SYNCHRONOUS_NORMAL, SQLITE_SYNCHRONOUS_FULL, @@ -15,9 +16,10 @@ enum SQLiteSynchronous %{ %} enum SQLiteJournalMode %{ + SQLITE_JOURNAL_MODE_DEFAULT, SQLITE_JOURNAL_MODE_DELETE, SQLITE_JOURNAL_MODE_TRUNCATE, - SQLITE_JOURNAL_MODE_PERIST, + SQLITE_JOURNAL_MODE_PERSIST, SQLITE_JOURNAL_MODE_MEMORY, SQLITE_JOURNAL_MODE_WAL, SQLITE_JOURNAL_MODE_OFF, From d592942ccba15470659587e72cf70eb0a5604c53 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 26 Nov 2024 12:26:38 +0000 Subject: [PATCH 3/4] Test synchronous/journal mode options for SQLite log writer Also adds some small tweaks and adds the new feature to NEWS. --- NEWS | 5 ++++ .../frameworks/logging/writers/sqlite.zeek | 16 +++++------ src/logging/writers/sqlite/SQLite.cc | 11 ++++---- src/logging/writers/sqlite/SQLite.h | 1 + src/logging/writers/sqlite/sqlite.bif | 2 -- .../.stderr | 2 ++ .../results | 5 ++++ .../frameworks/logging/sqlite/pragma.zeek | 28 +++++++++++++++++++ 8 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results create mode 100644 testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek diff --git a/NEWS b/NEWS index 8b5382f528..fa36d249f1 100644 --- a/NEWS +++ b/NEWS @@ -81,6 +81,11 @@ New Functionality of Zeek's AST after ZAM optimizations ran. This hook executes right before the ``zeek_init()`` event is enqueued. +* The SQLite logger now supports setting the value of the SQLite synchronous mode, + as well as of the journal mode. For example, WAL mode can be enabled by setting: + + redef LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL; + Changed Functionality --------------------- diff --git a/scripts/base/frameworks/logging/writers/sqlite.zeek b/scripts/base/frameworks/logging/writers/sqlite.zeek index a813eb056a..19253ccb53 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.zeek +++ b/scripts/base/frameworks/logging/writers/sqlite.zeek @@ -21,7 +21,8 @@ export { ## String to use for empty fields. This should be different from ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; - + + ## Values supported for SQLite's PRAGMA synchronous statement. type SQLiteSynchronous: enum { SQLITE_SYNCHRONOUS_DEFAULT, SQLITE_SYNCHRONOUS_OFF, @@ -40,22 +41,19 @@ export { SQLITE_JOURNAL_MODE_WAL, SQLITE_JOURNAL_MODE_OFF, }; - - ## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous + + ## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous ## statement with the provided value after connecting to the SQLite database. See ## `SQLite's synchronous documentation `_ ## for more details around performance and data safety trade offs. - const synchronous = SQLITE_SYNCHRONOUS_DEFAULT &redef; - - ## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA - ## journal_mode statement with the provided value after connecting to + + ## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA + ## journal_mode statement with the provided value after connecting to ## the SQLite database. ## `SQLite's journal_mode documentation `_ ## for more details around performance, data safety trade offs ## and interaction with the PRAGMA synchronous statement. const journal_mode = SQLITE_JOURNAL_MODE_DEFAULT &redef; - - } diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index f5f86fe415..12dc7ba3cc 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -131,7 +131,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) ) return false; - char* errorMsg = 0; + char* errorMsg = nullptr; int res; switch ( synchronous ) { case SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break; @@ -145,8 +145,9 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con break; default: Error("Invalid LogSQLite::synchronous enum"); return false; } + if ( res != SQLITE_OK ) { - Error(Fmt("Error setting synchronous pragma %s", errorMsg)); + Error(Fmt("Error setting synchronous pragma: %s", errorMsg)); sqlite3_free(errorMsg); return false; } @@ -169,13 +170,13 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con case SQLITE_JOURNAL_MODE_OFF: res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); break; default: Error("Invalid LogSQLite::journal_mode enum"); return false; } + if ( res != SQLITE_OK ) { - Error(Fmt("Error setting journal_mode pragma %s", errorMsg)); + Error(Fmt("Error setting journal_mode pragma: %s", errorMsg)); sqlite3_free(errorMsg); return false; } - string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n"; //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here. @@ -211,7 +212,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con create += "\n);"; - errorMsg = 0; + errorMsg = nullptr; res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); if ( res != SQLITE_OK ) { Error(Fmt("Error executing table creation statement: %s", errorMsg)); diff --git a/src/logging/writers/sqlite/SQLite.h b/src/logging/writers/sqlite/SQLite.h index a7c2312f08..be488cecbf 100644 --- a/src/logging/writers/sqlite/SQLite.h +++ b/src/logging/writers/sqlite/SQLite.h @@ -55,6 +55,7 @@ private: SQLITE_SYNCHRONOUS_FULL, SQLITE_SYNCHRONOUS_EXTRA, }; + enum SQLiteJournalMode { SQLITE_JOURNAL_MODE_DEFAULT, SQLITE_JOURNAL_MODE_DELETE, diff --git a/src/logging/writers/sqlite/sqlite.bif b/src/logging/writers/sqlite/sqlite.bif index a95c4056e3..4214c3f2d7 100644 --- a/src/logging/writers/sqlite/sqlite.bif +++ b/src/logging/writers/sqlite/sqlite.bif @@ -27,5 +27,3 @@ enum SQLiteJournalMode %{ const synchronous: SQLiteSynchronous; const journal_mode: SQLiteJournalMode; - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr new file mode 100644 index 0000000000..899e2ce169 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +end of stderr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results new file mode 100644 index 0000000000..9a6e979f75 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Should be delete +delete +Should be WAL +wal diff --git a/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek b/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek new file mode 100644 index 0000000000..3024b45f0a --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek @@ -0,0 +1,28 @@ +# This test exercises the SQLIte pragmas for synchronous and jornal_mode. +# Sadly, most of these do not have a way to test that they succeeded. So +# we mostly do the inverse test - if there are no error messages in .stderr +# everything should be fine. +# +# We can test for WAL journaling mode, as this persists and can be queried from +# the sqlite file. + +# @TEST-REQUIRES: which sqlite3 +# @TEST-REQUIRES: has-writer Zeek::SQLiteWriter +# @TEST-GROUP: sqlite +# +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_DELETE +# @TEST-EXEC: echo "Should be delete" > results +# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results +# @TEST-EXEC: rm http.sqlite +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_OFF LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_TRUNCATE +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_NORMAL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_PERSIST +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_MEMORY +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL +# @TEST-EXEC: echo "Should be WAL" >> results +# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_FULL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_OFF +# @TEST-EXEC: btest-diff results +# @TEST-EXEC: echo "end of stderr" >> .stderr # btest-diff does not like the canonifier to have an empty output +# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v tablename' btest-diff .stderr + +@load base/protocols/http From 35ad35ad0f7196d7d21ed561e6538463f802b927 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 26 Nov 2024 16:31:13 +0000 Subject: [PATCH 4/4] Options for SQLite log writer, eliminate duplicate definitions Patch provided by Arne Welzel, see GH-4063 --- src/logging/writers/sqlite/SQLite.cc | 32 +++++++++++++++++----------- src/logging/writers/sqlite/SQLite.h | 18 ---------------- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index 12dc7ba3cc..5186c14a5c 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -134,13 +134,17 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con char* errorMsg = nullptr; int res; switch ( synchronous ) { - case SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break; - case SQLITE_SYNCHRONOUS_OFF: res = sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, &errorMsg); break; - case SQLITE_SYNCHRONOUS_NORMAL: + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break; + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_OFF: + res = sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_NORMAL: res = sqlite3_exec(db, "PRAGMA synchronous=NORMAL;", NULL, NULL, &errorMsg); break; - case SQLITE_SYNCHRONOUS_FULL: res = sqlite3_exec(db, "PRAGMA synchronous=FULL;", NULL, NULL, &errorMsg); break; - case SQLITE_SYNCHRONOUS_EXTRA: + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_FULL: + res = sqlite3_exec(db, "PRAGMA synchronous=FULL;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_EXTRA: res = sqlite3_exec(db, "PRAGMA synchronous=EXTRA;", NULL, NULL, &errorMsg); break; default: Error("Invalid LogSQLite::synchronous enum"); return false; @@ -153,21 +157,25 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con } switch ( journal_mode ) { - case SQLITE_JOURNAL_MODE_DEFAULT: res = SQLITE_OK; break; - case SQLITE_JOURNAL_MODE_DELETE: + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_DEFAULT: res = SQLITE_OK; break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_DELETE: res = sqlite3_exec(db, "PRAGMA journal_mode=DELETE;", NULL, NULL, &errorMsg); break; - case SQLITE_JOURNAL_MODE_TRUNCATE: + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_TRUNCATE: res = sqlite3_exec(db, "PRAGMA journal_mode=TRUNCATE;", NULL, NULL, &errorMsg); break; - case SQLITE_JOURNAL_MODE_PERSIST: + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_PERSIST: res = sqlite3_exec(db, "PRAGMA journal_mode=PERSIST;", NULL, NULL, &errorMsg); break; - case SQLITE_JOURNAL_MODE_MEMORY: + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_MEMORY: res = sqlite3_exec(db, "PRAGMA journal_mode=MEMORY;", NULL, NULL, &errorMsg); break; - case SQLITE_JOURNAL_MODE_WAL: res = sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, &errorMsg); break; - case SQLITE_JOURNAL_MODE_OFF: res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_WAL: + res = sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, &errorMsg); + break; + case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_OFF: + res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); + break; default: Error("Invalid LogSQLite::journal_mode enum"); return false; } diff --git a/src/logging/writers/sqlite/SQLite.h b/src/logging/writers/sqlite/SQLite.h index be488cecbf..5fd20bf45f 100644 --- a/src/logging/writers/sqlite/SQLite.h +++ b/src/logging/writers/sqlite/SQLite.h @@ -48,24 +48,6 @@ private: int64_t synchronous; int64_t journal_mode; - enum SQLiteSynchronous { - SQLITE_SYNCHRONOUS_DEFAULT, - SQLITE_SYNCHRONOUS_OFF, - SQLITE_SYNCHRONOUS_NORMAL, - SQLITE_SYNCHRONOUS_FULL, - SQLITE_SYNCHRONOUS_EXTRA, - }; - - enum SQLiteJournalMode { - SQLITE_JOURNAL_MODE_DEFAULT, - SQLITE_JOURNAL_MODE_DELETE, - SQLITE_JOURNAL_MODE_TRUNCATE, - SQLITE_JOURNAL_MODE_PERSIST, - SQLITE_JOURNAL_MODE_MEMORY, - SQLITE_JOURNAL_MODE_WAL, - SQLITE_JOURNAL_MODE_OFF, - }; - threading::formatter::Ascii* io; };