diff --git a/NEWS b/NEWS index 8b5382f528..fa36d249f1 100644 --- a/NEWS +++ b/NEWS @@ -81,6 +81,11 @@ New Functionality of Zeek's AST after ZAM optimizations ran. This hook executes right before the ``zeek_init()`` event is enqueued. +* The SQLite logger now supports setting the value of the SQLite synchronous mode, + as well as of the journal mode. For example, WAL mode can be enabled by setting: + + redef LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL; + Changed Functionality --------------------- diff --git a/scripts/base/frameworks/logging/writers/sqlite.zeek b/scripts/base/frameworks/logging/writers/sqlite.zeek index a813eb056a..19253ccb53 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.zeek +++ b/scripts/base/frameworks/logging/writers/sqlite.zeek @@ -21,7 +21,8 @@ export { ## String to use for empty fields. This should be different from ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; - + + ## Values supported for SQLite's PRAGMA synchronous statement. type SQLiteSynchronous: enum { SQLITE_SYNCHRONOUS_DEFAULT, SQLITE_SYNCHRONOUS_OFF, @@ -40,22 +41,19 @@ export { SQLITE_JOURNAL_MODE_WAL, SQLITE_JOURNAL_MODE_OFF, }; - - ## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous + + ## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous ## statement with the provided value after connecting to the SQLite database. See ## `SQLite's synchronous documentation `_ ## for more details around performance and data safety trade offs. - const synchronous = SQLITE_SYNCHRONOUS_DEFAULT &redef; - - ## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA - ## journal_mode statement with the provided value after connecting to + + ## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA + ## journal_mode statement with the provided value after connecting to ## the SQLite database. ## `SQLite's journal_mode documentation `_ ## for more details around performance, data safety trade offs ## and interaction with the PRAGMA synchronous statement. const journal_mode = SQLITE_JOURNAL_MODE_DEFAULT &redef; - - } diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index f5f86fe415..12dc7ba3cc 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -131,7 +131,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) ) return false; - char* errorMsg = 0; + char* errorMsg = nullptr; int res; switch ( synchronous ) { case SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break; @@ -145,8 +145,9 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con break; default: Error("Invalid LogSQLite::synchronous enum"); return false; } + if ( res != SQLITE_OK ) { - Error(Fmt("Error setting synchronous pragma %s", errorMsg)); + Error(Fmt("Error setting synchronous pragma: %s", errorMsg)); sqlite3_free(errorMsg); return false; } @@ -169,13 +170,13 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con case SQLITE_JOURNAL_MODE_OFF: res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); break; default: Error("Invalid LogSQLite::journal_mode enum"); return false; } + if ( res != SQLITE_OK ) { - Error(Fmt("Error setting journal_mode pragma %s", errorMsg)); + Error(Fmt("Error setting journal_mode pragma: %s", errorMsg)); sqlite3_free(errorMsg); return false; } - string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n"; //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here. @@ -211,7 +212,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con create += "\n);"; - errorMsg = 0; + errorMsg = nullptr; res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); if ( res != SQLITE_OK ) { Error(Fmt("Error executing table creation statement: %s", errorMsg)); diff --git a/src/logging/writers/sqlite/SQLite.h b/src/logging/writers/sqlite/SQLite.h index a7c2312f08..be488cecbf 100644 --- a/src/logging/writers/sqlite/SQLite.h +++ b/src/logging/writers/sqlite/SQLite.h @@ -55,6 +55,7 @@ private: SQLITE_SYNCHRONOUS_FULL, SQLITE_SYNCHRONOUS_EXTRA, }; + enum SQLiteJournalMode { SQLITE_JOURNAL_MODE_DEFAULT, SQLITE_JOURNAL_MODE_DELETE, diff --git a/src/logging/writers/sqlite/sqlite.bif b/src/logging/writers/sqlite/sqlite.bif index a95c4056e3..4214c3f2d7 100644 --- a/src/logging/writers/sqlite/sqlite.bif +++ b/src/logging/writers/sqlite/sqlite.bif @@ -27,5 +27,3 @@ enum SQLiteJournalMode %{ const synchronous: SQLiteSynchronous; const journal_mode: SQLiteJournalMode; - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr new file mode 100644 index 0000000000..899e2ce169 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +end of stderr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results new file mode 100644 index 0000000000..9a6e979f75 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Should be delete +delete +Should be WAL +wal diff --git a/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek b/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek new file mode 100644 index 0000000000..3024b45f0a --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek @@ -0,0 +1,28 @@ +# This test exercises the SQLIte pragmas for synchronous and jornal_mode. +# Sadly, most of these do not have a way to test that they succeeded. So +# we mostly do the inverse test - if there are no error messages in .stderr +# everything should be fine. +# +# We can test for WAL journaling mode, as this persists and can be queried from +# the sqlite file. + +# @TEST-REQUIRES: which sqlite3 +# @TEST-REQUIRES: has-writer Zeek::SQLiteWriter +# @TEST-GROUP: sqlite +# +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_DELETE +# @TEST-EXEC: echo "Should be delete" > results +# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results +# @TEST-EXEC: rm http.sqlite +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_OFF LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_TRUNCATE +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_NORMAL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_PERSIST +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_MEMORY +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL +# @TEST-EXEC: echo "Should be WAL" >> results +# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_FULL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_OFF +# @TEST-EXEC: btest-diff results +# @TEST-EXEC: echo "end of stderr" >> .stderr # btest-diff does not like the canonifier to have an empty output +# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v tablename' btest-diff .stderr + +@load base/protocols/http