Merge remote-tracking branch 'origin/topic/johanna/sqlite-pragmas'

* origin/topic/johanna/sqlite-pragmas:
  Options for SQLite log writer, eliminate duplicate definitions
  Test synchronous/journal mode options for SQLite log writer
  Added default options for synchronous and journal mode
  Support for synchronous and journal_mode

CHANGES

@@ -1,3 +1,11 @@
+7.1.0-dev.635 | 2024-11-27 08:29:19 +0000
+
+  * Support setting the synchronous flag and the journal_mode for the SQLite log writer.
+
+    These options allow for a significant speed-up of SQLite logging, at the expense
+    of some data security. To change the default settings, redef the LogSQLite::synchronous
+    and LogSQLite::journal_mode consts. (Mymaqn)
+
 7.1.0-dev.629 | 2024-11-26 17:45:08 +0100
 
   * ci/test.sh: Run doctest with TZ=UTC (Arne Welzel, Corelight)

NEWS

@@ -81,6 +81,11 @@ New Functionality
   of Zeek's AST after ZAM optimizations ran. This hook executes right before
   the ``zeek_init()`` event is enqueued.
 
+* The SQLite logger now supports setting the value of the SQLite synchronous mode,
+  as well as of the journal mode. For example, WAL mode can be enabled by setting:
+
+  redef LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL;
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-7.1.0-dev.629
+7.1.0-dev.635

scripts/base/frameworks/logging/writers/sqlite.zeek

@@ -21,5 +21,39 @@ export {
 	## String to use for empty fields. This should be different from
 	## *unset_field* to make the output unambiguous.
 	const empty_field = Log::empty_field &redef;
+
+	## Values supported for SQLite's PRAGMA synchronous statement.
+	type SQLiteSynchronous: enum {
+		SQLITE_SYNCHRONOUS_DEFAULT,
+		SQLITE_SYNCHRONOUS_OFF,
+		SQLITE_SYNCHRONOUS_NORMAL,
+		SQLITE_SYNCHRONOUS_FULL,
+		SQLITE_SYNCHRONOUS_EXTRA,
+	};
+
+	## Values supported for SQLite's PRAGMA journal_mode statement.
+	type SQLiteJournalMode: enum {
+		SQLITE_JOURNAL_MODE_DEFAULT,
+		SQLITE_JOURNAL_MODE_DELETE,
+		SQLITE_JOURNAL_MODE_TRUNCATE,
+		SQLITE_JOURNAL_MODE_PERSIST,
+		SQLITE_JOURNAL_MODE_MEMORY,
+		SQLITE_JOURNAL_MODE_WAL,
+		SQLITE_JOURNAL_MODE_OFF,
+	};
+
+	## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous
+	## statement with the provided value after connecting to the SQLite database. See
+	## `SQLite's synchronous documentation <https://www.sqlite.org/pragma.html#pragma_synchronous>`_
+	## for more details around performance and data safety trade offs.
+	const synchronous = SQLITE_SYNCHRONOUS_DEFAULT &redef;
+
+	## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA
+	## journal_mode statement with the provided value after connecting to
+	## the SQLite database.
+	## `SQLite's journal_mode documentation <https://www.sqlite.org/pragma.html#pragma_journal_mode>`_
+	## for more details around performance, data safety trade offs
+	## and interaction with the PRAGMA synchronous statement.
+	const journal_mode = SQLITE_JOURNAL_MODE_DEFAULT &redef;
 }
 

src/logging/writers/sqlite/SQLite.cc

@@ -26,6 +26,9 @@ SQLite::SQLite(WriterFrontend* frontend) : WriterBackend(frontend), fields(), nu
 
     empty_field.assign((const char*)BifConst::LogSQLite::empty_field->Bytes(), BifConst::LogSQLite::empty_field->Len());
 
+    synchronous = BifConst::LogSQLite::synchronous->AsInt();
+    journal_mode = BifConst::LogSQLite::journal_mode->AsInt();
+
     threading::formatter::Ascii::SeparatorInfo sep_info(string(), set_separator, unset_field, empty_field);
     io = new threading::formatter::Ascii(this, sep_info);
 }
@@ -128,6 +131,60 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
                                     SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) )
         return false;
 
+    char* errorMsg = nullptr;
+    int res;
+    switch ( synchronous ) {
+        case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break;
+        case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_OFF:
+            res = sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_NORMAL:
+            res = sqlite3_exec(db, "PRAGMA synchronous=NORMAL;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_FULL:
+            res = sqlite3_exec(db, "PRAGMA synchronous=FULL;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteSynchronous::SQLITE_SYNCHRONOUS_EXTRA:
+            res = sqlite3_exec(db, "PRAGMA synchronous=EXTRA;", NULL, NULL, &errorMsg);
+            break;
+        default: Error("Invalid LogSQLite::synchronous enum"); return false;
+    }
+
+    if ( res != SQLITE_OK ) {
+        Error(Fmt("Error setting synchronous pragma: %s", errorMsg));
+        sqlite3_free(errorMsg);
+        return false;
+    }
+
+    switch ( journal_mode ) {
+        case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_DEFAULT: res = SQLITE_OK; break;
+        case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_DELETE:
+            res = sqlite3_exec(db, "PRAGMA journal_mode=DELETE;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_TRUNCATE:
+            res = sqlite3_exec(db, "PRAGMA journal_mode=TRUNCATE;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_PERSIST:
+            res = sqlite3_exec(db, "PRAGMA journal_mode=PERSIST;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_MEMORY:
+            res = sqlite3_exec(db, "PRAGMA journal_mode=MEMORY;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_WAL:
+            res = sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, &errorMsg);
+            break;
+        case BifEnum::LogSQLite::SQLiteJournalMode::SQLITE_JOURNAL_MODE_OFF:
+            res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg);
+            break;
+        default: Error("Invalid LogSQLite::journal_mode enum"); return false;
+    }
+
+    if ( res != SQLITE_OK ) {
+        Error(Fmt("Error setting journal_mode pragma: %s", errorMsg));
+        sqlite3_free(errorMsg);
+        return false;
+    }
+
     string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n";
     //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here.
 
@@ -163,8 +220,8 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
 
     create += "\n);";
 
-    char* errorMsg = 0;
+    errorMsg = nullptr;
-    int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg);
+    res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg);
     if ( res != SQLITE_OK ) {
         Error(Fmt("Error executing table creation statement: %s", errorMsg));
         sqlite3_free(errorMsg);

src/logging/writers/sqlite/SQLite.h

@@ -45,6 +45,9 @@ private:
     std::string unset_field;
     std::string empty_field;
 
+    int64_t synchronous;
+    int64_t journal_mode;
+
     threading::formatter::Ascii* io;
 };
 

src/logging/writers/sqlite/sqlite.bif

@@ -7,3 +7,23 @@ const set_separator: string;
 const empty_field: string;
 const unset_field: string;
 
+enum SQLiteSynchronous %{
+	SQLITE_SYNCHRONOUS_DEFAULT,
+	SQLITE_SYNCHRONOUS_OFF,
+	SQLITE_SYNCHRONOUS_NORMAL,
+	SQLITE_SYNCHRONOUS_FULL,
+	SQLITE_SYNCHRONOUS_EXTRA,
+%}
+
+enum SQLiteJournalMode %{
+	SQLITE_JOURNAL_MODE_DEFAULT,
+	SQLITE_JOURNAL_MODE_DELETE,
+	SQLITE_JOURNAL_MODE_TRUNCATE,
+	SQLITE_JOURNAL_MODE_PERSIST,
+	SQLITE_JOURNAL_MODE_MEMORY,
+	SQLITE_JOURNAL_MODE_WAL,
+	SQLITE_JOURNAL_MODE_OFF,
+%}
+
+const synchronous: SQLiteSynchronous;
+const journal_mode: SQLiteJournalMode;

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+end of stderr

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Should be delete
+delete
+Should be WAL
+wal

testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek

@@ -0,0 +1,28 @@
+# This test exercises the SQLIte pragmas for synchronous and jornal_mode.
+# Sadly, most of these do not have a way to test that they succeeded. So
+# we mostly do the inverse test - if there are no error messages in .stderr
+# everything should be fine.
+#
+# We can test for WAL journaling mode, as this persists and can be queried from
+# the sqlite file.
+
+# @TEST-REQUIRES: which sqlite3
+# @TEST-REQUIRES: has-writer Zeek::SQLiteWriter
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_DELETE
+# @TEST-EXEC: echo "Should be delete" > results
+# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results
+# @TEST-EXEC: rm http.sqlite
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_OFF LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_TRUNCATE
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_NORMAL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_PERSIST
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_MEMORY
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL
+# @TEST-EXEC: echo "Should be WAL" >> results
+# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_FULL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_OFF
+# @TEST-EXEC: btest-diff results
+# @TEST-EXEC: echo "end of stderr" >> .stderr # grep -v returns false on empty output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v tablename' btest-diff .stderr
+
+@load base/protocols/http