Test synchronous/journal mode options for SQLite log writer

Also adds some small tweaks and adds the new feature to NEWS.

NEWS

@@ -81,6 +81,11 @@ New Functionality
   of Zeek's AST after ZAM optimizations ran. This hook executes right before
   the ``zeek_init()`` event is enqueued.
 
+* The SQLite logger now supports setting the value of the SQLite synchronous mode,
+  as well as of the journal mode. For example, WAL mode can be enabled by setting:
+
+  redef LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL;
+
 Changed Functionality
 ---------------------
 

scripts/base/frameworks/logging/writers/sqlite.zeek

@@ -21,7 +21,8 @@ export {
 	## String to use for empty fields. This should be different from
 	## *unset_field* to make the output unambiguous.
 	const empty_field = Log::empty_field &redef;
-	
+
+	## Values supported for SQLite's PRAGMA synchronous statement.
 	type SQLiteSynchronous: enum {
 		SQLITE_SYNCHRONOUS_DEFAULT,
 		SQLITE_SYNCHRONOUS_OFF,
@@ -40,22 +41,19 @@ export {
 		SQLITE_JOURNAL_MODE_WAL,
 		SQLITE_JOURNAL_MODE_OFF,
 	};
-	
+
-	## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous 
+	## If changed from SQLITE_SYNCHRONOUS_DEFAULT, runs the PRAGMA synchronous
 	## statement with the provided value after connecting to the SQLite database. See
 	## `SQLite's synchronous documentation <https://www.sqlite.org/pragma.html#pragma_synchronous>`_
 	## for more details around performance and data safety trade offs.
-	
 	const synchronous = SQLITE_SYNCHRONOUS_DEFAULT &redef;
-	
+
-	## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA 
+	## If changed from SQLITE_JOURNAL_MODE_DEFAULT, runs the PRAGMA
-	## journal_mode statement with the provided value after connecting to 
+	## journal_mode statement with the provided value after connecting to
 	## the SQLite database.
 	## `SQLite's journal_mode documentation <https://www.sqlite.org/pragma.html#pragma_journal_mode>`_
 	## for more details around performance, data safety trade offs
 	## and interaction with the PRAGMA synchronous statement.
 	const journal_mode = SQLITE_JOURNAL_MODE_DEFAULT &redef;
-
-	
 }
 

src/logging/writers/sqlite/SQLite.cc

@@ -131,7 +131,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
                                     SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) )
         return false;
 
-    char* errorMsg = 0;
+    char* errorMsg = nullptr;
     int res;
     switch ( synchronous ) {
         case SQLITE_SYNCHRONOUS_DEFAULT: res = SQLITE_OK; break;
@@ -145,8 +145,9 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
             break;
         default: Error("Invalid LogSQLite::synchronous enum"); return false;
     }
+
     if ( res != SQLITE_OK ) {
-        Error(Fmt("Error setting synchronous pragma %s", errorMsg));
+        Error(Fmt("Error setting synchronous pragma: %s", errorMsg));
         sqlite3_free(errorMsg);
         return false;
     }
@@ -169,13 +170,13 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
         case SQLITE_JOURNAL_MODE_OFF: res = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, &errorMsg); break;
         default: Error("Invalid LogSQLite::journal_mode enum"); return false;
     }
+
     if ( res != SQLITE_OK ) {
-        Error(Fmt("Error setting journal_mode pragma %s", errorMsg));
+        Error(Fmt("Error setting journal_mode pragma: %s", errorMsg));
         sqlite3_free(errorMsg);
         return false;
     }
 
-
     string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n";
     //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here.
 
@@ -211,7 +212,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con
 
     create += "\n);";
 
-    errorMsg = 0;
+    errorMsg = nullptr;
     res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg);
     if ( res != SQLITE_OK ) {
         Error(Fmt("Error executing table creation statement: %s", errorMsg));

src/logging/writers/sqlite/SQLite.h

@@ -55,6 +55,7 @@ private:
         SQLITE_SYNCHRONOUS_FULL,
         SQLITE_SYNCHRONOUS_EXTRA,
     };
+
     enum SQLiteJournalMode {
         SQLITE_JOURNAL_MODE_DEFAULT,
         SQLITE_JOURNAL_MODE_DELETE,

src/logging/writers/sqlite/sqlite.bif

@@ -27,5 +27,3 @@ enum SQLiteJournalMode %{
 
 const synchronous: SQLiteSynchronous;
 const journal_mode: SQLiteJournalMode;
-
-

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/.stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+end of stderr

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.pragma/results

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Should be delete
+delete
+Should be WAL
+wal

testing/btest/scripts/base/frameworks/logging/sqlite/pragma.zeek

@@ -0,0 +1,28 @@
+# This test exercises the SQLIte pragmas for synchronous and jornal_mode.
+# Sadly, most of these do not have a way to test that they succeeded. So
+# we mostly do the inverse test - if there are no error messages in .stderr
+# everything should be fine.
+#
+# We can test for WAL journaling mode, as this persists and can be queried from
+# the sqlite file.
+
+# @TEST-REQUIRES: which sqlite3
+# @TEST-REQUIRES: has-writer Zeek::SQLiteWriter
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_DELETE
+# @TEST-EXEC: echo "Should be delete" > results
+# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results
+# @TEST-EXEC: rm http.sqlite
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_OFF LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_TRUNCATE
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_NORMAL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_PERSIST
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_MEMORY
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_EXTRA LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL
+# @TEST-EXEC: echo "Should be WAL" >> results
+# @TEST-EXEC: sqlite3 http.sqlite "PRAGMA journal_mode" >> results
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_SQLITE LogSQLite::synchronous=LogSQLite::SQLITE_SYNCHRONOUS_FULL LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_OFF
+# @TEST-EXEC: btest-diff results
+# @TEST-EXEC: echo "end of stderr" >> .stderr # btest-diff does not like the canonifier to have an empty output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v tablename' btest-diff .stderr
+
+@load base/protocols/http