SQLite: Add backend option for pragma timeout

scripts/policy/frameworks/storage/backend/sqlite/main.zeek

@@ -30,7 +30,17 @@ export {
 			["journal_mode"] = "WAL",
 			["synchronous"] = "normal",
 			["temp_store"] = "memory"
-		);
+		) &ordered;
+
+		## The total amount of time that an SQLite backend will spend attempting
+		## to run an individual pragma command before giving up and returning an
+		## initialization error. Setting this to zero will result in the backend
+		## attempting forever until success.
+		pragma_timeout: interval &default=500 msec;
+
+		## The amount of time that at SQLite backend will wait between failures
+		## to run an individual pragma command.
+		pragma_wait_on_busy: interval &default=5 msec;
 	};
 }
 

src/storage/backend/sqlite/SQLite.cc

@@ -9,26 +9,29 @@
 #include "zeek/Val.h"
 #include "zeek/storage/ReturnCode.h"
 
+using namespace std::chrono_literals;
+
 namespace zeek::storage::backend::sqlite {
 
 OperationResult SQLite::RunPragma(std::string_view name, std::optional<std::string_view> value) {
     char* errorMsg = nullptr;
+    std::chrono::milliseconds time_spent = 0ms;
 
     std::string cmd = util::fmt("pragma %.*s", static_cast<int>(name.size()), name.data());
     if ( value && ! value->empty() )
         cmd += util::fmt(" = %.*s", static_cast<int>(value->size()), value->data());
 
-    while ( attempts < 5 ) {
+    while ( pragma_timeout == 0ms || time_spent < pragma_timeout ) {
         int res = sqlite3_exec(db, cmd.c_str(), NULL, NULL, &errorMsg);
         if ( res == SQLITE_OK ) {
             break;
         }
         else if ( res == SQLITE_BUSY ) {
             // If we got back that the database is busy, it likely means that another process is trying to
-            // do their pragmas at startup too. Sleep for a little bit and try again.
+            // do their pragmas at startup too. Exponentially back off and try again after a sleep.
             sqlite3_free(errorMsg);
-            std::this_thread::sleep_for(std::chrono::milliseconds(100));
+            std::this_thread::sleep_for(pragma_wait_on_busy);
-            ++attempts;
+            time_spent += pragma_wait_on_busy;
         }
         else {
             std::string err = util::fmt("Error while executing pragma '%s': %s", cmd.c_str(), errorMsg);
@@ -37,7 +40,7 @@ OperationResult SQLite::RunPragma(std::string_view name, std::optional<std::stri
         }
     }
 
-    if ( attempts == 5 ) {
+    if ( pragma_timeout != 0ms && time_spent >= pragma_timeout ) {
         std::string err =
             util::fmt("Database was busy while executing %.*s pragma", static_cast<int>(name.size()), name.data());
         return {ReturnCode::INITIALIZATION_FAILED, std::move(err)};
@@ -71,6 +74,12 @@ OperationResult SQLite::DoOpen(OpenResultCallback* cb, RecordValPtr options) {
     full_path = zeek::filesystem::path(path->ToStdString()).string();
     table_name = backend_options->GetField<StringVal>("table_name")->ToStdString();
 
+    auto pragma_timeout_val = backend_options->GetField<IntervalVal>("pragma_timeout");
+    pragma_timeout = std::chrono::milliseconds(static_cast<int64_t>(pragma_timeout_val->Get() * 1000));
+
+    auto pragma_wof_val = backend_options->GetField<IntervalVal>("pragma_wait_on_busy");
+    pragma_wait_on_busy = std::chrono::milliseconds(static_cast<int64_t>(pragma_timeout_val->Get() * 1000));
+
     if ( auto open_res =
              CheckError(sqlite3_open_v2(full_path.c_str(), &db,
                                         SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL));

src/storage/backend/sqlite/SQLite.h

@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include <chrono>
+
 #include "zeek/storage/Backend.h"
 
 // Forward declare these to avoid including sqlite3.h here
@@ -62,6 +64,8 @@ private:
 
     std::string full_path;
     std::string table_name;
+    std::chrono::milliseconds pragma_timeout;
+    std::chrono::milliseconds pragma_wait_on_busy;
 };
 
 } // namespace zeek::storage::backend::sqlite

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out

@@ -4,7 +4,7 @@ Storage::backend_opened, Storage::STORAGE_BACKEND_SQLITE, [serializer=Storage::S
 [journal_mode] = WAL,
 [temp_store] = memory,
 [busy_timeout] = 5000
-}]]
+}, pragma_timeout=500.0 msecs, pragma_wait_on_failure=5.0 msecs]]
 open result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>]
 put result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<uninitialized>]
 get result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=value5678]