diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek index 90a512e454..5dba77b20a 100644 --- a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -30,7 +30,17 @@ export { ["journal_mode"] = "WAL", ["synchronous"] = "normal", ["temp_store"] = "memory" - ); + ) &ordered; + + ## The total amount of time that an SQLite backend will spend attempting + ## to run an individual pragma command before giving up and returning an + ## initialization error. Setting this to zero will result in the backend + ## attempting forever until success. + pragma_timeout: interval &default=500 msec; + + ## The amount of time that at SQLite backend will wait between failures + ## to run an individual pragma command. + pragma_wait_on_busy: interval &default=5 msec; }; } diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index a852138824..4d007fa5c4 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -9,26 +9,29 @@ #include "zeek/Val.h" #include "zeek/storage/ReturnCode.h" +using namespace std::chrono_literals; + namespace zeek::storage::backend::sqlite { OperationResult SQLite::RunPragma(std::string_view name, std::optional value) { char* errorMsg = nullptr; + std::chrono::milliseconds time_spent = 0ms; std::string cmd = util::fmt("pragma %.*s", static_cast(name.size()), name.data()); if ( value && ! value->empty() ) cmd += util::fmt(" = %.*s", static_cast(value->size()), value->data()); - while ( attempts < 5 ) { + while ( pragma_timeout == 0ms || time_spent < pragma_timeout ) { int res = sqlite3_exec(db, cmd.c_str(), NULL, NULL, &errorMsg); if ( res == SQLITE_OK ) { break; } else if ( res == SQLITE_BUSY ) { // If we got back that the database is busy, it likely means that another process is trying to - // do their pragmas at startup too. Sleep for a little bit and try again. + // do their pragmas at startup too. Exponentially back off and try again after a sleep. sqlite3_free(errorMsg); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - ++attempts; + std::this_thread::sleep_for(pragma_wait_on_busy); + time_spent += pragma_wait_on_busy; } else { std::string err = util::fmt("Error while executing pragma '%s': %s", cmd.c_str(), errorMsg); @@ -37,7 +40,7 @@ OperationResult SQLite::RunPragma(std::string_view name, std::optional= pragma_timeout ) { std::string err = util::fmt("Database was busy while executing %.*s pragma", static_cast(name.size()), name.data()); return {ReturnCode::INITIALIZATION_FAILED, std::move(err)}; @@ -71,6 +74,12 @@ OperationResult SQLite::DoOpen(OpenResultCallback* cb, RecordValPtr options) { full_path = zeek::filesystem::path(path->ToStdString()).string(); table_name = backend_options->GetField("table_name")->ToStdString(); + auto pragma_timeout_val = backend_options->GetField("pragma_timeout"); + pragma_timeout = std::chrono::milliseconds(static_cast(pragma_timeout_val->Get() * 1000)); + + auto pragma_wof_val = backend_options->GetField("pragma_wait_on_busy"); + pragma_wait_on_busy = std::chrono::milliseconds(static_cast(pragma_timeout_val->Get() * 1000)); + if ( auto open_res = CheckError(sqlite3_open_v2(full_path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL)); diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 598db99a3f..a5db127fb7 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -2,6 +2,8 @@ #pragma once +#include + #include "zeek/storage/Backend.h" // Forward declare these to avoid including sqlite3.h here @@ -62,6 +64,8 @@ private: std::string full_path; std::string table_name; + std::chrono::milliseconds pragma_timeout; + std::chrono::milliseconds pragma_wait_on_busy; }; } // namespace zeek::storage::backend::sqlite diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index d0934b75b8..3052fe3a40 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -4,7 +4,7 @@ Storage::backend_opened, Storage::STORAGE_BACKEND_SQLITE, [serializer=Storage::S [journal_mode] = WAL, [temp_store] = memory, [busy_timeout] = 5000 -}]] +}, pragma_timeout=500.0 msecs, pragma_wait_on_failure=5.0 msecs]] open result, [code=Storage::SUCCESS, error_str=, value=] put result, [code=Storage::SUCCESS, error_str=, value=] get result, [code=Storage::SUCCESS, error_str=, value=value5678]