Add basic SQLite storage backend

scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek

@@ -0,0 +1 @@
+@load ./main.zeek

scripts/policy/frameworks/storage/backend/sqlite/main.zeek

@@ -0,0 +1,22 @@
+##! SQLite storage backend support
+
+@load base/frameworks/storage/main
+
+module Storage::Backend::SQLite;
+
+export {
+	## Options record for the built-in SQLite backend.
+	type Options: record {
+		## Path to the database file on disk. Setting this to ":memory:"
+		## will tell SQLite to use an in-memory database. Relative paths
+		## will be opened relative to the directory where Zeek was
+		## started from. Zeek will not create intermediate directories
+		## if they do not already exist. See
+		## https://www.sqlite.org/c3ref/open.html for more rules on
+		## paths that can be passed here.
+		database_path: string;
+
+		## Name of the table used for storing data.
+		table_name: string;
+	};
+}

scripts/test-all-policy.zeek

@@ -83,6 +83,8 @@
 # @load frameworks/spicy/record-spicy-batch.zeek
 # @load frameworks/spicy/resource-usage.zeek
 @load frameworks/software/windows-version-detection.zeek
+@load frameworks/storage/backend/sqlite/__load__.zeek
+@load frameworks/storage/backend/sqlite/main.zeek
 @load frameworks/telemetry/log.zeek
 @load integration/collective-intel/__load__.zeek
 @load integration/collective-intel/main.zeek

src/storage/backend/CMakeLists.txt

@@ -1 +1 @@
-
+add_subdirectory(sqlite)

src/storage/backend/sqlite/CMakeLists.txt

@@ -0,0 +1,3 @@
+zeek_add_plugin(
+    Zeek Storage_Backend_SQLite
+    SOURCES SQLite.cc Plugin.cc)

src/storage/backend/sqlite/Plugin.cc

@@ -0,0 +1,22 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/plugin/Plugin.h"
+
+#include "zeek/storage/Component.h"
+#include "zeek/storage/backend/sqlite/SQLite.h"
+
+namespace zeek::storage::backend::sqlite {
+
+class Plugin : public plugin::Plugin {
+public:
+    plugin::Configuration Configure() override {
+        AddComponent(new storage::Component("SQLITE", backend::sqlite::SQLite::Instantiate));
+
+        plugin::Configuration config;
+        config.name = "Zeek::Storage_Backend_SQLite";
+        config.description = "SQLite backend for storage framework";
+        return config;
+    }
+} plugin;
+
+} // namespace zeek::storage::backend::sqlite

src/storage/backend/sqlite/SQLite.cc

@@ -0,0 +1,176 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/storage/backend/sqlite/SQLite.h"
+
+#include "zeek/3rdparty/sqlite3.h"
+#include "zeek/Func.h"
+#include "zeek/Val.h"
+
+namespace zeek::storage::backend::sqlite {
+
+storage::BackendPtr SQLite::Instantiate(std::string_view tag) { return make_intrusive<SQLite>(tag); }
+
+/**
+ * Called by the manager system to open the backend.
+ */
+ErrorResult SQLite::DoOpen(RecordValPtr options) {
+    if ( sqlite3_threadsafe() == 0 ) {
+        std::string res =
+            "SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of "
+            "SQLite. Aborting";
+        Error(res.c_str());
+        return res;
+    }
+
+    // Allow connections to same DB to use single data/schema cache. Also
+    // allows simultaneous writes to one file.
+#ifndef ZEEK_TSAN
+    sqlite3_enable_shared_cache(1);
+#endif
+
+    StringValPtr path = options->GetField<StringVal>("database_path");
+    full_path = zeek::filesystem::path(path->ToStdString()).string();
+    table_name = options->GetField<StringVal>("table_name")->ToStdString();
+
+    auto open_res =
+        checkError(sqlite3_open_v2(full_path.c_str(), &db,
+                                   SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL));
+    if ( open_res.has_value() ) {
+        sqlite3_close_v2(db);
+        db = nullptr;
+        return open_res;
+    }
+
+    std::string create = "create table if not exists " + table_name + " (";
+    create.append("key_str text primary key, value_str text not null);");
+
+    char* errorMsg = nullptr;
+    int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg);
+    if ( res != SQLITE_OK ) {
+        std::string err = util::fmt("Error executing table creation statement: %s", errorMsg);
+        Error(err.c_str());
+        sqlite3_free(errorMsg);
+        sqlite3_close(db);
+        db = nullptr;
+        return err;
+    }
+
+    return std::nullopt;
+}
+
+/**
+ * Finalizes the backend when it's being closed.
+ */
+void SQLite::Close() {
+    if ( db ) {
+        if ( int res = sqlite3_close_v2(db); res != SQLITE_OK )
+            Error("Sqlite could not close connection");
+
+        db = nullptr;
+    }
+}
+
+/**
+ * The workhorse method for Put(). This must be implemented by plugins.
+ */
+ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) {
+    if ( ! db )
+        return "Database was not open";
+
+    auto json_key = key->ToJSON();
+    auto json_value = value->ToJSON();
+
+    std::string stmt = "INSERT INTO ";
+    stmt.append(table_name);
+    stmt.append("(key_str, value_str) VALUES('");
+    stmt.append(json_key->ToStdStringView());
+    stmt.append("', '");
+    stmt.append(json_value->ToStdStringView());
+    if ( ! overwrite )
+        stmt.append("');");
+    else {
+        // if overwriting, add an UPSERT conflict resolution block
+        stmt.append("') ON CONFLICT(key_str) DO UPDATE SET value_str='");
+        stmt.append(json_value->ToStdStringView());
+        stmt.append("';");
+    }
+
+    char* errorMsg = nullptr;
+    int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg);
+    if ( res != SQLITE_OK ) {
+        return errorMsg;
+    }
+
+    return std::nullopt;
+}
+
+/**
+ * The workhorse method for Get(). This must be implemented for plugins.
+ */
+ValResult SQLite::DoGet(ValPtr key, ValResultCallback* cb) {
+    if ( ! db )
+        return zeek::unexpected<std::string>("Database was not open");
+
+    auto json_key = key->ToJSON();
+
+    std::string stmt = "SELECT value_str from " + table_name + " where key_str = '";
+    stmt.append(json_key->ToStdStringView());
+    stmt.append("';");
+
+    char* errorMsg = nullptr;
+    sqlite3_stmt* st;
+    auto res = checkError(sqlite3_prepare_v2(db, stmt.c_str(), static_cast<int>(stmt.size() + 1), &st, NULL));
+    if ( res.has_value() )
+        return zeek::unexpected<std::string>(util::fmt("Failed to prepare select statement: %s", res.value().c_str()));
+
+    int errorcode = sqlite3_step(st);
+    if ( errorcode == SQLITE_ROW ) {
+        // Column 1 is the value
+        const char* text = (const char*)sqlite3_column_text(st, 0);
+        auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil);
+        if ( std::holds_alternative<ValPtr>(val) ) {
+            ValPtr val_v = std::get<ValPtr>(val);
+            return val_v;
+        }
+        else {
+            return zeek::unexpected<std::string>(std::get<std::string>(val));
+        }
+    }
+
+    return zeek::unexpected<std::string>(util::fmt("Failed to find row for key: %s", sqlite3_errstr(errorcode)));
+}
+
+/**
+ * The workhorse method for Erase(). This must be implemented for plugins.
+ */
+ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) {
+    if ( ! db )
+        return "Database was not open";
+
+    auto json_key = key->ToJSON();
+
+    std::string stmt = "DELETE from " + table_name + " where key_str = \'";
+    stmt.append(json_key->ToStdStringView());
+    stmt.append("\'");
+
+    char* errorMsg = nullptr;
+    int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg);
+    if ( res != SQLITE_OK ) {
+        return errorMsg;
+    }
+
+    return std::nullopt;
+}
+
+// returns true in case of error
+ErrorResult SQLite::checkError(int code) {
+    if ( code != SQLITE_OK && code != SQLITE_DONE ) {
+        std::string msg = util::fmt("SQLite call failed: %s", sqlite3_errmsg(db));
+        Error(msg.c_str());
+        return msg;
+    }
+
+    return std::nullopt;
+}
+
+} // namespace zeek::storage::backend::sqlite

src/storage/backend/sqlite/SQLite.h

@@ -0,0 +1,61 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/storage/Backend.h"
+
+// Forward declare these to avoid including sqlite3.h here
+struct sqlite3;
+struct sqlite3_stmt;
+
+namespace zeek::storage::backend::sqlite {
+
+class SQLite : public Backend {
+public:
+    SQLite(std::string_view tag) : Backend(false, tag) {}
+    ~SQLite() override = default;
+
+    static BackendPtr Instantiate(std::string_view tag);
+
+    /**
+     * Called by the manager system to open the backend.
+     */
+    ErrorResult DoOpen(RecordValPtr options) override;
+
+    /**
+     * Finalizes the backend when it's being closed.
+     */
+    void Close() override;
+
+    /**
+     * Returns whether the backend is opened.
+     */
+    bool IsOpen() override { return db != nullptr; }
+
+    /**
+     * The workhorse method for Put().
+     */
+    ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0,
+                      ErrorResultCallback* cb = nullptr) override;
+
+    /**
+     * The workhorse method for Get().
+     */
+    ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) override;
+
+    /**
+     * The workhorse method for Erase().
+     */
+    ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override;
+
+    // TODO: add support for checking for expired data
+
+private:
+    ErrorResult checkError(int code);
+
+    sqlite3* db = nullptr;
+    std::string full_path;
+    std::string table_name;
+};
+
+} // namespace zeek::storage::backend::sqlite

testing/btest/Baseline/language.expire_func-copy/output

@@ -1,5 +1,14 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 @XXXXXXXXXX.XXXXXX expired a
+@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49656/tcp, resp_h=172.16.238.131, resp_p=22/tcp, proto=6]
+@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17]
+@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6]
+@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=17500/udp, resp_h=172.16.238.255, resp_p=17500/udp, proto=17]
+@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6]
+@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17]
+@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17]
+@XXXXXXXXXX.XXXXXX expired [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17]
+@XXXXXXXXXX.XXXXXX expired a
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17]
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17]
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6]
@@ -9,15 +18,6 @@
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17]
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6]
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17]
-@XXXXXXXXXX.XXXXXX expired a
-@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49656/tcp, resp_h=172.16.238.131, resp_p=22/tcp, proto=6]
-@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17]
-@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6]
-@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=17500/udp, resp_h=172.16.238.255, resp_p=17500/udp, proto=17]
-@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6]
-@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17]
-@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17]
-@XXXXXXXXXX.XXXXXX expired [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17]
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49659/tcp, resp_h=172.16.238.131, resp_p=21/tcp, proto=6]
 @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17]
 @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17]

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in /Users/tim/Desktop/projects/storage-framework/testing/btest/.tmp/scripts.base.frameworks.storage.sqlite-basic/sqlite-basic.zeek, line 42: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F))

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out

@@ -0,0 +1,8 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+T
+value
+T
+value2
+T
+got empty result
+value2

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <no location>: SQLite call failed: unable to open database file ()
+error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str))

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek

@@ -0,0 +1,56 @@
+# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving/erasing basic data
+# @TEST-EXEC: zeek -b %INPUT > out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff .stderr
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/sqlite
+
+# Create a typename here that can be passed down into open_backend.
+type str: string;
+
+event zeek_init() {
+	# Create a database file in the .tmp directory with a 'testing' table
+	local opts : Storage::Backend::SQLite::Options;
+	opts$database_path = "test.sqlite";
+	opts$table_name = "testing";
+
+	local key = "key1111";
+	local value = "value";
+
+	# Test inserting/retrieving a key/value pair that we know won't be in
+	# the backend yet.
+	local b = Storage::open_backend(Storage::SQLITE, opts, str, str);
+	local res = Storage::put(b, [$key=key, $value=value, $overwrite=T, $async_mode=F]);
+	print res;
+
+	local res2 = Storage::get(b, key, F);
+	print res2;
+
+	# Test overwriting a value with put()
+	local value2 = "value2";
+	local res3 = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]);
+	print res3;
+
+	local res4 = Storage::get(b, key, F);
+	print res4;
+
+	# Test erasing a key and getting a "false" result
+	local res5 = Storage::erase(b, key, F);
+	print res5;
+
+	local res6 = Storage::get(b, key, F);
+	if ( ! res6 as bool ) {
+		print "got empty result";
+	}
+
+	# Insert something back into the database to test reopening
+	Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]);
+
+	Storage::close_backend(b);
+
+	# Test reopening the same database and getting the data back out of it
+	local b2 = Storage::open_backend(Storage::SQLITE, opts, str, str);
+	local res7 = Storage::get(b2, key, F);
+	print res7;
+}

testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek

@@ -0,0 +1,21 @@
+# @TEST-DOC: Tests various error handling scenarios for the storage framework
+# @TEST-EXEC: zeek -b %INPUT > out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+
+@load base/frameworks/storage
+@load base/frameworks/reporter
+@load policy/frameworks/storage/backend/sqlite
+
+# Create a typename here that can be passed down into open_backend.
+type str: string;
+
+event zeek_init() {
+	# Test opening a database with an invalid path
+	local opts : Storage::Backend::SQLite::Options;
+	opts$database_path = "/this/path/should/not/exist/test.sqlite";
+	opts$table_name = "testing";
+
+	# This should report an error in .stderr and reporter.log
+	local b = Storage::open_backend(Storage::SQLITE, opts, str, str);
+}