diff --git a/scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek b/scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek new file mode 100644 index 0000000000..6086306018 --- /dev/null +++ b/scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek @@ -0,0 +1 @@ +@load ./main.zeek \ No newline at end of file diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek new file mode 100644 index 0000000000..100d97239e --- /dev/null +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -0,0 +1,22 @@ +##! SQLite storage backend support + +@load base/frameworks/storage/main + +module Storage::Backend::SQLite; + +export { + ## Options record for the built-in SQLite backend. + type Options: record { + ## Path to the database file on disk. Setting this to ":memory:" + ## will tell SQLite to use an in-memory database. Relative paths + ## will be opened relative to the directory where Zeek was + ## started from. Zeek will not create intermediate directories + ## if they do not already exist. See + ## https://www.sqlite.org/c3ref/open.html for more rules on + ## paths that can be passed here. + database_path: string; + + ## Name of the table used for storing data. + table_name: string; + }; +} diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 79db7e235b..34968b2ed6 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -83,6 +83,8 @@ # @load frameworks/spicy/record-spicy-batch.zeek # @load frameworks/spicy/resource-usage.zeek @load frameworks/software/windows-version-detection.zeek +@load frameworks/storage/backend/sqlite/__load__.zeek +@load frameworks/storage/backend/sqlite/main.zeek @load frameworks/telemetry/log.zeek @load integration/collective-intel/__load__.zeek @load integration/collective-intel/main.zeek diff --git a/src/storage/backend/CMakeLists.txt b/src/storage/backend/CMakeLists.txt index 8b13789179..a5af49edbb 100644 --- a/src/storage/backend/CMakeLists.txt +++ b/src/storage/backend/CMakeLists.txt @@ -1 +1 @@ - +add_subdirectory(sqlite) diff --git a/src/storage/backend/sqlite/CMakeLists.txt b/src/storage/backend/sqlite/CMakeLists.txt new file mode 100644 index 0000000000..56ba56b1ed --- /dev/null +++ b/src/storage/backend/sqlite/CMakeLists.txt @@ -0,0 +1,3 @@ +zeek_add_plugin( + Zeek Storage_Backend_SQLite + SOURCES SQLite.cc Plugin.cc) diff --git a/src/storage/backend/sqlite/Plugin.cc b/src/storage/backend/sqlite/Plugin.cc new file mode 100644 index 0000000000..f6d56e77fb --- /dev/null +++ b/src/storage/backend/sqlite/Plugin.cc @@ -0,0 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/plugin/Plugin.h" + +#include "zeek/storage/Component.h" +#include "zeek/storage/backend/sqlite/SQLite.h" + +namespace zeek::storage::backend::sqlite { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() override { + AddComponent(new storage::Component("SQLITE", backend::sqlite::SQLite::Instantiate)); + + plugin::Configuration config; + config.name = "Zeek::Storage_Backend_SQLite"; + config.description = "SQLite backend for storage framework"; + return config; + } +} plugin; + +} // namespace zeek::storage::backend::sqlite diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc new file mode 100644 index 0000000000..8c593a69b8 --- /dev/null +++ b/src/storage/backend/sqlite/SQLite.cc @@ -0,0 +1,176 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/backend/sqlite/SQLite.h" + +#include "zeek/3rdparty/sqlite3.h" +#include "zeek/Func.h" +#include "zeek/Val.h" + +namespace zeek::storage::backend::sqlite { + +storage::BackendPtr SQLite::Instantiate(std::string_view tag) { return make_intrusive(tag); } + +/** + * Called by the manager system to open the backend. + */ +ErrorResult SQLite::DoOpen(RecordValPtr options) { + if ( sqlite3_threadsafe() == 0 ) { + std::string res = + "SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of " + "SQLite. Aborting"; + Error(res.c_str()); + return res; + } + + // Allow connections to same DB to use single data/schema cache. Also + // allows simultaneous writes to one file. +#ifndef ZEEK_TSAN + sqlite3_enable_shared_cache(1); +#endif + + StringValPtr path = options->GetField("database_path"); + full_path = zeek::filesystem::path(path->ToStdString()).string(); + table_name = options->GetField("table_name")->ToStdString(); + + auto open_res = + checkError(sqlite3_open_v2(full_path.c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL)); + if ( open_res.has_value() ) { + sqlite3_close_v2(db); + db = nullptr; + return open_res; + } + + std::string create = "create table if not exists " + table_name + " ("; + create.append("key_str text primary key, value_str text not null);"); + + char* errorMsg = nullptr; + int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); + if ( res != SQLITE_OK ) { + std::string err = util::fmt("Error executing table creation statement: %s", errorMsg); + Error(err.c_str()); + sqlite3_free(errorMsg); + sqlite3_close(db); + db = nullptr; + return err; + } + + return std::nullopt; +} + +/** + * Finalizes the backend when it's being closed. + */ +void SQLite::Close() { + if ( db ) { + if ( int res = sqlite3_close_v2(db); res != SQLITE_OK ) + Error("Sqlite could not close connection"); + + db = nullptr; + } +} + +/** + * The workhorse method for Put(). This must be implemented by plugins. + */ +ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { + if ( ! db ) + return "Database was not open"; + + auto json_key = key->ToJSON(); + auto json_value = value->ToJSON(); + + std::string stmt = "INSERT INTO "; + stmt.append(table_name); + stmt.append("(key_str, value_str) VALUES('"); + stmt.append(json_key->ToStdStringView()); + stmt.append("', '"); + stmt.append(json_value->ToStdStringView()); + if ( ! overwrite ) + stmt.append("');"); + else { + // if overwriting, add an UPSERT conflict resolution block + stmt.append("') ON CONFLICT(key_str) DO UPDATE SET value_str='"); + stmt.append(json_value->ToStdStringView()); + stmt.append("';"); + } + + char* errorMsg = nullptr; + int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg); + if ( res != SQLITE_OK ) { + return errorMsg; + } + + return std::nullopt; +} + +/** + * The workhorse method for Get(). This must be implemented for plugins. + */ +ValResult SQLite::DoGet(ValPtr key, ValResultCallback* cb) { + if ( ! db ) + return zeek::unexpected("Database was not open"); + + auto json_key = key->ToJSON(); + + std::string stmt = "SELECT value_str from " + table_name + " where key_str = '"; + stmt.append(json_key->ToStdStringView()); + stmt.append("';"); + + char* errorMsg = nullptr; + sqlite3_stmt* st; + auto res = checkError(sqlite3_prepare_v2(db, stmt.c_str(), static_cast(stmt.size() + 1), &st, NULL)); + if ( res.has_value() ) + return zeek::unexpected(util::fmt("Failed to prepare select statement: %s", res.value().c_str())); + + int errorcode = sqlite3_step(st); + if ( errorcode == SQLITE_ROW ) { + // Column 1 is the value + const char* text = (const char*)sqlite3_column_text(st, 0); + auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil); + if ( std::holds_alternative(val) ) { + ValPtr val_v = std::get(val); + return val_v; + } + else { + return zeek::unexpected(std::get(val)); + } + } + + return zeek::unexpected(util::fmt("Failed to find row for key: %s", sqlite3_errstr(errorcode))); +} + +/** + * The workhorse method for Erase(). This must be implemented for plugins. + */ +ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { + if ( ! db ) + return "Database was not open"; + + auto json_key = key->ToJSON(); + + std::string stmt = "DELETE from " + table_name + " where key_str = \'"; + stmt.append(json_key->ToStdStringView()); + stmt.append("\'"); + + char* errorMsg = nullptr; + int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg); + if ( res != SQLITE_OK ) { + return errorMsg; + } + + return std::nullopt; +} + +// returns true in case of error +ErrorResult SQLite::checkError(int code) { + if ( code != SQLITE_OK && code != SQLITE_DONE ) { + std::string msg = util::fmt("SQLite call failed: %s", sqlite3_errmsg(db)); + Error(msg.c_str()); + return msg; + } + + return std::nullopt; +} + +} // namespace zeek::storage::backend::sqlite diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h new file mode 100644 index 0000000000..4aca4e1439 --- /dev/null +++ b/src/storage/backend/sqlite/SQLite.h @@ -0,0 +1,61 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/storage/Backend.h" + +// Forward declare these to avoid including sqlite3.h here +struct sqlite3; +struct sqlite3_stmt; + +namespace zeek::storage::backend::sqlite { + +class SQLite : public Backend { +public: + SQLite(std::string_view tag) : Backend(false, tag) {} + ~SQLite() override = default; + + static BackendPtr Instantiate(std::string_view tag); + + /** + * Called by the manager system to open the backend. + */ + ErrorResult DoOpen(RecordValPtr options) override; + + /** + * Finalizes the backend when it's being closed. + */ + void Close() override; + + /** + * Returns whether the backend is opened. + */ + bool IsOpen() override { return db != nullptr; } + + /** + * The workhorse method for Put(). + */ + ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + ErrorResultCallback* cb = nullptr) override; + + /** + * The workhorse method for Get(). + */ + ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) override; + + /** + * The workhorse method for Erase(). + */ + ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override; + + // TODO: add support for checking for expired data + +private: + ErrorResult checkError(int code); + + sqlite3* db = nullptr; + std::string full_path; + std::string table_name; +}; + +} // namespace zeek::storage::backend::sqlite diff --git a/testing/btest/Baseline/language.expire_func-copy/output b/testing/btest/Baseline/language.expire_func-copy/output index 60db90a1b2..17b843d3f7 100644 --- a/testing/btest/Baseline/language.expire_func-copy/output +++ b/testing/btest/Baseline/language.expire_func-copy/output @@ -1,5 +1,14 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. @XXXXXXXXXX.XXXXXX expired a +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49656/tcp, resp_h=172.16.238.131, resp_p=22/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=17500/udp, resp_h=172.16.238.255, resp_p=17500/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired a @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] @@ -9,15 +18,6 @@ @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired a -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49656/tcp, resp_h=172.16.238.131, resp_p=22/tcp, proto=6] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=17500/udp, resp_h=172.16.238.255, resp_p=17500/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49659/tcp, resp_h=172.16.238.131, resp_p=21/tcp, proto=6] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr new file mode 100644 index 0000000000..d172ece6f0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in /Users/tim/Desktop/projects/storage-framework/testing/btest/.tmp/scripts.base.frameworks.storage.sqlite-basic/sqlite-basic.zeek, line 42: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F)) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out new file mode 100644 index 0000000000..c54d0a144d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +T +value +T +value2 +T +got empty result +value2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr new file mode 100644 index 0000000000..6ed2d3c5d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in : SQLite call failed: unable to open database file () +error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek new file mode 100644 index 0000000000..c28bb5041b --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -0,0 +1,56 @@ +# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving/erasing basic data +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/sqlite + +# Create a typename here that can be passed down into open_backend. +type str: string; + +event zeek_init() { + # Create a database file in the .tmp directory with a 'testing' table + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "test.sqlite"; + opts$table_name = "testing"; + + local key = "key1111"; + local value = "value"; + + # Test inserting/retrieving a key/value pair that we know won't be in + # the backend yet. + local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + local res = Storage::put(b, [$key=key, $value=value, $overwrite=T, $async_mode=F]); + print res; + + local res2 = Storage::get(b, key, F); + print res2; + + # Test overwriting a value with put() + local value2 = "value2"; + local res3 = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); + print res3; + + local res4 = Storage::get(b, key, F); + print res4; + + # Test erasing a key and getting a "false" result + local res5 = Storage::erase(b, key, F); + print res5; + + local res6 = Storage::get(b, key, F); + if ( ! res6 as bool ) { + print "got empty result"; + } + + # Insert something back into the database to test reopening + Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); + + Storage::close_backend(b); + + # Test reopening the same database and getting the data back out of it + local b2 = Storage::open_backend(Storage::SQLITE, opts, str, str); + local res7 = Storage::get(b2, key, F); + print res7; +} diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek new file mode 100644 index 0000000000..159972c48b --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -0,0 +1,21 @@ +# @TEST-DOC: Tests various error handling scenarios for the storage framework +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +@load base/frameworks/storage +@load base/frameworks/reporter +@load policy/frameworks/storage/backend/sqlite + +# Create a typename here that can be passed down into open_backend. +type str: string; + +event zeek_init() { + # Test opening a database with an invalid path + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "/this/path/should/not/exist/test.sqlite"; + opts$table_name = "testing"; + + # This should report an error in .stderr and reporter.log + local b = Storage::open_backend(Storage::SQLITE, opts, str, str); +}