From 4968a5c6541e7d655b39fea5a638ae3c10dceaac Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 21 Jul 2016 12:08:57 -0700 Subject: [PATCH] Enable SQLite shared cache mode. This allows all threads accessing the same database to share sqlite objects. This, for example, fixes the issue with several threads simultaneously writing to the same database file. See https://www.sqlite.org/sharedcache.html Addresses BIT-1325 --- src/input/readers/sqlite/SQLite.cc | 3 + src/logging/writers/sqlite/SQLite.cc | 3 + .../ssh.select | 16 ++++ .../logging/sqlite/simultaneous-writes.bro | 90 +++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.simultaneous-writes/ssh.select create mode 100644 testing/btest/scripts/base/frameworks/logging/sqlite/simultaneous-writes.bro diff --git a/src/input/readers/sqlite/SQLite.cc b/src/input/readers/sqlite/SQLite.cc index 9352d04742..5750b8303a 100644 --- a/src/input/readers/sqlite/SQLite.cc +++ b/src/input/readers/sqlite/SQLite.cc @@ -75,6 +75,9 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading: return false; } + // Allow connections to same DB to use single data/schema cache. Also allows simultaneous writes to one file. + sqlite3_enable_shared_cache(1); + if ( Info().mode != MODE_MANUAL ) { Error("SQLite only supports manual reading mode."); diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index ce04839337..cc482b1f3b 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -120,6 +120,9 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, return false; } + // Allow connections to same DB to use single data/schema cache. Also allows simultaneous writes to one file. + sqlite3_enable_shared_cache(1); + num_fields = arg_num_fields; fields = arg_fields; diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.simultaneous-writes/ssh.select b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.simultaneous-writes/ssh.select new file mode 100644 index 0000000000..f85a502b45 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.simultaneous-writes/ssh.select @@ -0,0 +1,16 @@ +1|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|1469128060.6589|100.0|hurz|2,4,1,3|BB,AA,CC|(empty)|10,20,30|(empty)|SSH::foo +{ +if (0 < SSH::i) + return (Foo); +else + return (Bar); + +} +1|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|1469128060.6589|100.0|hurz|2,4,1,3|BB,AA,CC|(empty)|10,20,30|(empty)|SSH::foo +{ +if (0 < SSH::i) + return (Foo); +else + return (Bar); + +} diff --git a/testing/btest/scripts/base/frameworks/logging/sqlite/simultaneous-writes.bro b/testing/btest/scripts/base/frameworks/logging/sqlite/simultaneous-writes.bro new file mode 100644 index 0000000000..2e864aa791 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/sqlite/simultaneous-writes.bro @@ -0,0 +1,90 @@ +# Test simultaneous writes to the same database file. +# +# @TEST-REQUIRES: which sqlite3 +# @TEST-REQUIRES: has-writer Bro::SQLiteWriter +# @TEST-GROUP: sqlite +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: sqlite3 ssh.sqlite 'select * from ssh' > ssh.select +# @TEST-EXEC: sqlite3 ssh.sqlite 'select * from sshtwo' >> ssh.select +# @TEST-EXEC: btest-diff ssh.select +# +# Testing all possible types. + +redef LogSQLite::unset_field = "(unset)"; + +module SSH; + +export { + redef enum Log::ID += { LOG, LOG2 }; + + type Log: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of count; + ve: vector of string; + f: function(i: count) : string; + } &log; +} + +function foo(i : count) : string + { + if ( i > 0 ) + return "Foo"; + else + return "Bar"; + } + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + Log::create_stream(SSH::LOG2, [$columns=Log]); + Log::remove_filter(SSH::LOG, "default"); + Log::remove_filter(SSH::LOG2, "default"); + + local filter: Log::Filter = [$name="sqlite", $path="ssh", $config=table(["tablename"] = "ssh"), $writer=Log::WRITER_SQLITE]; + Log::add_filter(SSH::LOG, filter); + local filter2 = copy(filter); + filter2$name = "sqlite2"; + filter2$config = table(["tablename"] = "sshtwo"); + Log::add_filter(SSH::LOG2, filter2); + + local empty_set: set[string]; + local empty_vector: vector of string; + + local out = [ + $b=T, + $i=-42, + $e=SSH::LOG, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1,2,3,4), + $ss=set("AA", "BB", "CC"), + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector, + $f=foo + ]; + + Log::write(SSH::LOG, out); + Log::write(SSH::LOG2, out); +} +