SQLite: Add additional btests, which also cover general storage functionality

- New erase/overwrite tests
- Change existing sqlite-basic test to use async
- Test passing bad keys to validate backend type checking
- New test for compound keys and values

testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out

@@ -0,0 +1,7 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+put result, T
+get result, {
+[2] = b,
+[1] = a,
+[3] = c
+}

testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/erase.zeek, line 28: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F))

testing/btest/Baseline/scripts.base.frameworks.storage.erase/out

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+erase result, T
+get result, got empty result

testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+put result, T
+get result, value7890
+get result same as inserted, T

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/.stderr

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+1362692526.869344 received termination signal
+1362692526.869344 warning in <...>/find-filtered-trace.zeek, line 69: The analyzed trace file was determined to contain only TCP control packets, which may indicate it's been pre-filtered.  By default, Zeek reports the missing segments for this type of trace, but the 'detect_filtered_trace' option may be toggled if that's not desired.

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+put result, T
+get result, value5678
+get result same as inserted, T

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in /Users/tim/Desktop/projects/storage-framework/testing/btest/.tmp/scripts.base.frameworks.storage.sqlite-basic/sqlite-basic.zeek, line 42: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F))
+received termination signal

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out

@@ -1,8 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-T
+put result, T
-value
+get result, value5678
-T
+get result same as inserted, T
-value2
-T
-got empty result
-value2

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr

@@ -1,2 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str))
+error in <...>/sqlite-error-handling.zeek, line 28: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::put(b, (coerce [$key=bad_key, $value=value, $async_mode=F] to Storage::PutArgs)))

testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out

@@ -1 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Put result on closed handle: 0

testing/btest/scripts/base/frameworks/storage/compound-types.zeek

@@ -0,0 +1,74 @@
+# @TEST-DOC: Test operations using more complicated types
+# @TEST-EXEC: zeek -b %INPUT > out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff .stderr
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/sqlite
+
+type Color: enum {
+	Red = 10,
+	White = 20,
+	Blue = 30
+};
+
+type Rec: record
+{
+	hello: string;
+	t: bool;
+	f: bool;
+	n: count &optional;
+	m: count &optional;  # not in input
+	def: count &default = 123;
+	i: int;
+	pi: double;
+	a: string_vec;
+	c1: Color;
+	p: port;
+	ti: time;
+	it: interval;
+	ad: addr;
+	s: subnet;
+	re: pattern;
+	su: subnet_set;
+};
+
+type tbl: table[count] of string;
+
+event zeek_init() {
+	# Create a database file in the .tmp directory with a 'testing' table
+	local opts : Storage::Backend::SQLite::Options;
+	opts$database_path = "types_test.sqlite";
+	opts$table_name = "types_testing";
+
+	local key : Rec;
+	key$hello = "hello";
+	key$t = T;
+	key$f = F;
+	key$n = 1234;
+	key$m = 5678;
+	key$i = -2345;
+	key$pi = 345.0;
+	key$a = ["a","b","c"];
+	key$c1 = Red;
+	key$p = 1234/tcp;
+	key$ti = current_time();
+	key$it = 15sec;
+	key$ad = 1.2.3.4;
+	key$s = 255.255.255.0/24;
+	key$re = /.*/;
+	key$su = [255.255.255.0/24];
+
+	local value : tbl;
+	value[1] = "a";
+	value[2] = "b";
+	value[3] = "c";
+
+	local b = Storage::open_backend(Storage::SQLITE, opts, Rec, tbl);
+
+	local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]);
+	print "put result", res;
+
+	local res2 = Storage::get(b, key, F);
+	print "get result", res2;
+}

testing/btest/scripts/base/frameworks/storage/erase.zeek

@@ -0,0 +1,34 @@
+# @TEST-DOC: Erase existing data in a SQLite backend
+# @TEST-EXEC: cp $FILES/storage-test.sqlite ./storage-test.sqlite
+# @TEST-EXEC: zeek -b %INPUT > out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/sqlite
+
+# Create a typename here that can be passed down into get().
+type str: string;
+
+event zeek_init() {
+	# Create a database file in the .tmp directory with a 'testing' table
+	local opts : Storage::Backend::SQLite::Options;
+	opts$database_path = "storage-test.sqlite";
+	opts$table_name = "testing";
+
+	local key = "key1234";
+
+	# Test inserting/retrieving a key/value pair that we know won't be in
+	# the backend yet.
+	local b = Storage::open_backend(Storage::SQLITE, opts, str, str);
+
+	local res = Storage::erase(b, key, F);
+	print "erase result", res;
+
+	local res2 = Storage::get(b, key, F);
+	if ( ! res2 as bool ) {
+		print "get result, got empty result";
+	}
+
+	Storage::close_backend(b);
+}

testing/btest/scripts/base/frameworks/storage/overwriting.zeek

@@ -0,0 +1,31 @@
+# @TEST-DOC: Overwriting existing data in a SQLite backend
+# @TEST-EXEC: cp $FILES/storage-test.sqlite ./storage-test.sqlite
+# @TEST-EXEC: zeek -b %INPUT > out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff .stderr
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/sqlite
+
+# Create a typename here that can be passed down into get().
+type str: string;
+
+event zeek_init() {
+	local opts : Storage::Backend::SQLite::Options;
+	opts$database_path = "storage-test.sqlite";
+	opts$table_name = "testing";
+
+	local key = "key1234";
+	local value = "value7890";
+
+	local b = Storage::open_backend(Storage::SQLITE, opts, str, str);
+
+	local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]);
+	print "put result", res;
+
+	local res2 = Storage::get(b, key, F);
+	print "get result", res2;
+	print "get result same as inserted", value == (res2 as string);
+
+	Storage::close_backend(b);
+}

testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek

@@ -0,0 +1,47 @@
+# @TEST-DOC: Tests that sqlite async works fine while reading pcaps
+# @TEST-EXEC: zeek -C -r $TRACES/http/get.trace %INPUT > out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/sqlite
+
+redef exit_only_after_terminate = T;
+
+# Create a typename here that can be passed down into get().
+type str: string;
+
+event zeek_init() {
+	# Create a database file in the .tmp directory with a 'testing' table
+	local opts : Storage::Backend::SQLite::Options;
+	opts$database_path = "test.sqlite";
+	opts$table_name = "testing";
+
+	local key = "key1234";
+	local value = "value5678";
+
+	# Test inserting/retrieving a key/value pair that we know won't be in
+	# the backend yet.
+	local b = Storage::open_backend(Storage::SQLITE, opts, str, str);
+
+	when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) {
+		print "put result", res;
+
+		when [b, key, value] ( local res2 = Storage::get(b, key) ) {
+			print "get result", res2;
+			print "get result same as inserted", value == (res2 as string);
+
+			Storage::close_backend(b);
+
+			terminate();
+		}
+		timeout 5 sec {
+			print "get requeest timed out";
+			terminate();
+		}
+	}
+	timeout 5 sec {
+		print "put request timed out";
+		terminate();
+	}
+}

testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek

@@ -1,4 +1,4 @@
-# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving/erasing basic data
+# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving data, using async methods
 # @TEST-EXEC: zeek -b %INPUT > out
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: btest-diff .stderr
@@ -6,7 +6,9 @@
 @load base/frameworks/storage
 @load policy/frameworks/storage/backend/sqlite
 
-# Create a typename here that can be passed down into open_backend.
+redef exit_only_after_terminate = T;
+
+# Create a typename here that can be passed down into get().
 type str: string;
 
 event zeek_init() {
@@ -15,42 +17,31 @@ event zeek_init() {
 	opts$database_path = "test.sqlite";
 	opts$table_name = "testing";
 
-	local key = "key1111";
+	local key = "key1234";
-	local value = "value";
+	local value = "value5678";
 
 	# Test inserting/retrieving a key/value pair that we know won't be in
 	# the backend yet.
 	local b = Storage::open_backend(Storage::SQLITE, opts, str, str);
-	local res = Storage::put(b, [$key=key, $value=value, $overwrite=T, $async_mode=F]);
-	print res;
 
-	local res2 = Storage::get(b, key, F);
+	when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) {
-	print res2;
+		print "put result", res;
 
-	# Test overwriting a value with put()
+		when [b, key, value] ( local res2 = Storage::get(b, key) ) {
-	local value2 = "value2";
+			print "get result", res2;
-	local res3 = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]);
+			print "get result same as inserted", value == (res2 as string);
-	print res3;
-
-	local res4 = Storage::get(b, key, F);
-	print res4;
-
-	# Test erasing a key and getting a "false" result
-	local res5 = Storage::erase(b, key, F);
-	print res5;
-
-	local res6 = Storage::get(b, key, F);
-	if ( ! res6 as bool ) {
-		print "got empty result";
-	}
-
-	# Insert something back into the database to test reopening
-	Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]);
 
 			Storage::close_backend(b);
 
-	# Test reopening the same database and getting the data back out of it
+			terminate();
-	local b2 = Storage::open_backend(Storage::SQLITE, opts, str, str);
+		}
-	local res7 = Storage::get(b2, key, F);
+		timeout 5 sec {
-	print res7;
+			print "get requeest timed out";
+			terminate();
+		}
+	}
+	timeout 5 sec {
+		print "put request timed out";
+		terminate();
+	}
 }

testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek

@@ -18,4 +18,17 @@ event zeek_init() {
 
 	# This should report an error in .stderr and reporter.log
 	local b = Storage::open_backend(Storage::SQLITE, opts, str, str);
+
+	# Open a valid database file
+	opts$database_path = "test.sqlite";
+	b = Storage::open_backend(Storage::SQLITE, opts, str, str);
+
+	local bad_key: count = 12345;
+	local value = "abcde";
+	Storage::put(b, [$key=bad_key, $value=value, $async_mode=F]);
+
+	# Close the backend and then attempt to use the closed handle
+	Storage::close_backend(b);
+	local res = Storage::put(b, [$key="a", $value="b", $async_mode=F]);
+	print fmt("Put result on closed handle: %d", res);
 }