Merge branch 'topic/robin/sqlite-merge'

Closes #997.

* topic/robin/sqlite-merge: (25 commits)
  Fix to make sqlite test consistent, and updating coverage baselines
  Avoid a CMake warning about 3rdparty looking like a number.
  Fixing linker error.
  and there is no has-reader.
  make sqlite3 executable required and add test-cases for errors
  Renaming src/external -> src/3rdparty
  fix a few small rough edges (mostly comments that do no longer apply)
  fix bug in input-manager regarding enums that a writer reads without 0-terminating the string
  actually make sqlite work again (tests passed because the writer was not actually defined because of the define.)
  add sqlite distribution.
  fix warnings, update baselines, handle rotation
  add sqlite tests and fix small vector/set escaping bugs
  fix small bug with vectors and sets.
  make work with newer AsciiFormatter.
  start adding a different text for empty records for the sqlite writer.
  no, you will never guess from where I copied this file...
  make sqlite support more or less work for logging and input
  make sqlite-writer more stable.
  make it compile with new version of AsciiInputOutput
  and adapt to AsciiInputOutput - seems to work...
  ...

Conflicts:
	scripts/base/frameworks/input/__load__.bro
	src/CMakeLists.txt
	src/input.bif
	src/input/Manager.cc
	src/main.cc
	src/types.bif
	testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

CHANGES

@@ -1,4 +1,11 @@
 
+2.1-619 | 2013-05-15 16:01:42 -0700
+
+  * SQLite reader and writer combo. This allows to read/write
+    persistent data from on disk SQLite databases. The current
+    interface is quite low-level, we'll add higher-level abstractions
+    in the future. (Bernhard Amann)
+
 2.1-576 | 2013-05-15 14:29:09 -0700
 
   * Initial version of new file analysis framework. This moves most of

VERSION

@@ -1 +1 @@
-2.1-576
+2.1-619

aux/broctl

@@ -1 +1 @@
-Subproject commit 19eb25de240b533573891fe9c5bc5bc72a92d9f2
+Subproject commit 7b19aa2d40094167eed509d19103be23257d9b1f

doc/scripts/DocSourcesList.cmake

@@ -39,6 +39,7 @@ rest_target(${psd} base/frameworks/input/readers/ascii.bro)
 rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
 rest_target(${psd} base/frameworks/input/readers/binary.bro)
 rest_target(${psd} base/frameworks/input/readers/raw.bro)
+rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
 rest_target(${psd} base/frameworks/intel/cluster.bro)
 rest_target(${psd} base/frameworks/intel/input.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
@@ -49,6 +50,7 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
 rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
+rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)

scripts/base/frameworks/input/__load__.bro

@@ -3,4 +3,4 @@
 @load ./readers/raw
 @load ./readers/benchmark
 @load ./readers/binary
-
+@load ./readers/sqlite

scripts/base/frameworks/input/readers/sqlite.bro

@@ -0,0 +1,17 @@
+##! Interface for the SQLite input reader.
+##!
+##! The defaults are set to match Bro's ASCII output.
+
+module InputSQLite;
+
+export {
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long.
+	const set_separator = Input::set_separator &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = Input::unset_field &redef;
+
+	## String to use for empty fields.
+	const empty_field = Input::empty_field &redef;
+}

scripts/base/frameworks/logging/__load__.bro

@@ -2,5 +2,6 @@
 @load ./postprocessors
 @load ./writers/ascii
 @load ./writers/dataseries
+@load ./writers/sqlite
 @load ./writers/elasticsearch
 @load ./writers/none

scripts/base/frameworks/logging/writers/sqlite.bro

@@ -0,0 +1,17 @@
+##! Interface for the SQLite log writer.  Redefinable options are available
+##! to tweak the output format of the SQLite reader.
+
+module LogSQLite;
+
+export {
+	## Separator between set elements.
+	const set_separator = Log::set_separator &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = Log::unset_field &redef;
+
+	## String to use for empty fields. This should be different from
+        ## *unset_field* to make the output non-ambigious.
+	const empty_field = Log::empty_field &redef;
+}
+

src/CMakeLists.txt

@@ -184,7 +184,7 @@ macro(BINPAC_TARGET pacFile)
     list(APPEND ALL_BINPAC_INPUTS ${ARGV})
     list(APPEND ALL_BINPAC_OUTPUTS
          ${CMAKE_CURRENT_BINARY_DIR}/${basename}_pac.h
-         ${CMAKE_CURRENT_BINARY_DIR}/${basename}_pac.cc) 
+         ${CMAKE_CURRENT_BINARY_DIR}/${basename}_pac.cc)
 endmacro(BINPAC_TARGET)
 
 binpac_target(binpac-lib.pac)
@@ -227,6 +227,10 @@ binpac_target(modbus.pac
 
 find_package (Threads)
 
+# Avoid CMake warning about "3rdparty" looking like a number.
+cmake_policy(PUSH)
+cmake_policy(SET CMP0012 NEW)
+
 # This macro stores associated headers for any C/C++ source files given
 # as arguments (past _var) as a list in the CMake variable named "_var".
 macro(COLLECT_HEADERS _var)
@@ -246,6 +250,8 @@ macro(COLLECT_HEADERS _var)
     endforeach ()
 endmacro(COLLECT_HEADERS _var)
 
+cmake_policy(POP)
+
 # define a command that's used to run the make_dbg_constants.pl script
 # building the bro binary depends on the outputs of this script
 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
@@ -438,6 +444,7 @@ set(bro_SRCS
     logging/WriterFrontend.cc
     logging/writers/Ascii.cc
     logging/writers/DataSeries.cc
+    logging/writers/SQLite.cc
     logging/writers/ElasticSearch.cc
     logging/writers/None.cc
 
@@ -448,6 +455,7 @@ set(bro_SRCS
     input/readers/Raw.cc
     input/readers/Benchmark.cc
     input/readers/Binary.cc
+    input/readers/SQLite.cc
 
     file_analysis/Manager.cc
     file_analysis/File.cc
@@ -459,6 +467,8 @@ set(bro_SRCS
     file_analysis/Hash.cc
     file_analysis/DataEvent.cc
 
+    "3rdparty/sqlite3.c"
+
     nb_dns.c
     digest.h
 )
@@ -467,7 +477,7 @@ collect_headers(bro_HEADERS ${bro_SRCS})
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
-target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
 
 install(TARGETS bro DESTINATION bin)
 install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)

src/input.bif

@@ -60,3 +60,8 @@ const timedspread: double;
 
 module InputBinary;
 const chunk_size: count;
+
+module InputSQLite;
+const set_separator: string;
+const unset_field: string;
+const empty_field: string;

src/input/Manager.cc

@@ -9,6 +9,7 @@
 #include "readers/Raw.h"
 #include "readers/Benchmark.h"
 #include "readers/Binary.h"
+#include "readers/SQLite.h"
 
 #include "Event.h"
 #include "EventHandler.h"
@@ -36,6 +37,7 @@ ReaderDefinition input_readers[] = {
 	{ BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate },
 	{ BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate },
 	{ BifEnum::Input::READER_BINARY, "Binary", 0, reader::Binary::Instantiate },
+	{ BifEnum::Input::READER_SQLITE, "SQLite", 0, reader::SQLite::Instantiate },
 
 	// End marker
 	{ BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 }
@@ -2116,19 +2118,25 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 		}
 
 	case TYPE_ENUM: {
-		// well, this is kind of stupid, because EnumType just mangles the module name and the var name together again...
+		// Convert to string first to not have to deal with missing
-		// but well
+		// \0's...
-		string module = extract_module_name(val->val.string_val.data);
+		string module_string(val->val.string_val.data, val->val.string_val.length);
-		string var = extract_var_name(val->val.string_val.data);
+		string var_string(val->val.string_val.data, val->val.string_val.length);
+
+		string module = extract_module_name(module_string.c_str());
+		string var = extract_var_name(var_string.c_str());
+
+		// Well, this is kind of stupid, because EnumType just
+		// mangles the module name and the var name together again...
+		// but well.
 		bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str());
 		if ( index == -1 )
-			reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s",
+			reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s, var size: %zu",
-			                        module.c_str(), var.c_str());
+			                        module.c_str(), var.c_str(), var.size());
 
-		return new EnumVal(index, request_type->Ref()->AsEnumType() );
+		return new EnumVal(index, request_type->Ref()->AsEnumType());
 		}
 
-
 	default:
 		reporter->InternalError("unsupported type for input_read");
 	}

src/input/readers/SQLite.cc

@@ -0,0 +1,317 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "config.h"
+
+#include "SQLite.h"
+#include "NetVar.h"
+
+#include <fstream>
+#include <sstream>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "../../threading/SerialTypes.h"
+
+using namespace input::reader;
+using threading::Value;
+using threading::Field;
+
+SQLite::SQLite(ReaderFrontend *frontend) : ReaderBackend(frontend)
+	{
+	set_separator.assign(
+			(const char*) BifConst::LogSQLite::set_separator->Bytes(),
+			BifConst::InputSQLite::set_separator->Len()
+			);
+
+	unset_field.assign(
+			(const char*) BifConst::LogSQLite::unset_field->Bytes(),
+			BifConst::InputSQLite::unset_field->Len()
+			);
+
+	empty_field.assign(
+			(const char*) BifConst::LogAscii::empty_field->Bytes(),
+			BifConst::InputSQLite::empty_field->Len()
+			);
+
+	io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
+	}
+
+SQLite::~SQLite()
+	{
+	DoClose();
+	delete io;
+	}
+
+void SQLite::DoClose()
+	{
+	if ( db != 0 )
+		{
+		sqlite3_close(db);
+		db = 0;
+		}
+	}
+
+bool SQLite::checkError( int code )
+	{
+	if ( code != SQLITE_OK && code != SQLITE_DONE )
+		{
+		Error(Fmt("SQLite call failed: %s", sqlite3_errmsg(db)));
+		return true;
+		}
+
+	return false;
+	}
+
+bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* arg_fields)
+	{
+	if ( sqlite3_threadsafe() == 0 )
+		{
+		Error("SQLite reports that it is not threadsafe. Bro needs a threadsafe version of SQLite. Aborting");
+		return false;
+		}
+
+	started = false;
+
+	string fullpath(info.source);
+	fullpath.append(".sqlite");
+
+	string dbname;
+	map<const char*, const char*>::const_iterator it = info.config.find("dbname");
+	if ( it == info.config.end() )
+		{
+		MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to source %s", info.source));
+		dbname = info.source;
+		}
+	else
+		dbname = it->second;
+
+	string query;
+	it = info.config.find("query");
+	if ( it == info.config.end() )
+		{
+		Error(Fmt("No query specified when setting up SQLite data source. Aborting.", info.source));
+		return false;
+		}
+	else
+		query = it->second;
+
+	if ( checkError(sqlite3_open_v2(
+					fullpath.c_str(),
+					&db,
+					SQLITE_OPEN_READWRITE |
+					SQLITE_OPEN_NOMUTEX
+					,
+					NULL)) )
+		return false;
+
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	// create the prepared select statement that we will re-use forever...
+	if ( checkError(sqlite3_prepare_v2( db, query.c_str(), query.size()+1, &st, NULL )) )
+		{
+		return false;
+		}
+
+	DoUpdate();
+
+	return true;
+	}
+
+// pos = field position
+// subpos = subfield position, only used for port-field
+Value* SQLite::EntryToVal(sqlite3_stmt *st, const threading::Field *field, int pos, int subpos)
+	{
+	if ( sqlite3_column_type(st, pos ) == SQLITE_NULL )
+		return new Value(field->type, false);
+
+	Value* val = new Value(field->type, true);
+
+	switch ( field->type ) {
+	case TYPE_ENUM:
+	case TYPE_STRING:
+		{
+		const char *text = (const char*) sqlite3_column_text(st, pos);
+		int length = sqlite3_column_bytes(st, pos);
+
+		char *out = new char[length];
+		memcpy(out, text, length);
+
+		val->val.string_val.length = length;
+		val->val.string_val.data = out;
+		break;
+		}
+
+	case TYPE_BOOL:
+		{
+		if ( sqlite3_column_type(st, pos) != SQLITE_INTEGER )
+			{
+			Error("Invalid data type for boolean - expected Integer");
+			return 0;
+			}
+
+		int res = sqlite3_column_int(st, pos);
+
+		if ( res == 0 || res == 1 )
+			val->val.int_val = res;
+		else
+			{
+			Error(Fmt("Invalid value for boolean: %d", res));
+			return 0;
+			}
+		break;
+		}
+
+	case TYPE_INT:
+		val->val.int_val = sqlite3_column_int64(st, pos);
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		val->val.double_val = sqlite3_column_double(st, pos);
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		val->val.uint_val = sqlite3_column_int64(st, pos);
+		break;
+
+	case TYPE_PORT:
+		{
+		val->val.port_val.port = sqlite3_column_int(st, pos);
+		val->val.port_val.proto = TRANSPORT_UNKNOWN;
+		if ( subpos != -1 )
+			{
+			const char *text = (const char*) sqlite3_column_text(st, subpos);
+			string s(text, sqlite3_column_bytes(st, subpos));
+			if ( text == 0 )
+				Error("Port protocol definition did not contain text");
+			else
+				val->val.port_val.proto = io->ParseProto(s);
+			}
+		break;
+		}
+
+	case TYPE_SUBNET:
+		{
+		const char *text = (const char*) sqlite3_column_text(st, pos);
+		string s(text, sqlite3_column_bytes(st, pos));
+		int pos = s.find("/");
+		int width = atoi(s.substr(pos+1).c_str());
+		string addr = s.substr(0, pos);
+
+		val->val.subnet_val.prefix = io->ParseAddr(addr);
+		val->val.subnet_val.length = width;
+		break;
+		}
+
+	case TYPE_ADDR:
+		{
+		const char *text = (const char*) sqlite3_column_text(st, pos);
+		string s(text, sqlite3_column_bytes(st, pos));
+		val->val.addr_val = io->ParseAddr(s);
+		break;
+		}
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		{
+		const char *text = (const char*) sqlite3_column_text(st, pos);
+		string s(text, sqlite3_column_bytes(st, pos));
+		val = io->ParseValue(s, "", field->type, field->subtype);
+		break;
+		}
+
+	default:
+		Error(Fmt("unsupported field format %d", field->type));
+		return 0;
+	}
+
+	return val;
+
+	}
+
+bool SQLite::DoUpdate()
+	{
+	int numcolumns = sqlite3_column_count(st);
+	int *mapping = new int [num_fields];
+	int *submapping = new int [num_fields];
+
+	// first set them all to -1
+	for ( unsigned int i = 0; i < num_fields; ++i )
+		{
+		mapping[i] = -1;
+		submapping[i] = -1;
+		}
+
+	for ( int i = 0; i < numcolumns; ++i )
+		{
+		const char *name = sqlite3_column_name(st, i);
+
+		for ( unsigned j = 0; j < num_fields; j++ )
+			{
+			if ( strcmp(fields[j]->name, name) == 0 )
+				{
+				if ( mapping[j] != -1 )
+					{
+					Error(Fmt("SQLite statement returns several columns with name %s! Cannot decide which to choose, aborting", name));
+					return false;
+					}
+
+				mapping[j] = i;
+				}
+
+			if ( fields[j]->secondary_name != 0 && strcmp(fields[j]->secondary_name, name) == 0 )
+				{
+				assert(fields[j]->type == TYPE_PORT);
+				if ( submapping[j] != -1 )
+					{
+					Error(Fmt("SQLite statement returns several columns with name %s! Cannot decide which to choose, aborting", name));
+					return false;
+					}
+
+				submapping[j] = i;
+				}
+			}
+		}
+
+	for ( unsigned int i = 0; i < num_fields; ++i )
+		{
+		if ( mapping[i] == -1 )
+			{
+			Error(Fmt("Required field %s not found after SQLite statement", fields[i]->name));
+			return false;
+			}
+		}
+
+	int errorcode;
+	while ( ( errorcode = sqlite3_step(st)) == SQLITE_ROW )
+		{
+		Value** ofields = new Value*[num_fields];
+
+		for ( unsigned int j = 0; j < num_fields; ++j)
+			{
+			ofields[j] = EntryToVal(st, fields[j], mapping[j], submapping[j]);
+			if ( ofields[j] == 0 )
+				return false;
+			}
+
+		SendEntry(ofields);
+		}
+
+	if ( checkError(errorcode) ) // check the last error code returned by sqlite
+		return false;
+
+	EndCurrentSend();
+
+	delete [] mapping;
+	delete [] submapping;
+
+	if ( checkError(sqlite3_reset(st)) )
+		return false;
+
+	return true;
+	}
+

src/input/readers/SQLite.h

@@ -0,0 +1,55 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef INPUT_READERS_POSTGRES_H
+#define INPUT_READERS_POSTGRES_H
+
+#include "config.h"
+
+#include <iostream>
+#include <vector>
+
+#include "../ReaderBackend.h"
+
+#include "threading/AsciiFormatter.h"
+#include "3rdparty/sqlite3.h"
+
+namespace input { namespace reader {
+
+class SQLite : public ReaderBackend {
+public:
+	SQLite(ReaderFrontend* frontend);
+	~SQLite();
+
+	static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new SQLite(frontend); }
+
+protected:
+	virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* arg_fields);
+	virtual void DoClose();
+	virtual bool DoUpdate();
+	virtual bool DoHeartbeat(double network_time, double current_time)	{ return true; }
+
+private:
+	bool checkError(int code);
+
+	threading::Value* EntryToVal(sqlite3_stmt *st, const threading::Field *field, int pos, int subpos);
+
+	const threading::Field* const * fields; // raw mapping
+	unsigned int num_fields;
+	int mode;
+	bool started;
+	string query;
+	sqlite3 *db;
+	sqlite3_stmt *st;
+	AsciiFormatter* io;
+
+	string set_separator;
+	string unset_field;
+	string empty_field;
+};
+
+
+}
+}
+
+#endif /* INPUT_READERS_POSTGRES_H */
+

src/logging.bif

@@ -82,6 +82,14 @@ const dump_schema: bool;
 const use_integer_for_time: bool;
 const num_threads: count;
 
+# Options for the SQLite writer
+
+module LogSQLite;
+
+const set_separator: string;
+const empty_field: string;
+const unset_field: string;
+
 # Options for the ElasticSearch writer.
 
 module LogElasticSearch;

src/logging/Manager.cc

@@ -26,6 +26,8 @@
 #include "writers/DataSeries.h"
 #endif
 
+#include "writers/SQLite.h"
+
 using namespace logging;
 
 // Structure describing a log writer type.
@@ -40,6 +42,7 @@ struct WriterDefinition {
 WriterDefinition log_writers[] = {
 	{ BifEnum::Log::WRITER_NONE,  "None", 0, writer::None::Instantiate },
 	{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
+	{ BifEnum::Log::WRITER_SQLITE, "SQLite", 0, writer::SQLite::Instantiate },
 
 #ifdef USE_ELASTICSEARCH
 	{ BifEnum::Log::WRITER_ELASTICSEARCH, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },

src/logging/writers/SQLite.cc

@@ -0,0 +1,375 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "config.h"
+
+#include <string>
+#include <errno.h>
+#include <vector>
+
+#include "../../NetVar.h"
+#include "../../threading/SerialTypes.h"
+
+#include "SQLite.h"
+
+using namespace logging;
+using namespace writer;
+using threading::Value;
+using threading::Field;
+
+SQLite::SQLite(WriterFrontend* frontend) : WriterBackend(frontend)
+	{
+	set_separator.assign(
+			(const char*) BifConst::LogSQLite::set_separator->Bytes(),
+			BifConst::LogSQLite::set_separator->Len()
+			);
+
+	unset_field.assign(
+			(const char*) BifConst::LogSQLite::unset_field->Bytes(),
+			BifConst::LogSQLite::unset_field->Len()
+			);
+
+	empty_field.assign(
+			(const char*) BifConst::LogSQLite::empty_field->Bytes(),
+			BifConst::LogSQLite::empty_field->Len()
+			);
+
+	db = 0;
+	io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
+	}
+
+SQLite::~SQLite()
+	{
+	if ( db != 0 )
+		{
+		sqlite3_close(db);
+		db = 0;
+		}
+
+	delete io;
+	}
+
+string SQLite::GetTableType(int arg_type, int arg_subtype) {
+	string type;
+
+	switch ( arg_type ) {
+	case TYPE_BOOL:
+		type = "boolean";
+		break;
+
+	case TYPE_INT:
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+	case TYPE_PORT: // note that we do not save the protocol at the moment. Just like in the case of the ascii-writer
+		type = "integer";
+		break;
+
+	case TYPE_SUBNET:
+	case TYPE_ADDR:
+		type = "text"; // sqlite3 does not have a type for internet addresses
+		break;
+
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+	case TYPE_DOUBLE:
+		type = "double precision";
+		break;
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		type = "text";
+		break;
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		type = "text"; // dirty - but sqlite does not directly support arrays. so - we just roll it into a ","-separated string.
+		break;
+
+	default:
+		Error(Fmt("unsupported field format %d ", arg_type));
+		return ""; // not the cleanest way to abort. But sqlite will complain on create table...
+	}
+
+	return type;
+}
+
+// returns true true in case of error
+bool SQLite::checkError(int code)
+	{
+	if ( code != SQLITE_OK && code != SQLITE_DONE )
+		{
+		Error(Fmt("SQLite call failed: %s", sqlite3_errmsg(db)));
+		return true;
+		}
+
+	return false;
+	}
+
+bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields,
+			    const Field* const * arg_fields)
+	{
+	if ( sqlite3_threadsafe() == 0 )
+		{
+		Error("SQLite reports that it is not threadsafe. Bro needs a threadsafe version of SQLite. Aborting");
+		return false;
+		}
+
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	string fullpath(info.path);
+	fullpath.append(".sqlite");
+	string dbname;
+
+	map<const char*, const char*>::const_iterator it = info.config.find("dbname");
+	if ( it == info.config.end() )
+		{
+		MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to path %s", info.path));
+		dbname = info.path;
+		}
+	else
+		dbname = it->second;
+
+	if ( checkError(sqlite3_open_v2(
+					fullpath.c_str(),
+					&db,
+					SQLITE_OPEN_READWRITE |
+					SQLITE_OPEN_CREATE |
+					SQLITE_OPEN_NOMUTEX
+					,
+					NULL)) )
+		return false;
+
+	string create = "CREATE TABLE IF NOT EXISTS " + dbname + " (\n";
+		//"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here.
+
+	for ( unsigned int i = 0; i < num_fields; ++i )
+		{
+		const Field* field = fields[i];
+
+		if ( i != 0 )
+			create += ",\n";
+
+		// sadly sqlite3 has no other method for escaping stuff. That I know of.
+		char* fieldname = sqlite3_mprintf("%Q", fields[i]->name);
+		if ( fieldname == 0 )
+			{
+			InternalError("Could not malloc memory");
+			return false;
+			}
+
+		create += fieldname;
+		sqlite3_free(fieldname);
+
+		string type = GetTableType(field->type, field->subtype);
+		if ( type == "" )
+			{
+			InternalError(Fmt("Could not determine type for field %lu:%s", i, fieldname));
+			return false;
+			}
+
+		create += " " + type;
+
+		/* if ( !field->optional ) {
+		 create += " NOT NULL";
+		 } */
+		}
+
+	create += "\n);";
+
+	char *errorMsg = 0;
+	int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg);
+	if ( res != SQLITE_OK )
+		{
+		Error(Fmt("Error executing table creation statement: %s", errorMsg));
+		sqlite3_free(errorMsg);
+		return false;
+		}
+
+	// create the prepared statement that will be re-used forever...
+	string insert = "VALUES (";
+	string names = "INSERT INTO " + dbname + " ( ";
+
+	for ( unsigned int i = 0; i < num_fields; i++ )
+		{
+		bool ac = true;
+
+		if ( i == 0 )
+			ac = false;
+		else
+			{
+			names += ", ";
+			insert += ", ";
+			}
+
+		insert += "?";
+
+		char* fieldname = sqlite3_mprintf("%Q", fields[i]->name);
+		if ( fieldname == 0 )
+			{
+			InternalError("Could not malloc memory");
+			return false;
+			}
+
+		names.append(fieldname);
+		sqlite3_free(fieldname);
+		}
+
+	insert += ");";
+	names += ") ";
+
+	insert = names + insert;
+
+	if ( checkError(sqlite3_prepare_v2(db, insert.c_str(), insert.size()+1, &st, NULL)) )
+		return false;
+
+	return true;
+	}
+
+// Format String
+char* SQLite::FS(const char* format, ...)
+	{
+	char* buf;
+
+	va_list al;
+	va_start(al, format);
+	int n = vasprintf(&buf, format, al);
+	va_end(al);
+
+	assert(n >= 0);
+
+	return buf;
+	}
+
+int SQLite::AddParams(Value* val, int pos)
+	{
+	if ( ! val->present )
+		return sqlite3_bind_null(st, pos);
+
+	switch ( val->type ) {
+	case TYPE_BOOL:
+		return sqlite3_bind_int(st, pos, val->val.int_val != 0 ? 1 : 0 );
+
+	case TYPE_INT:
+		return sqlite3_bind_int(st, pos, val->val.int_val);
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		return sqlite3_bind_int(st, pos, val->val.uint_val);
+
+	case TYPE_PORT:
+		return sqlite3_bind_int(st, pos, val->val.port_val.port);
+
+	case TYPE_SUBNET:
+		{
+		string out = io->Render(val->val.subnet_val);
+		return sqlite3_bind_text(st, pos, out.data(), out.size(), SQLITE_TRANSIENT);
+		}
+
+	case TYPE_ADDR:
+		{
+		string out = io->Render(val->val.addr_val);
+		return sqlite3_bind_text(st, pos, out.data(), out.size(), SQLITE_TRANSIENT);
+		}
+
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+	case TYPE_DOUBLE:
+		return sqlite3_bind_double(st, pos, val->val.double_val);
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		{
+		if ( ! val->val.string_val.length || val->val.string_val.length == 0 )
+			return sqlite3_bind_null(st, pos);
+
+		return sqlite3_bind_text(st, pos, val->val.string_val.data, val->val.string_val.length, SQLITE_TRANSIENT);
+		}
+
+	case TYPE_TABLE:
+		{
+		ODesc desc;
+		desc.Clear();
+		desc.EnableEscaping();
+		desc.AddEscapeSequence(set_separator);
+
+		if ( ! val->val.set_val.size )
+			desc.Add(empty_field);
+		else
+			for ( int j = 0; j < val->val.set_val.size; j++ )
+				{
+				if ( j > 0 )
+					desc.AddRaw(set_separator);
+
+				io->Describe(&desc, val->val.set_val.vals[j], fields[pos]->name);
+				}
+
+		desc.RemoveEscapeSequence(set_separator);
+		return sqlite3_bind_text(st, pos, (const char*) desc.Bytes(), desc.Len(), SQLITE_TRANSIENT);
+		}
+
+	case TYPE_VECTOR:
+		{
+		ODesc desc;
+		desc.Clear();
+		desc.EnableEscaping();
+		desc.AddEscapeSequence(set_separator);
+
+		if ( ! val->val.vector_val.size )
+			desc.Add(empty_field);
+		else
+			for ( int j = 0; j < val->val.vector_val.size; j++ )
+				{
+				if ( j > 0 )
+					desc.AddRaw(set_separator);
+
+				io->Describe(&desc, val->val.vector_val.vals[j], fields[pos]->name);
+				}
+
+		desc.RemoveEscapeSequence(set_separator);
+		return sqlite3_bind_text(st, pos, (const char*) desc.Bytes(), desc.Len(), SQLITE_TRANSIENT);
+		}
+
+	default:
+		Error(Fmt("unsupported field format %d", val->type));
+		return 0;
+	}
+	}
+
+bool SQLite::DoWrite(int num_fields, const Field* const * fields, Value** vals)
+	{
+	// bind parameters
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		if ( checkError(AddParams(vals[i], i+1)) )
+			return false;
+		}
+
+	// execute query
+	if ( checkError(sqlite3_step(st)) )
+		return false;
+
+	// clean up and make ready for next query execution
+	if ( checkError(sqlite3_clear_bindings(st)) )
+		return false;
+
+	if ( checkError(sqlite3_reset(st)) )
+		return false;
+
+	return true;
+	}
+
+bool SQLite::DoRotate(const char* rotated_path, double open, double close, bool terminating)
+	{
+	if ( ! FinishedRotation("/dev/null", Info().path, open, close, terminating))
+		{
+		Error(Fmt("error rotating %s", Info().path));
+		return false;
+		}
+
+	return true;
+        }
+

src/logging/writers/SQLite.h

@@ -0,0 +1,61 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Log writer for SQLITE logs.
+
+#ifndef LOGGING_WRITER_SQLITE_H
+#define LOGGING_WRITER_SQLITE_H
+
+#include "config.h"
+
+#include "../WriterBackend.h"
+
+#include "threading/AsciiFormatter.h"
+#include "3rdparty/sqlite3.h"
+
+namespace logging { namespace writer {
+
+class SQLite : public WriterBackend {
+public:
+	SQLite(WriterFrontend* frontend);
+	~SQLite();
+
+	static WriterBackend* Instantiate(WriterFrontend* frontend)
+		{ return new SQLite(frontend); }
+
+protected:
+	virtual bool DoInit(const WriterInfo& info, int arg_num_fields,
+			    const threading::Field* const* arg_fields);
+	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
+			     threading::Value** vals);
+	virtual bool DoSetBuf(bool enabled) { return true; }
+	virtual bool DoRotate(const char* rotated_path, double open,
+			      double close, bool terminating);
+	virtual bool DoFlush(double network_time)	{ return true; }
+	virtual bool DoFinish(double network_time)	{ return true; }
+	virtual bool DoHeartbeat(double network_time, double current_time)	{ return true; }
+
+private:
+	bool checkError(int code);
+
+	int AddParams(threading::Value* val, int pos);
+	string GetTableType(int, int);
+	char* FS(const char* format, ...);
+
+	const threading::Field* const * fields; // raw mapping
+	unsigned int num_fields;
+
+	sqlite3 *db;
+	sqlite3_stmt *st;
+
+	string set_separator;
+	string unset_field;
+	string empty_field;
+
+	AsciiFormatter* io;
+};
+
+}
+}
+
+#endif /* LOGGING_WRITER_SQLITE_H */
+

src/main.cc

@@ -63,6 +63,8 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 
 #include "binpac_bro.h"
 
+#include "3rdparty/sqlite3.h"
+
 Brofiler brofiler;
 
 magic_t magic_desc_cookie = 0;
@@ -737,6 +739,8 @@ int main(int argc, char** argv)
 	bro_init_magic(&magic_desc_cookie, MAGIC_NONE);
 	bro_init_magic(&magic_mime_cookie, MAGIC_MIME);
 
+	sqlite3_initialize();
+
 	// FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't
 	// seed the PRNG. We should do this here (but at least Linux, FreeBSD
 	// and Solaris provide /dev/urandom).
@@ -1096,6 +1100,8 @@ int main(int argc, char** argv)
 		curl_global_cleanup();
 #endif
 
+		sqlite3_shutdown();
+
 		terminate_bro();
 
 		// Close files after net_delete(), because net_delete()

src/types.bif

@@ -170,6 +170,7 @@ enum Writer %{
 	WRITER_NONE,
 	WRITER_ASCII,
 	WRITER_DATASERIES,
+	WRITER_SQLITE,
 	WRITER_ELASTICSEARCH,
 %}
 
@@ -197,6 +198,7 @@ enum Reader %{
 	READER_RAW,
 	READER_BENCHMARK,
 	READER_BINARY,
+	READER_SQLITE,
 %}
 
 enum Event %{

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-05-14-00-01-42
+#open	2013-05-15-23-01-21
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -21,6 +21,7 @@ scripts/base/init-bare.bro
       scripts/base/frameworks/logging/postprocessors/sftp.bro
     scripts/base/frameworks/logging/writers/ascii.bro
     scripts/base/frameworks/logging/writers/dataseries.bro
+    scripts/base/frameworks/logging/writers/sqlite.bro
     scripts/base/frameworks/logging/writers/elasticsearch.bro
     scripts/base/frameworks/logging/writers/none.bro
   scripts/base/frameworks/input/__load__.bro
@@ -30,9 +31,10 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/input/readers/raw.bro
     scripts/base/frameworks/input/readers/benchmark.bro
     scripts/base/frameworks/input/readers/binary.bro
+    scripts/base/frameworks/input/readers/sqlite.bro
   scripts/base/frameworks/file-analysis/__load__.bro
     scripts/base/frameworks/file-analysis/main.bro
       build/src/base/file_analysis.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-05-14-00-01-42
+#close	2013-05-15-23-01-21

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-05-14-00-01-43
+#open	2013-05-15-23-01-31
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -21,6 +21,7 @@ scripts/base/init-bare.bro
       scripts/base/frameworks/logging/postprocessors/sftp.bro
     scripts/base/frameworks/logging/writers/ascii.bro
     scripts/base/frameworks/logging/writers/dataseries.bro
+    scripts/base/frameworks/logging/writers/sqlite.bro
     scripts/base/frameworks/logging/writers/elasticsearch.bro
     scripts/base/frameworks/logging/writers/none.bro
   scripts/base/frameworks/input/__load__.bro
@@ -30,6 +31,7 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/input/readers/raw.bro
     scripts/base/frameworks/input/readers/benchmark.bro
     scripts/base/frameworks/input/readers/binary.bro
+    scripts/base/frameworks/input/readers/sqlite.bro
   scripts/base/frameworks/file-analysis/__load__.bro
     scripts/base/frameworks/file-analysis/main.bro
       build/src/base/file_analysis.bif.bro
@@ -138,4 +140,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-05-14-00-01-43
+#close	2013-05-15-23-01-31

testing/btest/Baseline/scripts.base.frameworks.input.sqlite.basic/out

@@ -0,0 +1,137 @@
+[ts=1300475167.096535, uid=dnGM1AdIVyh, id=[orig_h=141.142.220.202, orig_p=5353/unknown, resp_h=224.0.0.251, resp_p=5353/unknown], proto=udp, service=dns, duration=<uninitialized>, orig_bytes=<uninitialized>, resp_bytes=<uninitialized>, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=1, orig_ip_bytes=73, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475167.097012, uid=fv9q7WjEgp1, id=[orig_h=fe80::217:f2ff:fed7:cf65, orig_p=5353/unknown, resp_h=ff02::fb, resp_p=5353/unknown], proto=udp, service=<uninitialized>, duration=<uninitialized>, orig_bytes=<uninitialized>, resp_bytes=<uninitialized>, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=1, orig_ip_bytes=199, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475167.099816, uid=0Ox0H56yl88, id=[orig_h=141.142.220.50, orig_p=5353/unknown, resp_h=224.0.0.251, resp_p=5353/unknown], proto=udp, service=<uninitialized>, duration=<uninitialized>, orig_bytes=<uninitialized>, resp_bytes=<uninitialized>, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=1, orig_ip_bytes=179, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475168.853899, uid=rvmSc7rDQub, id=[orig_h=141.142.220.118, orig_p=43927/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000435, orig_bytes=38, resp_bytes=89, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=117, tunnel_parents={
+
+}]
+0
+[ts=1300475168.854378, uid=ogkztouSArh, id=[orig_h=141.142.220.118, orig_p=37676/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.00042, orig_bytes=52, resp_bytes=99, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=80, resp_pkts=1, resp_ip_bytes=127, tunnel_parents={
+
+}]
+0
+[ts=1300475168.854837, uid=0UIDdXFt7Tb, id=[orig_h=141.142.220.118, orig_p=40526/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000392, orig_bytes=38, resp_bytes=183, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=211, tunnel_parents={
+
+}]
+0
+[ts=1300475168.857956, uid=WqFYV51UIq7, id=[orig_h=141.142.220.118, orig_p=32902/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000317, orig_bytes=38, resp_bytes=89, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=117, tunnel_parents={
+
+}]
+0
+[ts=1300475168.858306, uid=ylcqZpbz6K2, id=[orig_h=141.142.220.118, orig_p=59816/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000343, orig_bytes=52, resp_bytes=99, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=80, resp_pkts=1, resp_ip_bytes=127, tunnel_parents={
+
+}]
+0
+[ts=1300475168.858713, uid=blhldTzA7Y6, id=[orig_h=141.142.220.118, orig_p=59714/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000375, orig_bytes=38, resp_bytes=183, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=211, tunnel_parents={
+
+}]
+0
+[ts=1300475168.891644, uid=Sc34cGJo3Kg, id=[orig_h=141.142.220.118, orig_p=58206/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000339, orig_bytes=38, resp_bytes=89, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=117, tunnel_parents={
+
+}]
+0
+[ts=1300475168.892037, uid=RzvFrfXSRfk, id=[orig_h=141.142.220.118, orig_p=38911/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000335, orig_bytes=52, resp_bytes=99, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=80, resp_pkts=1, resp_ip_bytes=127, tunnel_parents={
+
+}]
+0
+[ts=1300475168.892414, uid=GaaFI58mpbe, id=[orig_h=141.142.220.118, orig_p=59746/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000421, orig_bytes=38, resp_bytes=183, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=211, tunnel_parents={
+
+}]
+0
+[ts=1300475168.893988, uid=tr7M6tvAIQa, id=[orig_h=141.142.220.118, orig_p=45000/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000384, orig_bytes=38, resp_bytes=89, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=117, tunnel_parents={
+
+}]
+0
+[ts=1300475168.894422, uid=gV0TcSc2pb4, id=[orig_h=141.142.220.118, orig_p=48479/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000317, orig_bytes=52, resp_bytes=99, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=80, resp_pkts=1, resp_ip_bytes=127, tunnel_parents={
+
+}]
+0
+[ts=1300475168.894787, uid=MOG0z4PYOhk, id=[orig_h=141.142.220.118, orig_p=48128/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000423, orig_bytes=38, resp_bytes=183, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=66, resp_pkts=1, resp_ip_bytes=211, tunnel_parents={
+
+}]
+0
+[ts=1300475168.901749, uid=PlehgEduUyj, id=[orig_h=141.142.220.118, orig_p=56056/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000402, orig_bytes=36, resp_bytes=131, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=64, resp_pkts=1, resp_ip_bytes=159, tunnel_parents={
+
+}]
+0
+[ts=1300475168.902195, uid=4eZgk09f2Re, id=[orig_h=141.142.220.118, orig_p=55092/unknown, resp_h=141.142.2.2, resp_p=53/unknown], proto=udp, service=dns, duration=0.000374, orig_bytes=36, resp_bytes=198, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=Dd, orig_pkts=1, orig_ip_bytes=64, resp_pkts=1, resp_ip_bytes=226, tunnel_parents={
+
+}]
+0
+[ts=1300475169.899438, uid=3xwJPc7mQ9a, id=[orig_h=141.142.220.44, orig_p=5353/unknown, resp_h=224.0.0.251, resp_p=5353/unknown], proto=udp, service=dns, duration=<uninitialized>, orig_bytes=<uninitialized>, resp_bytes=<uninitialized>, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=1, orig_ip_bytes=85, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475170.862384, uid=yxTcvvTKWQ4, id=[orig_h=141.142.220.226, orig_p=137/unknown, resp_h=141.142.220.255, resp_p=137/unknown], proto=udp, service=dns, duration=2.613017, orig_bytes=350, resp_bytes=0, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=7, orig_ip_bytes=546, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475171.675372, uid=8bLW3XNfhCj, id=[orig_h=fe80::3074:17d5:2052:c324, orig_p=65373/unknown, resp_h=ff02::1:3, resp_p=5355/unknown], proto=udp, service=dns, duration=0.100096, orig_bytes=66, resp_bytes=0, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=2, orig_ip_bytes=162, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475171.677081, uid=rqjhiiRPjEe, id=[orig_h=141.142.220.226, orig_p=55131/unknown, resp_h=224.0.0.252, resp_p=5355/unknown], proto=udp, service=dns, duration=0.100021, orig_bytes=66, resp_bytes=0, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=2, orig_ip_bytes=122, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475173.116749, uid=hTPyfL3QSGa, id=[orig_h=fe80::3074:17d5:2052:c324, orig_p=54213/unknown, resp_h=ff02::1:3, resp_p=5355/unknown], proto=udp, service=dns, duration=0.099801, orig_bytes=66, resp_bytes=0, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=2, orig_ip_bytes=162, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475173.117362, uid=EruUQ9AJRj4, id=[orig_h=141.142.220.226, orig_p=55671/unknown, resp_h=224.0.0.252, resp_p=5355/unknown], proto=udp, service=dns, duration=0.099849, orig_bytes=66, resp_bytes=0, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=2, orig_ip_bytes=122, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475173.153679, uid=sw1bKJOMjuk, id=[orig_h=141.142.220.238, orig_p=56641/unknown, resp_h=141.142.220.255, resp_p=137/unknown], proto=udp, service=dns, duration=<uninitialized>, orig_bytes=<uninitialized>, resp_bytes=<uninitialized>, conn_state=S0, local_orig=<uninitialized>, missed_bytes=0, history=D, orig_pkts=1, orig_ip_bytes=78, resp_pkts=0, resp_ip_bytes=0, tunnel_parents={
+
+}]
+0
+[ts=1300475168.724007, uid=NPHCuyWykE7, id=[orig_h=141.142.220.118, orig_p=48649/unknown, resp_h=208.80.152.118, resp_p=80/unknown], proto=tcp, service=http, duration=0.119905, orig_bytes=525, resp_bytes=232, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=4, orig_ip_bytes=741, resp_pkts=3, resp_ip_bytes=396, tunnel_parents={
+
+}]
+0
+[ts=1300475168.892936, uid=VapPqRhPgJ4, id=[orig_h=141.142.220.118, orig_p=50000/unknown, resp_h=208.80.152.3, resp_p=80/unknown], proto=tcp, service=http, duration=0.229603, orig_bytes=1148, resp_bytes=734, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=6, orig_ip_bytes=1468, resp_pkts=4, resp_ip_bytes=950, tunnel_parents={
+
+}]
+0
+[ts=1300475168.859163, uid=3607hh8C3bc, id=[orig_h=141.142.220.118, orig_p=49998/unknown, resp_h=208.80.152.3, resp_p=80/unknown], proto=tcp, service=http, duration=0.215893, orig_bytes=1130, resp_bytes=734, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=6, orig_ip_bytes=1450, resp_pkts=4, resp_ip_bytes=950, tunnel_parents={
+
+}]
+0
+[ts=1300475168.855305, uid=tgYMrIvzDSg, id=[orig_h=141.142.220.118, orig_p=49996/unknown, resp_h=208.80.152.3, resp_p=80/unknown], proto=tcp, service=http, duration=0.218501, orig_bytes=1171, resp_bytes=733, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=6, orig_ip_bytes=1491, resp_pkts=4, resp_ip_bytes=949, tunnel_parents={
+
+}]
+0
+[ts=1300475168.895267, uid=xQsjPwNBrXd, id=[orig_h=141.142.220.118, orig_p=50001/unknown, resp_h=208.80.152.3, resp_p=80/unknown], proto=tcp, service=http, duration=0.227284, orig_bytes=1178, resp_bytes=734, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=6, orig_ip_bytes=1498, resp_pkts=4, resp_ip_bytes=950, tunnel_parents={
+
+}]
+0
+[ts=1300475168.902635, uid=Ap3GzMI1vM9, id=[orig_h=141.142.220.118, orig_p=35642/unknown, resp_h=208.80.152.2, resp_p=80/unknown], proto=tcp, service=http, duration=0.120041, orig_bytes=534, resp_bytes=412, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=4, orig_ip_bytes=750, resp_pkts=3, resp_ip_bytes=576, tunnel_parents={
+
+}]
+0
+[ts=1300475168.85533, uid=FTVcgrmNy52, id=[orig_h=141.142.220.118, orig_p=49997/unknown, resp_h=208.80.152.3, resp_p=80/unknown], proto=tcp, service=http, duration=0.21972, orig_bytes=1125, resp_bytes=734, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=6, orig_ip_bytes=1445, resp_pkts=4, resp_ip_bytes=950, tunnel_parents={
+
+}]
+0
+[ts=1300475169.780331, uid=1xFx4PGdeq5, id=[orig_h=141.142.220.235, orig_p=6705/unknown, resp_h=173.192.163.128, resp_p=80/unknown], proto=tcp, service=<uninitialized>, duration=<uninitialized>, orig_bytes=<uninitialized>, resp_bytes=<uninitialized>, conn_state=OTH, local_orig=<uninitialized>, missed_bytes=0, history=h, orig_pkts=0, orig_ip_bytes=0, resp_pkts=1, resp_ip_bytes=48, tunnel_parents={
+
+}]
+0
+[ts=1300475168.652003, uid=WIG1ud65z22, id=[orig_h=141.142.220.118, orig_p=35634/unknown, resp_h=208.80.152.2, resp_p=80/unknown], proto=tcp, service=<uninitialized>, duration=0.061329, orig_bytes=463, resp_bytes=350, conn_state=OTH, local_orig=<uninitialized>, missed_bytes=0, history=DdA, orig_pkts=2, orig_ip_bytes=567, resp_pkts=1, resp_ip_bytes=402, tunnel_parents={
+
+}]
+0
+[ts=1300475168.892913, uid=o2gAkl4V7sa, id=[orig_h=141.142.220.118, orig_p=49999/unknown, resp_h=208.80.152.3, resp_p=80/unknown], proto=tcp, service=http, duration=0.220961, orig_bytes=1137, resp_bytes=733, conn_state=S1, local_orig=<uninitialized>, missed_bytes=0, history=ShADad, orig_pkts=6, orig_ip_bytes=1457, resp_pkts=4, resp_ip_bytes=949, tunnel_parents={
+
+}]
+0
+End of data

testing/btest/Baseline/scripts.base.frameworks.input.sqlite.error/cmpfile

@@ -0,0 +1,6 @@
+>>>
+error: ../ssh/Input::READER_SQLITE: Init failed
+error: ../ssh/Input::READER_SQLITE: Required field vh not found after SQLite statement
+error: ../ssh/Input::READER_SQLITE: SQLite call failed: no such column: g
+error: ../ssh/Input::READER_SQLITE: terminating thread
+received termination signal

testing/btest/Baseline/scripts.base.frameworks.input.sqlite.port/out

@@ -0,0 +1,3 @@
+5353/udp
+6162/tcp
+End of data

testing/btest/Baseline/scripts.base.frameworks.input.sqlite.types/out

@@ -0,0 +1,15 @@
+[b=T, i=-42, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1358376849.393854, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], vs=[], vn=<uninitialized>]
+0
+1
+End of data

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr

@@ -0,0 +1,3 @@
+ssh/Log::WRITER_SQLITE: dbname configuration option not found. Defaulting to path ssh
+error: ssh/Log::WRITER_SQLITE: SQLite call failed: table ssh has no column named f
+error: ssh/Log::WRITER_SQLITE: terminating thread

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.types/ssh.select

@@ -0,0 +1,8 @@
+1|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|1363036624.07106|100.0|hurz|2,4,1,3|CC,AA,BB|(empty)|10,20,30|(empty)|SSH::foo
+{ 
+if (0 < SSH::i) 
+	return (Foo);
+else
+	return (Bar);
+
+}

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.wikipedia/conn.select

@@ -0,0 +1,34 @@
+1300475167.09654|UWkUyAuUGXf|141.142.220.202|5353|224.0.0.251|5353|udp|dns||||S0||0|D|1|73|0|0|(empty)
+1300475167.09701|arKYeMETxOg|fe80::217:f2ff:fed7:cf65|5353|ff02::fb|5353|udp|||||S0||0|D|1|199|0|0|(empty)
+1300475167.09982|k6kgXLOoSKl|141.142.220.50|5353|224.0.0.251|5353|udp|||||S0||0|D|1|179|0|0|(empty)
+1300475168.652|nQcgTWjvg4c|141.142.220.118|35634|208.80.152.2|80|tcp||0.0613288879394531|463|350|OTH||0|DdA|2|567|1|402|(empty)
+1300475168.72401|j4u32Pc5bif|141.142.220.118|48649|208.80.152.118|80|tcp|http|0.1199049949646|525|232|S1||0|ShADad|4|741|3|396|(empty)
+1300475168.8539|TEfuqmmG4bh|141.142.220.118|43927|141.142.2.2|53|udp|dns|0.000435113906860352|38|89|SF||0|Dd|1|66|1|117|(empty)
+1300475168.85438|FrJExwHcSal|141.142.220.118|37676|141.142.2.2|53|udp|dns|0.000420093536376953|52|99|SF||0|Dd|1|80|1|127|(empty)
+1300475168.85484|5OKnoww6xl4|141.142.220.118|40526|141.142.2.2|53|udp|dns|0.000391960144042969|38|183|SF||0|Dd|1|66|1|211|(empty)
+1300475168.85531|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|tcp|http|0.218501091003418|1171|733|S1||0|ShADad|6|1491|4|949|(empty)
+1300475168.85533|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|tcp|http|0.219720125198364|1125|734|S1||0|ShADad|6|1445|4|950|(empty)
+1300475168.85796|fRFu0wcOle6|141.142.220.118|32902|141.142.2.2|53|udp|dns|0.000317096710205078|38|89|SF||0|Dd|1|66|1|117|(empty)
+1300475168.85831|qSsw6ESzHV4|141.142.220.118|59816|141.142.2.2|53|udp|dns|0.000343084335327148|52|99|SF||0|Dd|1|80|1|127|(empty)
+1300475168.85871|iE6yhOq3SF|141.142.220.118|59714|141.142.2.2|53|udp|dns|0.000375032424926758|38|183|SF||0|Dd|1|66|1|211|(empty)
+1300475168.85916|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|tcp|http|0.215893030166626|1130|734|S1||0|ShADad|6|1450|4|950|(empty)
+1300475168.89164|qCaWGmzFtM5|141.142.220.118|58206|141.142.2.2|53|udp|dns|0.000339031219482422|38|89|SF||0|Dd|1|66|1|117|(empty)
+1300475168.89204|70MGiRM1Qf4|141.142.220.118|38911|141.142.2.2|53|udp|dns|0.000334978103637695|52|99|SF||0|Dd|1|80|1|127|(empty)
+1300475168.89241|h5DsfNtYzi1|141.142.220.118|59746|141.142.2.2|53|udp|dns|0.000420808792114258|38|183|SF||0|Dd|1|66|1|211|(empty)
+1300475168.89291|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|tcp|http|0.220960855484009|1137|733|S1||0|ShADad|6|1457|4|949|(empty)
+1300475168.89294|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|tcp|http|0.229603052139282|1148|734|S1||0|ShADad|6|1468|4|950|(empty)
+1300475168.89399|c4Zw9TmAE05|141.142.220.118|45000|141.142.2.2|53|udp|dns|0.000384092330932617|38|89|SF||0|Dd|1|66|1|117|(empty)
+1300475168.89442|EAr0uf4mhq|141.142.220.118|48479|141.142.2.2|53|udp|dns|0.000316858291625977|52|99|SF||0|Dd|1|80|1|127|(empty)
+1300475168.89479|GvmoxJFXdTa|141.142.220.118|48128|141.142.2.2|53|udp|dns|0.000422954559326172|38|183|SF||0|Dd|1|66|1|211|(empty)
+1300475168.89527|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|tcp|http|0.227283954620361|1178|734|S1||0|ShADad|6|1498|4|950|(empty)
+1300475168.90175|slFea8xwSmb|141.142.220.118|56056|141.142.2.2|53|udp|dns|0.000402212142944336|36|131|SF||0|Dd|1|64|1|159|(empty)
+1300475168.9022|UfGkYA2HI2g|141.142.220.118|55092|141.142.2.2|53|udp|dns|0.000374078750610352|36|198|SF||0|Dd|1|64|1|226|(empty)
+1300475168.90264|i2rO3KD1Syg|141.142.220.118|35642|208.80.152.2|80|tcp|http|0.120040893554688|534|412|S1||0|ShADad|4|750|3|576|(empty)
+1300475169.78033|2cx26uAvUPl|141.142.220.235|6705|173.192.163.128|80|tcp|||||OTH||0|h|0|0|1|48|(empty)
+1300475169.89944|BWaU4aSuwkc|141.142.220.44|5353|224.0.0.251|5353|udp|dns||||S0||0|D|1|85|0|0|(empty)
+1300475170.86238|10XodEwRycf|141.142.220.226|137|141.142.220.255|137|udp|dns|2.61301684379578|350|0|S0||0|D|7|546|0|0|(empty)
+1300475171.67537|zno26fFZkrh|fe80::3074:17d5:2052:c324|65373|ff02::1:3|5355|udp|dns|0.100096225738525|66|0|S0||0|D|2|162|0|0|(empty)
+1300475171.67708|v5rgkJBig5l|141.142.220.226|55131|224.0.0.252|5355|udp|dns|0.100020885467529|66|0|S0||0|D|2|122|0|0|(empty)
+1300475173.11675|eWZCH7OONC1|fe80::3074:17d5:2052:c324|54213|ff02::1:3|5355|udp|dns|0.0998010635375977|66|0|S0||0|D|2|162|0|0|(empty)
+1300475173.11736|0Pwk3ntf8O3|141.142.220.226|55671|224.0.0.252|5355|udp|dns|0.0998489856719971|66|0|S0||0|D|2|122|0|0|(empty)
+1300475173.15368|0HKorjr8Zp7|141.142.220.238|56641|141.142.220.255|137|udp|dns||||S0||0|D|1|78|0|0|(empty)

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.wikipedia/http.select

@@ -0,0 +1,14 @@
+1300475168.78402|j4u32Pc5bif|141.142.220.118|48649|208.80.152.118|80|1|GET|bits.wikimedia.org|/skins-1.5/monobook/main.css|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.91602|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/6/63/Wikipedia-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.91618|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.91836|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/b/bd/Bookshelf-40x201_6.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.9523|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.95231|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.95482|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.96269|i2rO3KD1Syg|141.142.220.118|35642|208.80.152.2|80|1|GET|meta.wikimedia.org|/images/wikimedia-button.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.97593|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.97644|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.97926|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475169.01459|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475169.01462|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475169.01493|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||

testing/btest/scripts/base/frameworks/input/sqlite/basic.bro

@@ -0,0 +1,102 @@
+#
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: cat conn.sql | sqlite3 conn.sqlite
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE conn.sql
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE conn (
+'ts' double precision,
+'uid' text,
+'id.orig_h' text,
+'id.orig_p' integer,
+'id.resp_h' text,
+'id.resp_p' integer,
+'proto' text,
+'service' text,
+'duration' double precision,
+'orig_bytes' integer,
+'resp_bytes' integer,
+'conn_state' text,
+'local_orig' boolean,
+'missed_bytes' integer,
+'history' text,
+'orig_pkts' integer,
+'orig_ip_bytes' integer,
+'resp_pkts' integer,
+'resp_ip_bytes' integer,
+'tunnel_parents' text
+);
+INSERT INTO "conn" VALUES(1.30047516709653496744e+09,'dnGM1AdIVyh','141.142.220.202',5353,'224.0.0.251',5353,'udp','dns',NULL,NULL,NULL,'S0',NULL,0,'D',1,73,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516709701204296e+09,'fv9q7WjEgp1','fe80::217:f2ff:fed7:cf65',5353,'ff02::fb',5353,'udp',NULL,NULL,NULL,NULL,'S0',NULL,0,'D',1,199,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516709981608392e+09,'0Ox0H56yl88','141.142.220.50',5353,'224.0.0.251',5353,'udp',NULL,NULL,NULL,NULL,'S0',NULL,0,'D',1,179,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885389900212e+09,'rvmSc7rDQub','141.142.220.118',43927,'141.142.2.2',53,'udp','dns',4.351139068603515625e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885437798497e+09,'ogkztouSArh','141.142.220.118',37676,'141.142.2.2',53,'udp','dns',4.20093536376953125e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885483694076e+09,'0UIDdXFt7Tb','141.142.220.118',40526,'141.142.2.2',53,'udp','dns',3.9196014404296875e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885795593258e+09,'WqFYV51UIq7','141.142.220.118',32902,'141.142.2.2',53,'udp','dns',3.17096710205078125e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885830593104e+09,'ylcqZpbz6K2','141.142.220.118',59816,'141.142.2.2',53,'udp','dns',3.430843353271484375e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885871291159e+09,'blhldTzA7Y6','141.142.220.118',59714,'141.142.2.2',53,'udp','dns',3.750324249267578125e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889164400098e+09,'Sc34cGJo3Kg','141.142.220.118',58206,'141.142.2.2',53,'udp','dns',3.39031219482421875e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889203691487e+09,'RzvFrfXSRfk','141.142.220.118',38911,'141.142.2.2',53,'udp','dns',3.349781036376953125e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889241409298e+09,'GaaFI58mpbe','141.142.220.118',59746,'141.142.2.2',53,'udp','dns',4.208087921142578125e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889398789407e+09,'tr7M6tvAIQa','141.142.220.118',45000,'141.142.2.2',53,'udp','dns',3.840923309326171875e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889442205426e+09,'gV0TcSc2pb4','141.142.220.118',48479,'141.142.2.2',53,'udp','dns',3.168582916259765625e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889478707315e+09,'MOG0z4PYOhk','141.142.220.118',48128,'141.142.2.2',53,'udp','dns',4.22954559326171875e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516890174889565e+09,'PlehgEduUyj','141.142.220.118',56056,'141.142.2.2',53,'udp','dns',4.022121429443359375e-04,36,131,'SF',NULL,0,'Dd',1,64,1,159,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516890219497676e+09,'4eZgk09f2Re','141.142.220.118',55092,'141.142.2.2',53,'udp','dns',3.740787506103515625e-04,36,198,'SF',NULL,0,'Dd',1,64,1,226,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516989943790432e+09,'3xwJPc7mQ9a','141.142.220.44',5353,'224.0.0.251',5353,'udp','dns',NULL,NULL,NULL,'S0',NULL,0,'D',1,85,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047517086238408089e+09,'yxTcvvTKWQ4','141.142.220.226',137,'141.142.220.255',137,'udp','dns',2.61301684379577636718e+00,350,0,'S0',NULL,0,'D',7,546,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047517167537188525e+09,'8bLW3XNfhCj','fe80::3074:17d5:2052:c324',65373,'ff02::1:3',5355,'udp','dns',1.00096225738525390625e-01,66,0,'S0',NULL,0,'D',2,162,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047517167708110807e+09,'rqjhiiRPjEe','141.142.220.226',55131,'224.0.0.252',5355,'udp','dns',1.00020885467529296875e-01,66,0,'S0',NULL,0,'D',2,122,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047517311674904827e+09,'hTPyfL3QSGa','fe80::3074:17d5:2052:c324',54213,'ff02::1:3',5355,'udp','dns',9.980106353759765625e-02,66,0,'S0',NULL,0,'D',2,162,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047517311736202235e+09,'EruUQ9AJRj4','141.142.220.226',55671,'224.0.0.252',5355,'udp','dns',9.98489856719970703125e-02,66,0,'S0',NULL,0,'D',2,122,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047517315367889406e+09,'sw1bKJOMjuk','141.142.220.238',56641,'141.142.220.255',137,'udp','dns',NULL,NULL,NULL,'S0',NULL,0,'D',1,78,0,0,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516872400689127e+09,'NPHCuyWykE7','141.142.220.118',48649,'208.80.152.118',80,'tcp','http',1.19904994964599609375e-01,525,232,'S1',NULL,0,'ShADad',4,741,3,396,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889293599126e+09,'VapPqRhPgJ4','141.142.220.118',50000,'208.80.152.3',80,'tcp','http',2.29603052139282226562e-01,1148,734,'S1',NULL,0,'ShADad',6,1468,4,950,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885916304588e+09,'3607hh8C3bc','141.142.220.118',49998,'208.80.152.3',80,'tcp','http',2.15893030166625976562e-01,1130,734,'S1',NULL,0,'ShADad',6,1450,4,950,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516885530495647e+09,'tgYMrIvzDSg','141.142.220.118',49996,'208.80.152.3',80,'tcp','http',2.1850109100341796875e-01,1171,733,'S1',NULL,0,'ShADad',6,1491,4,949,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516889526700977e+09,'xQsjPwNBrXd','141.142.220.118',50001,'208.80.152.3',80,'tcp','http',2.27283954620361328125e-01,1178,734,'S1',NULL,0,'ShADad',6,1498,4,950,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516890263509747e+09,'Ap3GzMI1vM9','141.142.220.118',35642,'208.80.152.2',80,'tcp','http',1.200408935546875e-01,534,412,'S1',NULL,0,'ShADad',4,750,3,576,'(empty)');
+INSERT INTO "conn" VALUES(1300475168.85533,'FTVcgrmNy52','141.142.220.118',49997,'208.80.152.3',80,'tcp','http',2.19720125198364257812e-01,1125,734,'S1',NULL,0,'ShADad',6,1445,4,950,'(empty)');
+INSERT INTO "conn" VALUES(1.30047516978033089643e+09,'1xFx4PGdeq5','141.142.220.235',6705,'173.192.163.128',80,'tcp',NULL,NULL,NULL,NULL,'OTH',NULL,0,'h',0,0,1,48,'(empty)');
+INSERT INTO "conn" VALUES(1.3004751686520030498e+09,'WIG1ud65z22','141.142.220.118',35634,'208.80.152.2',80,'tcp',NULL,6.1328887939453125e-02,463,350,'OTH',NULL,0,'DdA',2,567,1,402,'(empty)');
+INSERT INTO "conn" VALUES(1.3004751688929131031e+09,'o2gAkl4V7sa','141.142.220.118',49999,'208.80.152.3',80,'tcp','http',2.20960855484008789062e-01,1137,733,'S1',NULL,0,'ShADad',6,1457,4,949,'(empty)');
+COMMIT;
+@TEST-END-FILE
+
+@load base/protocols/conn
+
+redef exit_only_after_terminate = T; 
+redef Input::accept_unsupported_types = T;
+
+global outfile: file;
+
+module A;
+
+event line(description: Input::EventDescription, tpe: Input::Event, r: Conn::Info)
+	{
+	print outfile, r;
+	print outfile, |r$tunnel_parents|; # to make sure I got empty right
+	}
+
+event bro_init()
+	{
+	local config_strings: table[string] of string = {
+		 ["query"] = "select * from conn;",
+		 ["dbname"] = "conn"
+	};
+
+	outfile = open("../out");
+	Input::add_event([$source="../conn", $name="conn", $fields=Conn::Info, $ev=line, $want_record=T, $reader=Input::READER_SQLITE, $config=config_strings]);
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, "End of data";
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/sqlite/error.bro

@@ -0,0 +1,98 @@
+# @TEST-EXEC: cat ssh.sql | sqlite3 ssh.sqlite
+#
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: sed '1d' .stderr | sort > cmpfile
+# @TEST-EXEC: btest-diff cmpfile
+
+@TEST-START-FILE ssh.sql
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE ssh (
+'b' boolean,
+'i' integer,
+'e' text,
+'c' integer,
+'p' integer,
+'sn' text,
+'a' text,
+'d' double precision,
+'t' double precision,
+'iv' double precision,
+'s' text,
+'sc' text,
+'ss' text,
+'se' text,
+'vc' text,
+'vs' text,
+'vn' text
+);
+INSERT INTO "ssh" VALUES(1,-42,'SSH::LOG',21,123,'10.0.0.0/24','1.2.3.4',3.14,1.35837684939385390286e+09,100.0,'hurz','2,4,1,3','CC,AA,BB','(empty)','10,20,30','', null);
+COMMIT;
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		b: bool;
+		i: int;
+		e: Log::ID;
+		c: count;
+		p: port;
+		sn: subnet;
+		a: addr;
+		d: double;
+		t: time;
+		iv: interval;
+		s: string;
+		sc: set[count];
+		ss: set[string];
+		se: set[string];
+		vc: vector of count;
+		vs: vector of string;
+		vh: vector of string &optional;
+	} &log;
+}
+
+
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, p: SSH::Log)
+	{
+	print outfile, p;
+
+	print outfile, |p$se|;
+	print outfile, |p$vs|;
+	}
+
+event term_me()
+	{
+	terminate();
+	}
+
+event bro_init()
+	{
+	local config_strings: table[string] of string = {
+		 ["query"] = "select * from ssh;",
+		 ["dbname"] = "ssh"
+	};
+	
+	local config_strings2: table[string] of string = {
+		 ["query"] = "select b, g, h from ssh;",
+		 ["dbname"] = "ssh"
+	};
+
+	outfile = open("../out");
+	Input::add_event([$source="../ssh", $name="ssh", $fields=SSH::Log, $ev=line, $reader=Input::READER_SQLITE, $want_record=T, $config=config_strings]);
+	Input::add_event([$source="../ssh", $name="ssh2", $fields=SSH::Log, $ev=line, $reader=Input::READER_SQLITE, $want_record=T, $config=config_strings2]);
+
+	schedule +1secs { term_me() };
+
+	}

testing/btest/scripts/base/frameworks/input/sqlite/port.bro

@@ -0,0 +1,52 @@
+#
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: cat port.sql | sqlite3 port.sqlite
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE port.sql
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE port (
+'port' integer,
+'proto' text
+);
+INSERT INTO "port" VALUES(5353,'udp');
+INSERT INTO "port" VALUES(6162,'tcp');
+COMMIT;
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+global outfile: file;
+
+module A;
+
+type Val: record {
+	p: port &type_column="proto";
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, p: port)
+	{
+	print outfile, p;
+	}
+
+event bro_init()
+	{
+	local config_strings: table[string] of string = {
+		 ["query"] = "select port as p, proto from port;",
+		 ["dbname"] = "port"
+	};
+
+	outfile = open("../out");
+	Input::add_event([$source="../port", $name="port", $fields=Val, $ev=line, $reader=Input::READER_SQLITE, $want_record=F, $config=config_strings]);
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, "End of data";
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/sqlite/types.bro

@@ -0,0 +1,90 @@
+# @TEST-EXEC: cat ssh.sql | sqlite3 ssh.sqlite
+#
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE ssh.sql
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE ssh (
+'b' boolean,
+'i' integer,
+'e' text,
+'c' integer,
+'p' integer,
+'sn' text,
+'a' text,
+'d' double precision,
+'t' double precision,
+'iv' double precision,
+'s' text,
+'sc' text,
+'ss' text,
+'se' text,
+'vc' text,
+'vs' text,
+'vn' text
+);
+INSERT INTO "ssh" VALUES(1,-42,'SSH::LOG',21,123,'10.0.0.0/24','1.2.3.4',3.14,1.35837684939385390286e+09,100.0,'hurz','2,4,1,3','CC,AA,BB','(empty)','10,20,30','', null);
+COMMIT;
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		b: bool;
+		i: int;
+		e: Log::ID;
+		c: count;
+		p: port;
+		sn: subnet;
+		a: addr;
+		d: double;
+		t: time;
+		iv: interval;
+		s: string;
+		sc: set[count];
+		ss: set[string];
+		se: set[string];
+		vc: vector of count;
+		vs: vector of string;
+		vn: vector of string &optional;
+	} &log;
+}
+
+
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, p: SSH::Log)
+	{
+	print outfile, p;
+
+	print outfile, |p$se|;
+	print outfile, |p$vs|;
+	}
+
+event bro_init()
+	{
+	local config_strings: table[string] of string = {
+		 ["query"] = "select * from ssh;",
+		 ["dbname"] = "ssh"
+	};
+
+	outfile = open("../out");
+	Input::add_event([$source="../ssh", $name="ssh", $fields=SSH::Log, $ev=line, $reader=Input::READER_SQLITE, $want_record=T, $config=config_strings]);
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, "End of data";
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/logging/sqlite/error.bro

@@ -0,0 +1,106 @@
+#
+# @TEST-REQUIRES: has-writer SQLite 
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: cat ssh.sql | sqlite3 ssh.sqlite
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff .stderr
+#
+# Testing all possible types.
+#
+
+@TEST-START-FILE ssh.sql
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE ssh (
+'b' boolean,
+'i' integer,
+'e' text,
+'c' integer,
+'p' integer,
+'sn' text,
+'a' text,
+'d' double precision,
+'t' double precision,
+'iv' double precision,
+'s' text,
+'sc' text,
+'ss' text,
+'se' text,
+'vc' text,
+'ve' text
+);
+INSERT INTO "ssh" VALUES(1,-42,'SSH::LOG',21,123,'10.0.0.0/24','1.2.3.4',3.14,1.36859359634203600879e+09,100.0,'hurz','2,4,1,3','CC,AA,BB','(empty)','10,20,30','(empty)');
+COMMIT;
+@TEST-END-FILE
+
+ 
+
+redef LogSQLite::unset_field = "(unset)";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		b: bool;
+		i: int;
+		e: Log::ID;
+		c: count;
+		p: port;
+		sn: subnet;
+		a: addr;
+		d: double;
+		t: time;
+		iv: interval;
+		s: string;
+		sc: set[count];
+		ss: set[string];
+		se: set[string];
+		vc: vector of count;
+		ve: vector of string;
+		f: function(i: count) : string;
+	} &log;
+}
+
+function foo(i : count) : string
+	{
+	if ( i > 0 )
+		return "Foo";
+	else
+		return "Bar";
+	}
+
+event bro_init()
+{
+	Log::create_stream(SSH::LOG, [$columns=Log]);
+	Log::remove_filter(SSH::LOG, "default");
+
+	local filter: Log::Filter = [$name="sqlite", $path="ssh", $writer=Log::WRITER_SQLITE];
+	Log::add_filter(SSH::LOG, filter);
+
+	local empty_set: set[string];
+	local empty_vector: vector of string;
+
+	Log::write(SSH::LOG, [
+		$b=T,
+		$i=-42,
+		$e=SSH::LOG,
+		$c=21,
+		$p=123/tcp,
+		$sn=10.0.0.1/24,
+		$a=1.2.3.4,
+		$d=3.14,
+		$t=network_time(),
+		$iv=100secs,
+		$s="hurz",
+		$sc=set(1,2,3,4),
+		$ss=set("AA", "BB", "CC"),
+		$se=empty_set,
+		$vc=vector(10, 20, 30),
+		$ve=empty_vector,
+		$f=foo
+		]);
+}
+

testing/btest/scripts/base/frameworks/logging/sqlite/types.bro

@@ -0,0 +1,78 @@
+#
+# @TEST-REQUIRES: has-writer SQLite 
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: sqlite3 ssh.sqlite 'select * from ssh' > ssh.select
+# @TEST-EXEC: btest-diff ssh.select
+#
+# Testing all possible types.
+
+redef LogSQLite::unset_field = "(unset)";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		b: bool;
+		i: int;
+		e: Log::ID;
+		c: count;
+		p: port;
+		sn: subnet;
+		a: addr;
+		d: double;
+		t: time;
+		iv: interval;
+		s: string;
+		sc: set[count];
+		ss: set[string];
+		se: set[string];
+		vc: vector of count;
+		ve: vector of string;
+		f: function(i: count) : string;
+	} &log;
+}
+
+function foo(i : count) : string
+	{
+	if ( i > 0 )
+		return "Foo";
+	else
+		return "Bar";
+	}
+
+event bro_init()
+{
+	Log::create_stream(SSH::LOG, [$columns=Log]);
+	Log::remove_filter(SSH::LOG, "default");
+
+	local filter: Log::Filter = [$name="sqlite", $path="ssh", $writer=Log::WRITER_SQLITE];
+	Log::add_filter(SSH::LOG, filter);
+
+	local empty_set: set[string];
+	local empty_vector: vector of string;
+
+	Log::write(SSH::LOG, [
+		$b=T,
+		$i=-42,
+		$e=SSH::LOG,
+		$c=21,
+		$p=123/tcp,
+		$sn=10.0.0.1/24,
+		$a=1.2.3.4,
+		$d=3.14,
+		$t=network_time(),
+		$iv=100secs,
+		$s="hurz",
+		$sc=set(1,2,3,4),
+		$ss=set("AA", "BB", "CC"),
+		$se=empty_set,
+		$vc=vector(10, 20, 30),
+		$ve=empty_vector,
+		$f=foo
+		]);
+}
+

testing/btest/scripts/base/frameworks/logging/sqlite/wikipedia.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-REQUIRES: has-writer SQLite
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_SQLITE
+# @TEST-EXEC: sqlite3 conn.sqlite 'select * from conn order by ts' | sort -n > conn.select
+# @TEST-EXEC: sqlite3 http.sqlite 'select * from http order by ts' | sort -n > http.select
+# @TEST-EXEC: btest-diff conn.select
+# @TEST-EXEC: btest-diff http.select