From 363cfb850639a01d6f9b871fba7cd8ddc796d749 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 12:24:40 -0700 Subject: [PATCH 1/5] rename the dbname configuration option to tablename. Sorry for this - I noticed that I named this option quite unfortunately while writing the documentation. The patch also removes the dbname configuration option from the sqlite input reader - it was not used there at all anymore (and I did not notice that). --- src/input/readers/SQLite.cc | 12 +----------- src/logging/writers/SQLite.cc | 14 +++++++------- .../.stderr | 2 +- .../scripts/base/frameworks/input/sqlite/basic.bro | 1 - .../scripts/base/frameworks/input/sqlite/error.bro | 2 -- .../scripts/base/frameworks/input/sqlite/port.bro | 1 - .../scripts/base/frameworks/input/sqlite/types.bro | 1 - 7 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/input/readers/SQLite.cc b/src/input/readers/SQLite.cc index e600788b16..eff34593d8 100644 --- a/src/input/readers/SQLite.cc +++ b/src/input/readers/SQLite.cc @@ -78,18 +78,8 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading: string fullpath(info.source); fullpath.append(".sqlite"); - string dbname; - map::const_iterator it = info.config.find("dbname"); - if ( it == info.config.end() ) - { - MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to source %s", info.source)); - dbname = info.source; - } - else - dbname = it->second; - string query; - it = info.config.find("query"); + map::const_iterator it = info.config.find("query"); if ( it == info.config.end() ) { Error(Fmt("No query specified when setting up SQLite data source. Aborting.", info.source)); diff --git a/src/logging/writers/SQLite.cc b/src/logging/writers/SQLite.cc index 76e5b84dfb..37e3134659 100644 --- a/src/logging/writers/SQLite.cc +++ b/src/logging/writers/SQLite.cc @@ -124,16 +124,16 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, string fullpath(info.path); fullpath.append(".sqlite"); - string dbname; + string tablename; - map::const_iterator it = info.config.find("dbname"); + map::const_iterator it = info.config.find("tablename"); if ( it == info.config.end() ) { - MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to path %s", info.path)); - dbname = info.path; + MsgThread::Info(Fmt("tablename configuration option not found. Defaulting to path %s", info.path)); + tablename = info.path; } else - dbname = it->second; + tablename = it->second; if ( checkError(sqlite3_open_v2( fullpath.c_str(), @@ -145,7 +145,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, NULL)) ) return false; - string create = "CREATE TABLE IF NOT EXISTS " + dbname + " (\n"; + string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n"; //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here. for ( unsigned int i = 0; i < num_fields; ++i ) @@ -193,7 +193,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, // create the prepared statement that will be re-used forever... string insert = "VALUES ("; - string names = "INSERT INTO " + dbname + " ( "; + string names = "INSERT INTO " + tablename + " ( "; for ( unsigned int i = 0; i < num_fields; i++ ) { diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr index 96565881d3..2857d22ca9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr @@ -1,3 +1,3 @@ -ssh/Log::WRITER_SQLITE: dbname configuration option not found. Defaulting to path ssh +ssh/Log::WRITER_SQLITE: tablename configuration option not found. Defaulting to path ssh error: ssh/Log::WRITER_SQLITE: SQLite call failed: table ssh has no column named f error: ssh/Log::WRITER_SQLITE: terminating thread diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro b/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro index aa3a75ae4b..07906dab91 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro @@ -89,7 +89,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from conn;", - ["dbname"] = "conn" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/error.bro b/testing/btest/scripts/base/frameworks/input/sqlite/error.bro index c6712df99a..08938e6df5 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/error.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/error.bro @@ -83,12 +83,10 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from ssh;", - ["dbname"] = "ssh" }; local config_strings2: table[string] of string = { ["query"] = "select b, g, h from ssh;", - ["dbname"] = "ssh" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/port.bro b/testing/btest/scripts/base/frameworks/input/sqlite/port.bro index c2949b5b3e..6fc18139fe 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/port.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/port.bro @@ -39,7 +39,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select port as p, proto from port;", - ["dbname"] = "port" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/types.bro b/testing/btest/scripts/base/frameworks/input/sqlite/types.bro index ddcbefa67f..42f8717c12 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/types.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/types.bro @@ -77,7 +77,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from ssh;", - ["dbname"] = "ssh" }; outfile = open("../out"); From 4aa363b0d2c5e38ad04b0c7409538c4fc13e3b2c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 14:47:15 -0700 Subject: [PATCH 2/5] add check that the SQLite reader is only used in MANUAL reading mode --- src/input/readers/SQLite.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/input/readers/SQLite.cc b/src/input/readers/SQLite.cc index eff34593d8..d02cc17fc7 100644 --- a/src/input/readers/SQLite.cc +++ b/src/input/readers/SQLite.cc @@ -54,7 +54,7 @@ void SQLite::DoClose() } } -bool SQLite::checkError( int code ) +bool SQLite::checkError(int code) { if ( code != SQLITE_OK && code != SQLITE_DONE ) { @@ -73,6 +73,12 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading: return false; } + if ( Info().mode != MODE_MANUAL ) + { + Error("SQLite only supports manual reading mode."); + return false; + } + started = false; string fullpath(info.source); From 5c2b2195f741af13a4cb3d98e38303e765f51b63 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 15:22:52 -0700 Subject: [PATCH 3/5] First try at sqlite reader/writer documentation --- doc/frameworks/input.rst | 11 +- doc/frameworks/logging-input-sqlite.rst | 236 ++++++++++++++++++++++++ doc/frameworks/logging.rst | 1 + 3 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 doc/frameworks/logging-input-sqlite.rst diff --git a/doc/frameworks/input.rst b/doc/frameworks/input.rst index aca5091972..ef40756a26 100644 --- a/doc/frameworks/input.rst +++ b/doc/frameworks/input.rst @@ -1,4 +1,6 @@ +.. _framework-input: + =============== Input Framework =============== @@ -260,8 +262,13 @@ to optimize the speed of the input framework. It can generate arbitrary amounts of semi-random data in all Bro data types supported by the input framework. -In the future, the input framework will get support for new data sources -like, for example, different databases. +Currently, Bro supports the following readers in addition to the +aforementioned ones: + +.. toctree:: + :maxdepth: 1 + + logging-input-sqlite Add_table options ----------------- diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst new file mode 100644 index 0000000000..2a1373a232 --- /dev/null +++ b/doc/frameworks/logging-input-sqlite.rst @@ -0,0 +1,236 @@ + +============================================ +Logging to and reading from SQLite Databases +============================================ + +.. rst-class:: opening + + Starting with version 2.2, Bro features a SQLite logging writer + as well as a SQLite input reader. SQLite is a simple, file-based, + widely used SQL database system. Using SQLite allows Bro to write + and access data in a format that is easy to use in interchange with + other applications. Due to the transactional nature of SQLite, + databases can be used by several applications simultaneously. Hence, + they can, for example, be used to make data that changes regularly available + to Bro on a continuing basis. + +.. contents:: + +Warning +======= + +In contrast to the ASCII plugins, the SQLite plugins have not yet +seen extensive use in production environments. While we are not aware +of any issues with them at the moment, we urge to caution when using them +in production environments. There could be lingering issues which only occur +when the plugins are used with high amounts of data or in high-load environments. + +Logging Data into SQLite Databases +================================== + +Logging support for SQLite is available in all Bro installations starting with +version 2.2. There is no need to load any additional scripts or for any compile-time +configurations. + +Sending data from existing logging streams to SQLite is rather straightforward. You +have to define a filter which specifies SQLite as the writer. + +The following example code adds SQLite as a filter for the connection log: + +.. code:: bro + + event bro_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } + +Bro will create the database file ``/var/db/conn.sqlite``, if it does not already exist. +It will also create a table with the name ``conn`` (if it does not exist) and start +appending connection information to the table. + +At the moment, SQLite databases are not rotated the same way ASCII log-files are. You +have to take care to create them in an adequate location. + +Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a +log stream, you can remove the default filter: + +.. code:: bro + + Log::remove_filter(Conn::LOG, "default"); + +If you examine the resulting SQLite database, the schema will contain the same fields +that are present in the ASCII log files:: + + # sqlite3 /var/db/conn.sqlite + + SQLite version 3.8.0.2 2013-09-03 17:11:13 + Enter ".help" for instructions + Enter SQL statements terminated with a ";" + sqlite> .schema + CREATE TABLE conn ( + 'ts' double precision, + 'uid' text, + 'id.orig_h' text, + 'id.orig_p' integer, + ... + +To create a custom SQLite log file, you have to create a new log stream that contains +just the information you want to commit to the database. Please refer to the +:ref:`framework-logging` documentation. + +Reading Data from SQLite Databases +================================== + +Like logging support, support for reading data from SQLite databases is built into Bro starting +with version 2.2. + +Just as with the text-based input readers (please refer to the :ref:`framework-input` +documentation for them), the SQLite reader can be used to read data - in this case the result of +SQL queries - into tables or into events. + +Reading data into Tables +------------------------ + +To read data from a SQLite database, we first have to provide Bro with the information, how +the resulting data will be structured. For this example, we expect that we have a SQLite database, +which contains host IP addresses and the user accounts that are allowed to log into a specific +machine. + +The SQLite commands to create the schema are as follows:: + + create table machines_to_users ( + host text unique not null, + users text not null); + + insert into machines_to_users values ('192.168.17.1', 'bernhard,matthias,seth'); + insert into machines_to_users values ('192.168.17.2', 'bernhard'); + insert into machines_to_users values ('192.168.17.3', 'seth,matthias'); + +After creating a file called ``hosts.sqlite`` with this content, we can read the resulting table +into Bro: + +.. code:: bro + + type Idx: record { + host: addr; + }; + + type Val: record { + users: set[string]; + }; + + global hostslist: table[addr] of Val = table(); + + event bro_init() { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + + event Input::end_of_data(name: string, source: string) { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } + +Afterwards, that table can be used to check logins into hosts against the available +userlist. + +Reading data to Events +---------------------- + +The second mode is to use the SQLite reader to output the input data to events. Typically there +are two reasons to do this. First, when the structure of the input data is too complicated +for a direct table import. In this case, the data can be read into an event which can then +create the necessary data structures in Bro in scriptland. + +The second reason is, that the dataset is too big to hold it in memory. In this case, the checks +can be performed on-demand, when Bro encounters a situation where it needs additional information. + +An example for this would be an internal huge database with malware hashes. Live database queries +could be used to check the sporadically happening downloads against the database. + +The SQLite commands to create the schema are as follows:: + + create table malware_hashes ( + hash text unique not null, + description text not null); + + insert into malware_hashes values ('86f7e437faa5a7fce15d1ddcb9eaeaea377667b8', 'malware a'); + insert into malware_hashes values ('e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98', 'malware b'); + insert into malware_hashes values ('84a516841ba77a5b4648de2cd0dfcb30ea46dbb4', 'malware c'); + insert into malware_hashes values ('3c363836cf4e16666669a25da280a1865c2d2874', 'malware d'); + insert into malware_hashes values ('58e6b3a414a1e090dfc6029add0f3555ccba127f', 'malware e'); + insert into malware_hashes values ('4a0a19218e082a343a1b17e5333409af9d98f0f5', 'malware f'); + insert into malware_hashes values ('54fd1711209fb1c0781092374132c66e79e2241b', 'malware g'); + insert into malware_hashes values ('27d5482eebd075de44389774fce28c69f45c8a75', 'malware h'); + insert into malware_hashes values ('73f45106968ff8dc51fba105fa91306af1ff6666', 'ftp-trace'); + + +The following code uses the file-analysis framework to get the sha1 hashes of files that are +transmitted over the network. For each hash, a SQL-query is run against SQLite. If the query +returns with a result, we had a hit against our malware-database and output the matching hash. + +.. code:: bro + + @load frameworks/files/hash-all-files + + type Val: record { + hash: string; + description: string; + }; + + event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hot with hash %s, description %s", r$hash, r$description); + } + + global malware_source = "/var/db/malware"; + + event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $reader=Input::READER_SQLITE, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ) + ]); + } + } + + event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } + diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index 93cbf24e84..eb64df4ec3 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -387,3 +387,4 @@ Bro supports the following output formats other than ASCII: logging-dataseries logging-elasticsearch + logging-input-sqlite From dc685bbef3c3a51b2595481b38abe74676706fad Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 15:52:00 -0700 Subject: [PATCH 4/5] and restructure it a bit --- doc/frameworks/logging-input-sqlite.rst | 33 +++++++++++++------------ 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 2a1373a232..7c8c848c5f 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -19,9 +19,9 @@ Logging to and reading from SQLite Databases Warning ======= -In contrast to the ASCII plugins, the SQLite plugins have not yet +In contrast to the ASCII reader and writer, the SQLite plugins have not yet seen extensive use in production environments. While we are not aware -of any issues with them at the moment, we urge to caution when using them +of any issues with them, we urge to caution when using them in production environments. There could be lingering issues which only occur when the plugins are used with high amounts of data or in high-load environments. @@ -59,13 +59,6 @@ appending connection information to the table. At the moment, SQLite databases are not rotated the same way ASCII log-files are. You have to take care to create them in an adequate location. -Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a -log stream, you can remove the default filter: - -.. code:: bro - - Log::remove_filter(Conn::LOG, "default"); - If you examine the resulting SQLite database, the schema will contain the same fields that are present in the ASCII log files:: @@ -81,10 +74,18 @@ that are present in the ASCII log files:: 'id.orig_h' text, 'id.orig_p' integer, ... - + +Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a +log stream, you can remove the default filter: + +.. code:: bro + + Log::remove_filter(Conn::LOG, "default"); + + To create a custom SQLite log file, you have to create a new log stream that contains just the information you want to commit to the database. Please refer to the -:ref:`framework-logging` documentation. +:ref:`framework-logging` documentation on how to create custom log streams. Reading Data from SQLite Databases ================================== @@ -93,8 +94,8 @@ Like logging support, support for reading data from SQLite databases is built in with version 2.2. Just as with the text-based input readers (please refer to the :ref:`framework-input` -documentation for them), the SQLite reader can be used to read data - in this case the result of -SQL queries - into tables or into events. +documentation for them and for basic information on how to use the input-framework), the SQLite reader +can be used to read data - in this case the result of SQL queries - into tables or into events. Reading data into Tables ------------------------ @@ -202,7 +203,7 @@ returns with a result, we had a hit against our malware-database and output the event line(description: Input::EventDescription, tpe: Input::Event, r: Val) { - print fmt("malware-hot with hash %s, description %s", r$hash, r$description); + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); } global malware_source = "/var/db/malware"; @@ -220,10 +221,10 @@ returns with a result, we had a hit against our malware-database and output the $fields=Val, $ev=line, $want_record=T, - $reader=Input::READER_SQLITE, $config=table( ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) - ) + ), + $reader=Input::READER_SQLITE ]); } } From 613a04d1761d0b5c3fbfbe7531bffdff2a41911e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 16:00:22 -0700 Subject: [PATCH 5/5] and provide a bit of motivation to try the last example. --- doc/frameworks/logging-input-sqlite.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 7c8c848c5f..370c8d8573 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -235,3 +235,5 @@ returns with a result, we had a hit against our malware-database and output the Input::remove(name); } +If you run this script against the trace in ``testing/btest/Traces/ftp/ipv4.trace``, you +will get one hit.