diff --git a/CHANGES b/CHANGES index e149b3f4d0..c48febd8fe 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,17 @@ +2.2-beta-80 | 2013-10-18 13:18:05 -0700 + + * SQLite reader/writer documentation. (Bernhard Amann) + + * Check that the SQLite reader is only used in MANUAL reading mode. + (Bernhard Amann) + + * Rename the SQLite writer "dbname" configuration option to + "tablename". (Bernhard Amann) + + * Remove the "dbname" configuration option from the SQLite reader + where it wasn't used. (Bernhard Amann) + 2.2-beta-73 | 2013-10-14 14:28:25 -0700 * Fix misc. Coverity-reported issues (leaks, potential null pointer diff --git a/VERSION b/VERSION index 82d1470bc0..1ec4ef6d6a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-beta-73 +2.2-beta-80 diff --git a/doc/frameworks/input.rst b/doc/frameworks/input.rst index aca5091972..ef40756a26 100644 --- a/doc/frameworks/input.rst +++ b/doc/frameworks/input.rst @@ -1,4 +1,6 @@ +.. _framework-input: + =============== Input Framework =============== @@ -260,8 +262,13 @@ to optimize the speed of the input framework. It can generate arbitrary amounts of semi-random data in all Bro data types supported by the input framework. -In the future, the input framework will get support for new data sources -like, for example, different databases. +Currently, Bro supports the following readers in addition to the +aforementioned ones: + +.. toctree:: + :maxdepth: 1 + + logging-input-sqlite Add_table options ----------------- diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst new file mode 100644 index 0000000000..47ee1fa23f --- /dev/null +++ b/doc/frameworks/logging-input-sqlite.rst @@ -0,0 +1,239 @@ + +============================================ +Logging To and Reading From SQLite Databases +============================================ + +.. rst-class:: opening + + Starting with version 2.2, Bro features a SQLite logging writer + as well as a SQLite input reader. SQLite is a simple, file-based, + widely used SQL database system. Using SQLite allows Bro to write + and access data in a format that is easy to use in interchange with + other applications. Due to the transactional nature of SQLite, + databases can be used by several applications simultaneously. Hence, + they can, for example, be used to make data that changes regularly available + to Bro on a continuing basis. + +.. contents:: + +Warning +======= + +In contrast to the ASCII reader and writer, the SQLite plugins have not yet +seen extensive use in production environments. While we are not aware +of any issues with them, we urge to caution when using them +in production environments. There could be lingering issues which only occur +when the plugins are used with high amounts of data or in high-load environments. + +Logging Data into SQLite Databases +================================== + +Logging support for SQLite is available in all Bro installations starting with +version 2.2. There is no need to load any additional scripts or for any compile-time +configurations. + +Sending data from existing logging streams to SQLite is rather straightforward. You +have to define a filter which specifies SQLite as the writer. + +The following example code adds SQLite as a filter for the connection log: + +.. code:: bro + + event bro_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } + +Bro will create the database file ``/var/db/conn.sqlite``, if it does not already exist. +It will also create a table with the name ``conn`` (if it does not exist) and start +appending connection information to the table. + +At the moment, SQLite databases are not rotated the same way ASCII log-files are. You +have to take care to create them in an adequate location. + +If you examine the resulting SQLite database, the schema will contain the same fields +that are present in the ASCII log files:: + + # sqlite3 /var/db/conn.sqlite + + SQLite version 3.8.0.2 2013-09-03 17:11:13 + Enter ".help" for instructions + Enter SQL statements terminated with a ";" + sqlite> .schema + CREATE TABLE conn ( + 'ts' double precision, + 'uid' text, + 'id.orig_h' text, + 'id.orig_p' integer, + ... + +Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a +log stream, you can remove the default filter: + +.. code:: bro + + Log::remove_filter(Conn::LOG, "default"); + + +To create a custom SQLite log file, you have to create a new log stream that contains +just the information you want to commit to the database. Please refer to the +:ref:`framework-logging` documentation on how to create custom log streams. + +Reading Data from SQLite Databases +================================== + +Like logging support, support for reading data from SQLite databases is built into Bro starting +with version 2.2. + +Just as with the text-based input readers (please refer to the :ref:`framework-input` +documentation for them and for basic information on how to use the input-framework), the SQLite reader +can be used to read data - in this case the result of SQL queries - into tables or into events. + +Reading Data into Tables +------------------------ + +To read data from a SQLite database, we first have to provide Bro with the information, how +the resulting data will be structured. For this example, we expect that we have a SQLite database, +which contains host IP addresses and the user accounts that are allowed to log into a specific +machine. + +The SQLite commands to create the schema are as follows:: + + create table machines_to_users ( + host text unique not null, + users text not null); + + insert into machines_to_users values ('192.168.17.1', 'bernhard,matthias,seth'); + insert into machines_to_users values ('192.168.17.2', 'bernhard'); + insert into machines_to_users values ('192.168.17.3', 'seth,matthias'); + +After creating a file called ``hosts.sqlite`` with this content, we can read the resulting table +into Bro: + +.. code:: bro + + type Idx: record { + host: addr; + }; + + type Val: record { + users: set[string]; + }; + + global hostslist: table[addr] of Val = table(); + + event bro_init() { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + + event Input::end_of_data(name: string, source: string) { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } + +Afterwards, that table can be used to check logins into hosts against the available +userlist. + +Turning Data into Events +------------------------ + +The second mode is to use the SQLite reader to output the input data as events. Typically there +are two reasons to do this. First, when the structure of the input data is too complicated +for a direct table import. In this case, the data can be read into an event which can then +create the necessary data structures in Bro in scriptland. + +The second reason is, that the dataset is too big to hold it in memory. In this case, the checks +can be performed on-demand, when Bro encounters a situation where it needs additional information. + +An example for this would be an internal huge database with malware hashes. Live database queries +could be used to check the sporadically happening downloads against the database. + +The SQLite commands to create the schema are as follows:: + + create table malware_hashes ( + hash text unique not null, + description text not null); + + insert into malware_hashes values ('86f7e437faa5a7fce15d1ddcb9eaeaea377667b8', 'malware a'); + insert into malware_hashes values ('e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98', 'malware b'); + insert into malware_hashes values ('84a516841ba77a5b4648de2cd0dfcb30ea46dbb4', 'malware c'); + insert into malware_hashes values ('3c363836cf4e16666669a25da280a1865c2d2874', 'malware d'); + insert into malware_hashes values ('58e6b3a414a1e090dfc6029add0f3555ccba127f', 'malware e'); + insert into malware_hashes values ('4a0a19218e082a343a1b17e5333409af9d98f0f5', 'malware f'); + insert into malware_hashes values ('54fd1711209fb1c0781092374132c66e79e2241b', 'malware g'); + insert into malware_hashes values ('27d5482eebd075de44389774fce28c69f45c8a75', 'malware h'); + insert into malware_hashes values ('73f45106968ff8dc51fba105fa91306af1ff6666', 'ftp-trace'); + + +The following code uses the file-analysis framework to get the sha1 hashes of files that are +transmitted over the network. For each hash, a SQL-query is run against SQLite. If the query +returns with a result, we had a hit against our malware-database and output the matching hash. + +.. code:: bro + + @load frameworks/files/hash-all-files + + type Val: record { + hash: string; + description: string; + }; + + event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); + } + + global malware_source = "/var/db/malware"; + + event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ), + $reader=Input::READER_SQLITE + ]); + } + } + + event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } + +If you run this script against the trace in ``testing/btest/Traces/ftp/ipv4.trace``, you +will get one hit. diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index 93cbf24e84..eb64df4ec3 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -387,3 +387,4 @@ Bro supports the following output formats other than ASCII: logging-dataseries logging-elasticsearch + logging-input-sqlite diff --git a/src/input/readers/SQLite.cc b/src/input/readers/SQLite.cc index e600788b16..d02cc17fc7 100644 --- a/src/input/readers/SQLite.cc +++ b/src/input/readers/SQLite.cc @@ -54,7 +54,7 @@ void SQLite::DoClose() } } -bool SQLite::checkError( int code ) +bool SQLite::checkError(int code) { if ( code != SQLITE_OK && code != SQLITE_DONE ) { @@ -73,23 +73,19 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading: return false; } + if ( Info().mode != MODE_MANUAL ) + { + Error("SQLite only supports manual reading mode."); + return false; + } + started = false; string fullpath(info.source); fullpath.append(".sqlite"); - string dbname; - map::const_iterator it = info.config.find("dbname"); - if ( it == info.config.end() ) - { - MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to source %s", info.source)); - dbname = info.source; - } - else - dbname = it->second; - string query; - it = info.config.find("query"); + map::const_iterator it = info.config.find("query"); if ( it == info.config.end() ) { Error(Fmt("No query specified when setting up SQLite data source. Aborting.", info.source)); diff --git a/src/logging/writers/SQLite.cc b/src/logging/writers/SQLite.cc index 76e5b84dfb..37e3134659 100644 --- a/src/logging/writers/SQLite.cc +++ b/src/logging/writers/SQLite.cc @@ -124,16 +124,16 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, string fullpath(info.path); fullpath.append(".sqlite"); - string dbname; + string tablename; - map::const_iterator it = info.config.find("dbname"); + map::const_iterator it = info.config.find("tablename"); if ( it == info.config.end() ) { - MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to path %s", info.path)); - dbname = info.path; + MsgThread::Info(Fmt("tablename configuration option not found. Defaulting to path %s", info.path)); + tablename = info.path; } else - dbname = it->second; + tablename = it->second; if ( checkError(sqlite3_open_v2( fullpath.c_str(), @@ -145,7 +145,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, NULL)) ) return false; - string create = "CREATE TABLE IF NOT EXISTS " + dbname + " (\n"; + string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n"; //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here. for ( unsigned int i = 0; i < num_fields; ++i ) @@ -193,7 +193,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, // create the prepared statement that will be re-used forever... string insert = "VALUES ("; - string names = "INSERT INTO " + dbname + " ( "; + string names = "INSERT INTO " + tablename + " ( "; for ( unsigned int i = 0; i < num_fields; i++ ) { diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr index 96565881d3..2857d22ca9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr @@ -1,3 +1,3 @@ -ssh/Log::WRITER_SQLITE: dbname configuration option not found. Defaulting to path ssh +ssh/Log::WRITER_SQLITE: tablename configuration option not found. Defaulting to path ssh error: ssh/Log::WRITER_SQLITE: SQLite call failed: table ssh has no column named f error: ssh/Log::WRITER_SQLITE: terminating thread diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro b/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro index aa3a75ae4b..07906dab91 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro @@ -89,7 +89,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from conn;", - ["dbname"] = "conn" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/error.bro b/testing/btest/scripts/base/frameworks/input/sqlite/error.bro index c6712df99a..08938e6df5 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/error.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/error.bro @@ -83,12 +83,10 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from ssh;", - ["dbname"] = "ssh" }; local config_strings2: table[string] of string = { ["query"] = "select b, g, h from ssh;", - ["dbname"] = "ssh" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/port.bro b/testing/btest/scripts/base/frameworks/input/sqlite/port.bro index c2949b5b3e..6fc18139fe 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/port.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/port.bro @@ -39,7 +39,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select port as p, proto from port;", - ["dbname"] = "port" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/types.bro b/testing/btest/scripts/base/frameworks/input/sqlite/types.bro index ddcbefa67f..42f8717c12 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/types.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/types.bro @@ -77,7 +77,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from ssh;", - ["dbname"] = "ssh" }; outfile = open("../out");