diff --git a/.gitmodules b/.gitmodules index 4998cc6b80..24375ce23d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "src/3rdparty"] path = src/3rdparty url = git://git.bro.org/bro-3rdparty +[submodule "aux/plugins"] + path = aux/plugins + url = git://git.bro.org/bro-plugins diff --git a/aux/bro-aux b/aux/bro-aux index 8bd5f849c8..9ffdd276f9 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 8bd5f849c8f0e1c1e8397c3ad678eb8d3532fb24 +Subproject commit 9ffdd276f9c60db6fecff36751a15cdaec75ca4f diff --git a/aux/plugins b/aux/plugins new file mode 160000 index 0000000000..43b7ac7b4a --- /dev/null +++ b/aux/plugins @@ -0,0 +1 @@ +Subproject commit 43b7ac7b4aa192b8e2595c55192222cef057e65a diff --git a/cmake b/cmake index f2e8ba6b90..99486bfe54 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit f2e8ba6b90b3a2da9f1f77c55d0e718c25376bbb +Subproject commit 99486bfe5430d04169297b4e4debd5078f0a435f diff --git a/doc/frameworks/logging-dataseries.rst b/doc/frameworks/logging-dataseries.rst deleted file mode 100644 index cc479eae76..0000000000 --- a/doc/frameworks/logging-dataseries.rst +++ /dev/null @@ -1,186 +0,0 @@ - -============================= -Binary Output with DataSeries -============================= - -.. rst-class:: opening - - Bro's default ASCII log format is not exactly the most efficient - way for storing and searching large volumes of data. An an - alternative, Bro comes with experimental support for `DataSeries - `_ - output, an efficient binary format for recording structured bulk - data. DataSeries is developed and maintained at HP Labs. - -.. contents:: - -Installing DataSeries ---------------------- - -To use DataSeries, its libraries must be available at compile-time, -along with the supporting *Lintel* package. Generally, both are -distributed on `HP Labs' web site -`_. Currently, however, you need -to use recent development versions for both packages, which you can -download from github like this:: - - git clone http://github.com/dataseries/Lintel - git clone http://github.com/dataseries/DataSeries - -To build and install the two into ````, do:: - - ( cd Lintel && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX= .. && make && make install ) - ( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX= .. && make && make install ) - -Please refer to the packages' documentation for more information about -the installation process. In particular, there's more information on -required and optional `dependencies for Lintel -`_ -and `dependencies for DataSeries -`_. -For users on RedHat-style systems, you'll need the following:: - - yum install libxml2-devel boost-devel - -Compiling Bro with DataSeries Support -------------------------------------- - -Once you have installed DataSeries, Bro's ``configure`` should pick it -up automatically as long as it finds it in a standard system location. -Alternatively, you can specify the DataSeries installation prefix -manually with ``--with-dataseries=``. Keep an eye on -``configure``'s summary output, if it looks like the following, Bro -found DataSeries and will compile in the support:: - - # ./configure --with-dataseries=/usr/local - [...] - ====================| Bro Build Summary |===================== - [...] - DataSeries: true - [...] - ================================================================ - -Activating DataSeries ---------------------- - -The direct way to use DataSeries is to switch *all* log files over to -the binary format. To do that, just add ``redef -Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``. -For testing, you can also just pass that on the command line:: - - bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES - -With that, Bro will now write all its output into DataSeries files -``*.ds``. You can inspect these using DataSeries's set of command line -tools, which its installation process installs into ``/bin``. -For example, to convert a file back into an ASCII representation:: - - $ ds2txt conn.log - [... We skip a bunch of metadata here ...] - ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes - 1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0 - 1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0 - 1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0 - 1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117 - 1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127 - 1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211 - [...] - -(``--skip-all`` suppresses the metadata.) - -Note that the ASCII conversion is *not* equivalent to Bro's default -output format. - -You can also switch only individual files over to DataSeries by adding -code like this to your ``local.bro``: - -.. code:: bro - - event bro_init() - { - local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log. - f$writer = Log::WRITER_DATASERIES; # Change writer type. - Log::add_filter(Conn::LOG, f); # Replace filter with adapted version. - } - -Bro's DataSeries writer comes with a few tuning options, see -:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`. - -Working with DataSeries -======================= - -Here are a few examples of using DataSeries command line tools to work -with the output files. - -* Printing CSV:: - - $ ds2txt --csv conn.log - ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes - 1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0 - 1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0 - 1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0 - 1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0 - [...] - - Add ``--separator=X`` to set a different separator. - -* Extracting a subset of columns:: - - $ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log - 1258790493.773208 192.168.1.255 137 - 1258790451.402091 192.168.1.255 138 - 1258790493.787448 192.168.1.255 138 - 1258790615.268111 192.168.1.255 137 - 1258790615.289842 192.168.1.255 138 - [...] - -* Filtering rows:: - - $ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all conn.ds - 1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp 15.004568 798 0 S0 F 0 D 6 966 0 0 - 1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp 15.004581 798 0 S0 F 0 D 6 966 0 0 - 1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp 15.005071 798 0 S0 F 0 D 6 966 0 0 - 1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp 15.004824 798 0 S0 F 0 D 6 966 0 0 - 1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp 15.005078 798 0 S0 F 0 D 6 966 0 0 - 1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp 15.004082 798 0 S0 F 0 D 6 966 0 0 - [...] - -* Calculate some statistics: - - Mean/stddev/min/max over a column:: - - $ dsstatgroupby '*' basic duration from conn.ds - # Begin DSStatGroupByModule - # processed 2159 rows, where clause eliminated 0 rows - # count(*), mean(duration), stddev, min, max - 2159, 42.7938, 1858.34, 0, 86370 - [...] - - Quantiles of total connection volume:: - - $ dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds - [...] - 2159 data points, mean 24616 +- 343295 [0,1.26615e+07] - quantiles about every 216 data points: - 10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469 - tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262 - [...] - -The ``man`` pages for these tools show further options, and their -``-h`` option gives some more information (either can be a bit cryptic -unfortunately though). - -Deficiencies ------------- - -Due to limitations of the DataSeries format, one cannot inspect its -files before they have been fully written. In other words, when using -DataSeries, it's currently not possible to inspect the live log -files inside the spool directory before they are rotated to their -final location. It seems that this could be fixed with some effort, -and we will work with DataSeries development team on that if the -format gains traction among Bro users. - -Likewise, we're considering writing custom command line tools for -interacting with DataSeries files, making that a bit more convenient -than what the standard utilities provide. diff --git a/doc/frameworks/logging-elasticsearch.rst b/doc/frameworks/logging-elasticsearch.rst deleted file mode 100644 index 3f1eac859d..0000000000 --- a/doc/frameworks/logging-elasticsearch.rst +++ /dev/null @@ -1,89 +0,0 @@ - -========================================= -Indexed Logging Output with ElasticSearch -========================================= - -.. rst-class:: opening - - Bro's default ASCII log format is not exactly the most efficient - way for searching large volumes of data. ElasticSearch - is a new data storage technology for dealing with tons of data. - It's also a search engine built on top of Apache's Lucene - project. It scales very well, both for distributed indexing and - distributed searching. - -.. contents:: - -Warning -------- - -This writer plugin is still in testing and is not yet recommended for -production use! The approach to how logs are handled in the plugin is "fire -and forget" at this time, there is no error handling if the server fails to -respond successfully to the insertion request. - -Installing ElasticSearch ------------------------- - -Download the latest version from: http://www.elasticsearch.org/download/. -Once extracted, start ElasticSearch with:: - -# ./bin/elasticsearch - -For more detailed information, refer to the ElasticSearch installation -documentation: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html - -Compiling Bro with ElasticSearch Support ----------------------------------------- - -First, ensure that you have libcurl installed then run configure:: - - # ./configure - [...] - ====================| Bro Build Summary |===================== - [...] - cURL: true - [...] - ElasticSearch: true - [...] - ================================================================ - -Activating ElasticSearch ------------------------- - -The easiest way to enable ElasticSearch output is to load the -tuning/logs-to-elasticsearch.bro script. If you are using BroControl, -the following line in local.bro will enable it: - -.. console:: - - @load tuning/logs-to-elasticsearch - -With that, Bro will now write most of its logs into ElasticSearch in addition -to maintaining the Ascii logs like it would do by default. That script has -some tunable options for choosing which logs to send to ElasticSearch, refer -to the autogenerated script documentation for those options. - -There is an interface being written specifically to integrate with the data -that Bro outputs into ElasticSearch named Brownian. It can be found here:: - - https://github.com/grigorescu/Brownian - -Tuning ------- - -A common problem encountered with ElasticSearch is too many files being held -open. The ElasticSearch website has some suggestions on how to increase the -open file limit. - - - http://www.elasticsearch.org/tutorials/too-many-open-files/ - -TODO ----- - -Lots. - -- Perform multicast discovery for server. -- Better error detection. -- Better defaults (don't index loaded-plugins, for instance). -- diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index 47d3338e8a..e5990fea72 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -380,11 +380,11 @@ uncommon to need to delete that data before the end of the connection. Other Writers ------------- -Bro supports the following output formats other than ASCII: +Bro supports the following built-in output formats other than ASCII: .. toctree:: :maxdepth: 1 - logging-dataseries - logging-elasticsearch logging-input-sqlite + +Further formats are available as external plugins. diff --git a/scripts/base/frameworks/logging/__load__.bro b/scripts/base/frameworks/logging/__load__.bro index 44293de5cb..74c7362846 100644 --- a/scripts/base/frameworks/logging/__load__.bro +++ b/scripts/base/frameworks/logging/__load__.bro @@ -1,7 +1,5 @@ @load ./main @load ./postprocessors @load ./writers/ascii -@load ./writers/dataseries @load ./writers/sqlite -@load ./writers/elasticsearch @load ./writers/none diff --git a/scripts/base/frameworks/logging/writers/dataseries.bro b/scripts/base/frameworks/logging/writers/dataseries.bro deleted file mode 100644 index b24601d6b9..0000000000 --- a/scripts/base/frameworks/logging/writers/dataseries.bro +++ /dev/null @@ -1,60 +0,0 @@ -##! Interface for the DataSeries log writer. - -module LogDataSeries; - -export { - ## Compression to use with the DS output file. Options are: - ## - ## 'none' -- No compression. - ## 'lzf' -- LZF compression (very quick, but leads to larger output files). - ## 'lzo' -- LZO compression (very fast decompression times). - ## 'zlib' -- GZIP compression (slower than LZF, but also produces smaller output). - ## 'bz2' -- BZIP2 compression (slower than GZIP, but also produces smaller output). - const compression = "zlib" &redef; - - ## The extent buffer size. - ## Larger values here lead to better compression and more efficient writes, - ## but also increase the lag between the time events are received and - ## the time they are actually written to disk. - const extent_size = 65536 &redef; - - ## Should we dump the XML schema we use for this DS file to disk? - ## If yes, the XML schema shares the name of the logfile, but has - ## an XML ending. - const dump_schema = F &redef; - - ## How many threads should DataSeries spawn to perform compression? - ## Note that this dictates the number of threads per log stream. If - ## you're using a lot of streams, you may want to keep this number - ## relatively small. - ## - ## Default value is 1, which will spawn one thread / stream. - ## - ## Maximum is 128, minimum is 1. - const num_threads = 1 &redef; - - ## Should time be stored as an integer or a double? - ## Storing time as a double leads to possible precision issues and - ## can (significantly) increase the size of the resulting DS log. - ## That said, timestamps stored in double form are consistent - ## with the rest of Bro, including the standard ASCII log. Hence, we - ## use them by default. - const use_integer_for_time = F &redef; -} - -# Default function to postprocess a rotated DataSeries log file. It moves the -# rotated file to a new name that includes a timestamp with the opening time, -# and then runs the writer's default postprocessor command on it. -function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool - { - # Move file to name including both opening and closing time. - local dst = fmt("%s.%s.ds", info$path, - strftime(Log::default_rotation_date_format, info$open)); - - system(fmt("/bin/mv %s %s", info$fname, dst)); - - # Run default postprocessor. - return Log::run_rotation_postprocessor_cmd(info, dst); - } - -redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func }; diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro deleted file mode 100644 index 6292876bd0..0000000000 --- a/scripts/base/frameworks/logging/writers/elasticsearch.bro +++ /dev/null @@ -1,48 +0,0 @@ -##! Log writer for sending logs to an ElasticSearch server. -##! -##! Note: This module is in testing and is not yet considered stable! -##! -##! There is one known memory issue. If your elasticsearch server is -##! running slowly and taking too long to return from bulk insert -##! requests, the message queue to the writer thread will continue -##! growing larger and larger giving the appearance of a memory leak. - -module LogElasticSearch; - -export { - ## Name of the ES cluster. - const cluster_name = "elasticsearch" &redef; - - ## ES server. - const server_host = "127.0.0.1" &redef; - - ## ES port. - const server_port = 9200 &redef; - - ## Name of the ES index. - const index_prefix = "bro" &redef; - - ## The ES type prefix comes before the name of the related log. - ## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc. - const type_prefix = "" &redef; - - ## The time before an ElasticSearch transfer will timeout. Note that - ## the fractional part of the timeout will be ignored. In particular, - ## time specifications less than a second result in a timeout value of - ## 0, which means "no timeout." - const transfer_timeout = 2secs; - - ## The batch size is the number of messages that will be queued up before - ## they are sent to be bulk indexed. - const max_batch_size = 1000 &redef; - - ## The maximum amount of wall-clock time that is allowed to pass without - ## finishing a bulk log send. This represents the maximum delay you - ## would like to have with your logs before they are sent to ElasticSearch. - const max_batch_interval = 1min &redef; - - ## The maximum byte size for a buffered JSON string to send to the bulk - ## insert API. - const max_byte_size = 1024 * 1024 &redef; -} - diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro deleted file mode 100644 index b770b8f84b..0000000000 --- a/scripts/policy/tuning/logs-to-elasticsearch.bro +++ /dev/null @@ -1,36 +0,0 @@ -##! Load this script to enable global log output to an ElasticSearch database. - -module LogElasticSearch; - -export { - ## An elasticsearch specific rotation interval. - const rotation_interval = 3hr &redef; - - ## Optionally ignore any :bro:type:`Log::ID` from being sent to - ## ElasticSearch with this script. - const excluded_log_ids: set[Log::ID] &redef; - - ## If you want to explicitly only send certain :bro:type:`Log::ID` - ## streams, add them to this set. If the set remains empty, all will - ## be sent. The :bro:id:`LogElasticSearch::excluded_log_ids` option - ## will remain in effect as well. - const send_logs: set[Log::ID] &redef; -} - -event bro_init() &priority=-5 - { - if ( server_host == "" ) - return; - - for ( stream_id in Log::active_streams ) - { - if ( stream_id in excluded_log_ids || - (|send_logs| > 0 && stream_id !in send_logs) ) - next; - - local filter: Log::Filter = [$name = "default-es", - $writer = Log::WRITER_ELASTICSEARCH, - $interv = LogElasticSearch::rotation_interval]; - Log::add_filter(stream_id, filter); - } - } diff --git a/src/logging/writers/CMakeLists.txt b/src/logging/writers/CMakeLists.txt index 9718a412d0..867ad58c47 100644 --- a/src/logging/writers/CMakeLists.txt +++ b/src/logging/writers/CMakeLists.txt @@ -1,6 +1,4 @@ add_subdirectory(ascii) -add_subdirectory(dataseries) -add_subdirectory(elasticsearch) add_subdirectory(none) add_subdirectory(sqlite) diff --git a/src/logging/writers/dataseries/CMakeLists.txt b/src/logging/writers/dataseries/CMakeLists.txt deleted file mode 100644 index f7f54e4c0f..0000000000 --- a/src/logging/writers/dataseries/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ - -include(BroPlugin) - -find_package(Lintel) -find_package(DataSeries) -find_package(LibXML2) - -if (NOT DISABLE_DATASERIES AND - LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND) - - include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - - include_directories(BEFORE ${Lintel_INCLUDE_DIR}) - include_directories(BEFORE ${DataSeries_INCLUDE_DIR}) - include_directories(BEFORE ${LibXML2_INCLUDE_DIR}) - - bro_plugin_begin(Bro DataSeriesWriter) - bro_plugin_cc(DataSeries.cc Plugin.cc) - bro_plugin_bif(dataseries.bif) - bro_plugin_link_library(${Lintel_LIBRARIES}) - bro_plugin_link_library(${DataSeries_LIBRARIES}) - bro_plugin_link_library(${LibXML2_LIBRARIES}) - bro_plugin_end() - -endif() - diff --git a/src/logging/writers/dataseries/DataSeries.cc b/src/logging/writers/dataseries/DataSeries.cc deleted file mode 100644 index 5f039ac5d5..0000000000 --- a/src/logging/writers/dataseries/DataSeries.cc +++ /dev/null @@ -1,459 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "config.h" - -#include -#include -#include - -#include - -#include "NetVar.h" -#include "threading/SerialTypes.h" - -#include "DataSeries.h" -#include "dataseries.bif.h" - -using namespace logging; -using namespace writer; - -std::string DataSeries::LogValueToString(threading::Value *val) - { - // In some cases, no value is attached. If this is the case, return - // an empty string. - if( ! val->present ) - return ""; - - switch(val->type) { - case TYPE_BOOL: - return (val->val.int_val ? "true" : "false"); - - case TYPE_INT: - { - std::ostringstream ostr; - ostr << val->val.int_val; - return ostr.str(); - } - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - { - std::ostringstream ostr; - ostr << val->val.uint_val; - return ostr.str(); - } - - case TYPE_SUBNET: - return ascii->Render(val->val.subnet_val); - - case TYPE_ADDR: - return ascii->Render(val->val.addr_val); - - // Note: These two cases are relatively special. We need to convert - // these values into their integer equivalents to maximize precision. - // At the moment, there won't be a noticeable effect (Bro uses the - // double format everywhere internally, so we've already lost the - // precision we'd gain here), but timestamps may eventually switch to - // this representation within Bro. - // - // In the near-term, this *should* lead to better pack_relative (and - // thus smaller output files). - case TYPE_TIME: - case TYPE_INTERVAL: - if ( ds_use_integer_for_time ) - { - std::ostringstream ostr; - ostr << (uint64_t)(DataSeries::TIME_SCALE * val->val.double_val); - return ostr.str(); - } - else - return ascii->Render(val->val.double_val); - - case TYPE_DOUBLE: - return ascii->Render(val->val.double_val); - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - if ( ! val->val.string_val.length ) - return ""; - - return string(val->val.string_val.data, val->val.string_val.length); - - case TYPE_TABLE: - { - if ( ! val->val.set_val.size ) - return ""; - - string tmpString = ""; - - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - tmpString += ds_set_separator; - - tmpString += LogValueToString(val->val.set_val.vals[j]); - } - - return tmpString; - } - - case TYPE_VECTOR: - { - if ( ! val->val.vector_val.size ) - return ""; - - string tmpString = ""; - - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - tmpString += ds_set_separator; - - tmpString += LogValueToString(val->val.vector_val.vals[j]); - } - - return tmpString; - } - - default: - InternalError(Fmt("unknown type %s in DataSeries::LogValueToString", type_name(val->type))); - return "cannot be reached"; - } -} - -string DataSeries::GetDSFieldType(const threading::Field *field) -{ - switch(field->type) { - case TYPE_BOOL: - return "bool"; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - case TYPE_INT: - return "int64"; - - case TYPE_DOUBLE: - return "double"; - - case TYPE_TIME: - case TYPE_INTERVAL: - return ds_use_integer_for_time ? "int64" : "double"; - - case TYPE_SUBNET: - case TYPE_ADDR: - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_TABLE: - case TYPE_VECTOR: - case TYPE_FUNC: - return "variable32"; - - default: - InternalError(Fmt("unknown type %s in DataSeries::GetDSFieldType", type_name(field->type))); - return "cannot be reached"; - } -} - -string DataSeries::BuildDSSchemaFromFieldTypes(const vector& vals, string sTitle) - { - if( ! sTitle.size() ) - sTitle = "GenericBroStream"; - - string xmlschema = "\n"; - - for( size_t i = 0; i < vals.size(); ++i ) - { - xmlschema += "\t\n"; - } - - xmlschema += "\n"; - - for( size_t i = 0; i < vals.size(); ++i ) - { - xmlschema += "\n"; - } - - return xmlschema; -} - -std::string DataSeries::GetDSOptionsForType(const threading::Field *field) -{ - switch( field->type ) { - case TYPE_TIME: - case TYPE_INTERVAL: - { - std::string s; - s += "pack_relative=\"" + std::string(field->name) + "\""; - - if ( ! ds_use_integer_for_time ) - s += " pack_scale=\"1e-6\" print_format=\"%.6f\" pack_scale_warn=\"no\""; - else - s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\""; - - return s; - } - - case TYPE_SUBNET: - case TYPE_ADDR: - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_TABLE: - case TYPE_VECTOR: - return "pack_unique=\"yes\""; - - default: - return ""; - } -} - -DataSeries::DataSeries(WriterFrontend* frontend) : WriterBackend(frontend) -{ - ds_compression = string((const char *)BifConst::LogDataSeries::compression->Bytes(), - BifConst::LogDataSeries::compression->Len()); - ds_dump_schema = BifConst::LogDataSeries::dump_schema; - ds_extent_size = BifConst::LogDataSeries::extent_size; - ds_num_threads = BifConst::LogDataSeries::num_threads; - ds_use_integer_for_time = BifConst::LogDataSeries::use_integer_for_time; - ds_set_separator = ","; - - threading::formatter::Ascii::SeparatorInfo sep_info; - ascii = new threading::formatter::Ascii(this, sep_info); - - compress_type = Extent::compress_mode_none; - log_file = 0; - log_output = 0; -} - -DataSeries::~DataSeries() - { - delete ascii; - } - -bool DataSeries::OpenLog(string path) - { - log_file = new DataSeriesSink(path + ".ds", compress_type); - log_file->writeExtentLibrary(log_types); - - for( size_t i = 0; i < schema_list.size(); ++i ) - { - string fn = schema_list[i].field_name; - GeneralField* gf = 0; -#ifdef USE_PERFTOOLS_DEBUG - { - // GeneralField isn't cleaning up some results of xml parsing, reported - // here: https://github.com/dataseries/DataSeries/issues/1 - // Ignore for now to make leak tests pass. There's confidence that - // we do clean up the GeneralField* since the ExtentSeries dtor for - // member log_series would trigger an assert if dynamically allocated - // fields aren't deleted beforehand. - HeapLeakChecker::Disabler disabler; -#endif - gf = GeneralField::create(log_series, fn); -#ifdef USE_PERFTOOLS_DEBUG - } -#endif - extents.insert(std::make_pair(fn, gf)); - } - - if ( ds_extent_size < ROW_MIN ) - { - Warning(Fmt("%d is not a valid value for 'rows'. Using min of %d instead", (int)ds_extent_size, (int)ROW_MIN)); - ds_extent_size = ROW_MIN; - } - - else if( ds_extent_size > ROW_MAX ) - { - Warning(Fmt("%d is not a valid value for 'rows'. Using max of %d instead", (int)ds_extent_size, (int)ROW_MAX)); - ds_extent_size = ROW_MAX; - } - - log_output = new OutputModule(*log_file, log_series, log_type, ds_extent_size); - - return true; - } - -bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const * fields) - { - // We first construct an XML schema thing (and, if ds_dump_schema is - // set, dump it to path + ".ds.xml"). Assuming that goes well, we - // use that schema to build our output logfile and prepare it to be - // written to. - - // Note: compressor count must be set *BEFORE* DataSeriesSink is - // instantiated. - if( ds_num_threads < THREAD_MIN && ds_num_threads != 0 ) - { - Warning(Fmt("%d is too few threads! Using %d instead", (int)ds_num_threads, (int)THREAD_MIN)); - ds_num_threads = THREAD_MIN; - } - - if( ds_num_threads > THREAD_MAX ) - { - Warning(Fmt("%d is too many threads! Dropping back to %d", (int)ds_num_threads, (int)THREAD_MAX)); - ds_num_threads = THREAD_MAX; - } - - if( ds_num_threads > 0 ) - DataSeriesSink::setCompressorCount(ds_num_threads); - - for ( int i = 0; i < num_fields; i++ ) - { - const threading::Field* field = fields[i]; - SchemaValue val; - val.ds_type = GetDSFieldType(field); - val.field_name = string(field->name); - val.field_options = GetDSOptionsForType(field); - val.bro_type = field->TypeName(); - schema_list.push_back(val); - } - - string schema = BuildDSSchemaFromFieldTypes(schema_list, info.path); - - if( ds_dump_schema ) - { - string name = string(info.path) + ".ds.xml"; - FILE* pFile = fopen(name.c_str(), "wb" ); - - if( pFile ) - { - fwrite(schema.c_str(), 1, schema.length(), pFile); - fclose(pFile); - } - - else - Error(Fmt("cannot dump schema: %s", Strerror(errno))); - } - - compress_type = Extent::compress_all; - - if( ds_compression == "lzf" ) - compress_type = Extent::compress_mode_lzf; - - else if( ds_compression == "lzo" ) - compress_type = Extent::compress_mode_lzo; - - else if( ds_compression == "zlib" ) - compress_type = Extent::compress_mode_zlib; - - else if( ds_compression == "bz2" ) - compress_type = Extent::compress_mode_bz2; - - else if( ds_compression == "none" ) - compress_type = Extent::compress_mode_none; - - else if( ds_compression == "any" ) - compress_type = Extent::compress_all; - - else - Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'zlib', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str())); - - log_type = log_types.registerTypePtr(schema); - log_series.setType(log_type); - - return OpenLog(info.path); - } - -bool DataSeries::DoFlush(double network_time) -{ - // Flushing is handled by DataSeries automatically, so this function - // doesn't do anything. - return true; -} - -void DataSeries::CloseLog() - { - for( ExtentIterator iter = extents.begin(); iter != extents.end(); ++iter ) - delete iter->second; - - extents.clear(); - - // Don't delete the file before you delete the output, or bad things - // will happen. - delete log_output; - delete log_file; - - log_output = 0; - log_file = 0; - } - -bool DataSeries::DoFinish(double network_time) -{ - CloseLog(); - return true; -} - -bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields, - threading::Value** vals) -{ - log_output->newRecord(); - - for( size_t i = 0; i < (size_t)num_fields; ++i ) - { - ExtentIterator iter = extents.find(fields[i]->name); - assert(iter != extents.end()); - - if( iter != extents.end() ) - { - GeneralField *cField = iter->second; - - if( vals[i]->present ) - cField->set(LogValueToString(vals[i])); - } - } - - return true; -} - -bool DataSeries::DoRotate(const char* rotated_path, double open, double close, bool terminating) -{ - // Note that if DS files are rotated too often, the aggregate log - // size will be (much) larger. - CloseLog(); - - string dsname = string(Info().path) + ".ds"; - string nname = string(rotated_path) + ".ds"; - - if ( rename(dsname.c_str(), nname.c_str()) != 0 ) - { - char buf[256]; - strerror_r(errno, buf, sizeof(buf)); - Error(Fmt("failed to rename %s to %s: %s", dsname.c_str(), - nname.c_str(), buf)); - FinishedRotation(); - return false; - } - - if ( ! FinishedRotation(nname.c_str(), dsname.c_str(), open, close, terminating) ) - { - Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str())); - return false; - } - - return OpenLog(Info().path); -} - -bool DataSeries::DoSetBuf(bool enabled) -{ - // DataSeries is *always* buffered to some degree. This option is ignored. - return true; -} - -bool DataSeries::DoHeartbeat(double network_time, double current_time) -{ - return true; -} diff --git a/src/logging/writers/dataseries/DataSeries.h b/src/logging/writers/dataseries/DataSeries.h deleted file mode 100644 index cdc4d9a66c..0000000000 --- a/src/logging/writers/dataseries/DataSeries.h +++ /dev/null @@ -1,128 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// A binary log writer producing DataSeries output. See doc/data-series.rst -// for more information. - -#ifndef LOGGING_WRITER_DATA_SERIES_H -#define LOGGING_WRITER_DATA_SERIES_H - -#include -#include -#include -#include - -#include "logging/WriterBackend.h" -#include "threading/formatters/Ascii.h" - -namespace logging { namespace writer { - -class DataSeries : public WriterBackend { -public: - DataSeries(WriterFrontend* frontend); - ~DataSeries(); - - static WriterBackend* Instantiate(WriterFrontend* frontend) - { return new DataSeries(frontend); } - -protected: - // Overidden from WriterBackend. - - virtual bool DoInit(const WriterInfo& info, int num_fields, - const threading::Field* const * fields); - - virtual bool DoWrite(int num_fields, const threading::Field* const* fields, - threading::Value** vals); - virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(const char* rotated_path, double open, - double close, bool terminating); - virtual bool DoFlush(double network_time); - virtual bool DoFinish(double network_time); - virtual bool DoHeartbeat(double network_time, double current_time); - -private: - static const size_t ROW_MIN = 2048; // Minimum extent size. - static const size_t ROW_MAX = (1024 * 1024 * 100); // Maximum extent size. - static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn. - static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn. - static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers. - const char* TIME_UNIT() { return "microseconds"; } // DS name for time resolution when converted to integers. Must match TIME_SCALE. - - struct SchemaValue - { - string ds_type; - string bro_type; - string field_name; - string field_options; - }; - - /** - * Turns a log value into a std::string. Uses an ostringstream to do the - * heavy lifting, but still need to switch on the type to know which value - * in the union to give to the string string for processing. - * - * @param val The value we wish to convert to a string - * @return the string value of val - */ - std::string LogValueToString(threading::Value *val); - - /** - * Takes a field type and converts it to a relevant DataSeries type. - * - * @param field We extract the type from this and convert it into a relevant DS type. - * @return String representation of type that DataSeries can understand. - */ - string GetDSFieldType(const threading::Field *field); - - /** - * Are there any options we should put into the XML schema? - * - * @param field We extract the type from this and return any options that make sense for that type. - * @return Options that can be added directly to the XML (e.g. "pack_relative=\"yes\"") - */ - std::string GetDSOptionsForType(const threading::Field *field); - - /** - * Takes a list of types, a list of names, and a title, and uses it to construct a valid DataSeries XML schema - * thing, which is then returned as a std::string - * - * @param opts std::vector of strings containing a list of options to be appended to each field (e.g. "pack_relative=yes") - * @param sTitle Name of this schema. Ideally, these schemas would be aggregated and re-used. - */ - string BuildDSSchemaFromFieldTypes(const vector& vals, string sTitle); - - /** Closes the currently open file. */ - void CloseLog(); - - /** Opens a new file. */ - bool OpenLog(string path); - - typedef std::map ExtentMap; - typedef ExtentMap::iterator ExtentIterator; - - // Internal DataSeries structures we need to keep track of. - vector schema_list; - ExtentTypeLibrary log_types; - ExtentType::Ptr log_type; - ExtentSeries log_series; - ExtentMap extents; - int compress_type; - - DataSeriesSink* log_file; - OutputModule* log_output; - - // Options set from the script-level. - uint64 ds_extent_size; - uint64 ds_num_threads; - string ds_compression; - bool ds_dump_schema; - bool ds_use_integer_for_time; - string ds_set_separator; - - threading::formatter::Ascii* ascii; -}; - -} -} - -#endif - diff --git a/src/logging/writers/dataseries/Plugin.cc b/src/logging/writers/dataseries/Plugin.cc deleted file mode 100644 index 271f523ffa..0000000000 --- a/src/logging/writers/dataseries/Plugin.cc +++ /dev/null @@ -1,25 +0,0 @@ -// See the file in the main distribution directory for copyright. - - -#include "plugin/Plugin.h" - -#include "DataSeries.h" - -namespace plugin { -namespace Bro_DataSeriesWriter { - -class Plugin : public plugin::Plugin { -public: - plugin::Configuration Configure() - { - AddComponent(new ::logging::Component("DataSeries", ::logging::writer::DataSeries::Instantiate)); - - plugin::Configuration config; - config.name = "Bro::DataSeriesWriter"; - config.description = "DataSeries log writer"; - return config; - } -} plugin; - -} -} diff --git a/src/logging/writers/dataseries/dataseries.bif b/src/logging/writers/dataseries/dataseries.bif deleted file mode 100644 index 2c83a369f2..0000000000 --- a/src/logging/writers/dataseries/dataseries.bif +++ /dev/null @@ -1,10 +0,0 @@ - -# Options for the DataSeries writer. - -module LogDataSeries; - -const compression: string; -const extent_size: count; -const dump_schema: bool; -const use_integer_for_time: bool; -const num_threads: count; diff --git a/src/logging/writers/elasticsearch/CMakeLists.txt b/src/logging/writers/elasticsearch/CMakeLists.txt deleted file mode 100644 index 6240b3cf63..0000000000 --- a/src/logging/writers/elasticsearch/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ - -include(BroPlugin) - -find_package(LibCURL) - -if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND) - include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - bro_plugin_begin(Bro ElasticSearchWriter) - bro_plugin_cc(ElasticSearch.cc Plugin.cc) - bro_plugin_bif(elasticsearch.bif) - bro_plugin_link_library(${LibCURL_LIBRARIES}) - bro_plugin_end() -endif() - - diff --git a/src/logging/writers/elasticsearch/ElasticSearch.cc b/src/logging/writers/elasticsearch/ElasticSearch.cc deleted file mode 100644 index 8cf052f4ce..0000000000 --- a/src/logging/writers/elasticsearch/ElasticSearch.cc +++ /dev/null @@ -1,290 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// This is experimental code that is not yet ready for production usage. -// - - -#include "config.h" - -#include "util.h" // Needs to come first for stdint.h - -#include -#include -#include -#include - -#include "BroString.h" -#include "threading/SerialTypes.h" - -#include "ElasticSearch.h" -#include "elasticsearch.bif.h" - -using namespace logging; -using namespace writer; -using threading::Value; -using threading::Field; - -ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) - { - cluster_name_len = BifConst::LogElasticSearch::cluster_name->Len(); - cluster_name = new char[cluster_name_len + 1]; - memcpy(cluster_name, BifConst::LogElasticSearch::cluster_name->Bytes(), cluster_name_len); - cluster_name[cluster_name_len] = 0; - - index_prefix = string((const char*) BifConst::LogElasticSearch::index_prefix->Bytes(), BifConst::LogElasticSearch::index_prefix->Len()); - - es_server = string(Fmt("http://%s:%d", BifConst::LogElasticSearch::server_host->Bytes(), - (int) BifConst::LogElasticSearch::server_port)); - bulk_url = string(Fmt("%s/_bulk", es_server.c_str())); - - http_headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8"); - buffer.Clear(); - counter = 0; - current_index = string(); - prev_index = string(); - last_send = current_time(); - failing = false; - - transfer_timeout = static_cast(BifConst::LogElasticSearch::transfer_timeout); - - curl_handle = HTTPSetup(); - - json = new threading::formatter::JSON(this, threading::formatter::JSON::TS_MILLIS); -} - -ElasticSearch::~ElasticSearch() - { - delete [] cluster_name; - delete json; - } - -bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields) - { - return true; - } - -bool ElasticSearch::DoFlush(double network_time) - { - BatchIndex(); - return true; - } - -bool ElasticSearch::DoFinish(double network_time) - { - BatchIndex(); - curl_slist_free_all(http_headers); - curl_easy_cleanup(curl_handle); - return true; - } - -bool ElasticSearch::BatchIndex() - { - curl_easy_reset(curl_handle); - curl_easy_setopt(curl_handle, CURLOPT_URL, bulk_url.c_str()); - curl_easy_setopt(curl_handle, CURLOPT_POST, 1); - curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)buffer.Len()); - curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes()); - failing = ! HTTPSend(curl_handle); - - // We are currently throwing the data out regardless of if the send failed. Fire and forget! - buffer.Clear(); - counter = 0; - last_send = current_time(); - - return true; - } - -bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields, - Value** vals) - { - if ( current_index.empty() ) - UpdateIndex(network_time, Info().rotation_interval, Info().rotation_base); - - // Our action line looks like: - buffer.AddRaw("{\"index\":{\"_index\":\"", 20); - buffer.Add(current_index); - buffer.AddRaw("\",\"_type\":\"", 11); - buffer.Add(Info().path); - buffer.AddRaw("\"}}\n", 4); - - json->Describe(&buffer, num_fields, fields, vals); - - buffer.AddRaw("\n", 1); - - counter++; - if ( counter >= BifConst::LogElasticSearch::max_batch_size || - uint(buffer.Len()) >= BifConst::LogElasticSearch::max_byte_size ) - BatchIndex(); - - return true; - } - -bool ElasticSearch::UpdateIndex(double now, double rinterval, double rbase) - { - if ( rinterval == 0 ) - { - // if logs aren't being rotated, don't use a rotation oriented index name. - current_index = index_prefix; - } - else - { - double nr = calc_next_rotate(now, rinterval, rbase); - double interval_beginning = now - (rinterval - nr); - - struct tm tm; - char buf[128]; - time_t teatime = (time_t)interval_beginning; - localtime_r(&teatime, &tm); - strftime(buf, sizeof(buf), "%Y%m%d%H%M", &tm); - - prev_index = current_index; - current_index = index_prefix + "-" + buf; - - // Send some metadata about this index. - buffer.AddRaw("{\"index\":{\"_index\":\"@", 21); - buffer.Add(index_prefix); - buffer.AddRaw("-meta\",\"_type\":\"index\",\"_id\":\"", 30); - buffer.Add(current_index); - buffer.AddRaw("-", 1); - buffer.Add(Info().rotation_base); - buffer.AddRaw("-", 1); - buffer.Add(Info().rotation_interval); - buffer.AddRaw("\"}}\n{\"name\":\"", 13); - buffer.Add(current_index); - buffer.AddRaw("\",\"start\":", 10); - buffer.Add(interval_beginning); - buffer.AddRaw(",\"end\":", 7); - buffer.Add(interval_beginning+rinterval); - buffer.AddRaw("}\n", 2); - } - - //printf("%s - prev:%s current:%s\n", Info().path.c_str(), prev_index.c_str(), current_index.c_str()); - return true; - } - - -bool ElasticSearch::DoRotate(const char* rotated_path, double open, double close, bool terminating) - { - // Update the currently used index to the new rotation interval. - UpdateIndex(close, Info().rotation_interval, Info().rotation_base); - - // Only do this stuff if there was a previous index. - if ( ! prev_index.empty() ) - { - // FIXME: I think this section is taking too long and causing the thread to die. - - // Compress the previous index - //curl_easy_reset(curl_handle); - //curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_settings", es_server.c_str(), prev_index.c_str())); - //curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PUT"); - //curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "{\"index\":{\"store.compress.stored\":\"true\"}}"); - //curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) 42); - //HTTPSend(curl_handle); - - // Optimize the previous index. - // TODO: make this into variables. - //curl_easy_reset(curl_handle); - //curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_optimize?max_num_segments=1&wait_for_merge=false", es_server.c_str(), prev_index.c_str())); - //HTTPSend(curl_handle); - } - - if ( ! FinishedRotation(current_index.c_str(), prev_index.c_str(), open, close, terminating) ) - Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str())); - - return true; - } - -bool ElasticSearch::DoSetBuf(bool enabled) - { - // Nothing to do. - return true; - } - -bool ElasticSearch::DoHeartbeat(double network_time, double current_time) - { - if ( last_send > 0 && buffer.Len() > 0 && - current_time-last_send > BifConst::LogElasticSearch::max_batch_interval ) - { - BatchIndex(); - } - - return true; - } - - -CURL* ElasticSearch::HTTPSetup() - { - CURL* handle = curl_easy_init(); - if ( ! handle ) - { - Error("cURL did not initialize correctly."); - return 0; - } - - return handle; - } - -size_t ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata) - { - //TODO: Do some verification on the result? - return size; - } - -bool ElasticSearch::HTTPSend(CURL *handle) - { - curl_easy_setopt(handle, CURLOPT_HTTPHEADER, http_headers); - curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result. - // HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. - // The best (only?) way to disable that is to just use HTTP 1.0 - curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); - - // Some timeout options. These will need more attention later. - curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout); - curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout); - curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60); - - CURLcode return_code = curl_easy_perform(handle); - - switch ( return_code ) - { - case CURLE_COULDNT_CONNECT: - case CURLE_COULDNT_RESOLVE_HOST: - case CURLE_WRITE_ERROR: - case CURLE_RECV_ERROR: - { - if ( ! failing ) - Error(Fmt("ElasticSearch server may not be accessible.")); - - break; - } - - case CURLE_OPERATION_TIMEDOUT: - { - if ( ! failing ) - Warning(Fmt("HTTP operation with elasticsearch server timed out at %" PRIu64 " msecs.", transfer_timeout)); - - break; - } - - case CURLE_OK: - { - long http_code = 0; - curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code); - if ( http_code == 200 ) - // Hopefully everything goes through here. - return true; - else if ( ! failing ) - Error(Fmt("Received a non-successful status code back from ElasticSearch server, check the elasticsearch server log.")); - - break; - } - - default: - { - break; - } - } - // The "successful" return happens above - return false; - } diff --git a/src/logging/writers/elasticsearch/ElasticSearch.h b/src/logging/writers/elasticsearch/ElasticSearch.h deleted file mode 100644 index 5f3d229b5b..0000000000 --- a/src/logging/writers/elasticsearch/ElasticSearch.h +++ /dev/null @@ -1,86 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// Log writer for writing to an ElasticSearch database -// -// This is experimental code that is not yet ready for production usage. -// - -#ifndef LOGGING_WRITER_ELASTICSEARCH_H -#define LOGGING_WRITER_ELASTICSEARCH_H - -#include - -#include "logging/WriterBackend.h" -#include "threading/formatters/JSON.h" - -namespace logging { namespace writer { - -class ElasticSearch : public WriterBackend { -public: - ElasticSearch(WriterFrontend* frontend); - ~ElasticSearch(); - - static string LogExt(); - - static WriterBackend* Instantiate(WriterFrontend* frontend) - { return new ElasticSearch(frontend); } - -protected: - // Overidden from WriterBackend. - - virtual bool DoInit(const WriterInfo& info, int num_fields, - const threading::Field* const* fields); - - virtual bool DoWrite(int num_fields, const threading::Field* const* fields, - threading::Value** vals); - virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(const char* rotated_path, double open, - double close, bool terminating); - virtual bool DoFlush(double network_time); - virtual bool DoFinish(double network_time); - virtual bool DoHeartbeat(double network_time, double current_time); - -private: - bool AddFieldToBuffer(ODesc *b, threading::Value* val, const threading::Field* field); - bool AddValueToBuffer(ODesc *b, threading::Value* val); - bool BatchIndex(); - bool SendMappings(); - bool UpdateIndex(double now, double rinterval, double rbase); - - CURL* HTTPSetup(); - size_t HTTPReceive(void* ptr, int size, int nmemb, void* userdata); - bool HTTPSend(CURL *handle); - - // Buffers, etc. - ODesc buffer; - uint64 counter; - double last_send; - string current_index; - string prev_index; - - CURL* curl_handle; - - // From scripts - char* cluster_name; - int cluster_name_len; - - string es_server; - string bulk_url; - - struct curl_slist *http_headers; - - string path; - string index_prefix; - long transfer_timeout; - bool failing; - - uint64 batch_size; - - threading::formatter::JSON* json; -}; - -} -} - - -#endif diff --git a/src/logging/writers/elasticsearch/Plugin.cc b/src/logging/writers/elasticsearch/Plugin.cc deleted file mode 100644 index 2abb7080e4..0000000000 --- a/src/logging/writers/elasticsearch/Plugin.cc +++ /dev/null @@ -1,37 +0,0 @@ -// See the file in the main distribution directory for copyright. - -#include - -#include "plugin/Plugin.h" - -#include "ElasticSearch.h" - -namespace plugin { -namespace Bro_ElasticSearchWriter { - -class Plugin : public plugin::Plugin { -public: - plugin::Configuration Configure() - { - AddComponent(new ::logging::Component("ElasticSearch", ::logging::writer::ElasticSearch::Instantiate)); - - plugin::Configuration config; - config.name = "Bro::ElasticSearchWriter"; - config.description = "ElasticSearch log writer"; - return config; - } - - virtual void InitPreScript() - { - curl_global_init(CURL_GLOBAL_ALL); - } - - virtual void Done() - { - curl_global_cleanup(); - } - -} plugin; - -} -} diff --git a/src/logging/writers/elasticsearch/elasticsearch.bif b/src/logging/writers/elasticsearch/elasticsearch.bif deleted file mode 100644 index 3d56dd7dd4..0000000000 --- a/src/logging/writers/elasticsearch/elasticsearch.bif +++ /dev/null @@ -1,14 +0,0 @@ - -# Options for the ElasticSearch writer. - -module LogElasticSearch; - -const cluster_name: string; -const server_host: string; -const server_port: count; -const index_prefix: string; -const type_prefix: string; -const transfer_timeout: interval; -const max_batch_size: count; -const max_batch_interval: interval; -const max_byte_size: count; diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml deleted file mode 100644 index a1e65c254e..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out deleted file mode 100644 index 94f25c37f4..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out +++ /dev/null @@ -1,290 +0,0 @@ -test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 dataseries -test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 dataseries -test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 dataseries -test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 dataseries -test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 dataseries -test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 dataseries -test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 dataseries -test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 dataseries -test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 dataseries -test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataseries -> test.2011-03-07-03-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299466805.000000 10.0.0.1 20 10.0.0.2 1024 -1299470395.000000 10.0.0.2 20 10.0.0.3 0 -> test.2011-03-07-04-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299470405.000000 10.0.0.1 20 10.0.0.2 1025 -1299473995.000000 10.0.0.2 20 10.0.0.3 1 -> test.2011-03-07-05-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299474005.000000 10.0.0.1 20 10.0.0.2 1026 -1299477595.000000 10.0.0.2 20 10.0.0.3 2 -> test.2011-03-07-06-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299477605.000000 10.0.0.1 20 10.0.0.2 1027 -1299481195.000000 10.0.0.2 20 10.0.0.3 3 -> test.2011-03-07-07-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299481205.000000 10.0.0.1 20 10.0.0.2 1028 -1299484795.000000 10.0.0.2 20 10.0.0.3 4 -> test.2011-03-07-08-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299484805.000000 10.0.0.1 20 10.0.0.2 1029 -1299488395.000000 10.0.0.2 20 10.0.0.3 5 -> test.2011-03-07-09-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299488405.000000 10.0.0.1 20 10.0.0.2 1030 -1299491995.000000 10.0.0.2 20 10.0.0.3 6 -> test.2011-03-07-10-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299492005.000000 10.0.0.1 20 10.0.0.2 1031 -1299495595.000000 10.0.0.2 20 10.0.0.3 7 -> test.2011-03-07-11-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299495605.000000 10.0.0.1 20 10.0.0.2 1032 -1299499195.000000 10.0.0.2 20 10.0.0.3 8 -> test.2011-03-07-12-00-05.ds -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='test' -t id.orig_h id.orig_p id.resp_h id.resp_p -1299499205.000000 10.0.0.1 20 10.0.0.2 1033 -1299502795.000000 10.0.0.2 20 10.0.0.3 9 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt deleted file mode 100644 index 225217faea..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt +++ /dev/null @@ -1,34 +0,0 @@ -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='ssh' -t id.orig_h id.orig_p id.resp_h id.resp_p status country -1342748962.493341 1.2.3.4 1234 2.3.4.5 80 success unknown -1342748962.493341 1.2.3.4 1234 2.3.4.5 80 failure US -1342748962.493341 1.2.3.4 1234 2.3.4.5 80 failure UK -1342748962.493341 1.2.3.4 1234 2.3.4.5 80 success BR -1342748962.493341 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt deleted file mode 100644 index a832005c83..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt +++ /dev/null @@ -1,89 +0,0 @@ -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='conn' -ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents -1300475167096535 CXWv6p3arKYeMETxOg 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0 -1300475167097012 CjhGID4nQcgTWjvg4c fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp dns 0 0 0 S0 F 0 D 1 199 0 0 -1300475167099816 CCvvfg3TEfuqmmG4bh 141.142.220.50 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 179 0 0 -1300475168853899 CPbrpk1qSsw6ESzHV4 141.142.220.118 43927 141.142.2.2 53 udp dns 435 38 89 SF F 0 Dd 1 66 1 117 -1300475168854378 C6pKV8GSxOnSLghOa 141.142.220.118 37676 141.142.2.2 53 udp dns 420 52 99 SF F 0 Dd 1 80 1 127 -1300475168854837 CIPOse170MGiRM1Qf4 141.142.220.118 40526 141.142.2.2 53 udp dns 391 38 183 SF F 0 Dd 1 66 1 211 -1300475168857956 CMXxB5GvmoxJFXdTa 141.142.220.118 32902 141.142.2.2 53 udp dns 317 38 89 SF F 0 Dd 1 66 1 117 -1300475168858306 Caby8b1slFea8xwSmb 141.142.220.118 59816 141.142.2.2 53 udp dns 343 52 99 SF F 0 Dd 1 80 1 127 -1300475168858713 Che1bq3i2rO3KD1Syg 141.142.220.118 59714 141.142.2.2 53 udp dns 375 38 183 SF F 0 Dd 1 66 1 211 -1300475168891644 CEle3f3zno26fFZkrh 141.142.220.118 58206 141.142.2.2 53 udp dns 339 38 89 SF F 0 Dd 1 66 1 117 -1300475168892037 CwSkQu4eWZCH7OONC1 141.142.220.118 38911 141.142.2.2 53 udp dns 334 52 99 SF F 0 Dd 1 80 1 127 -1300475168892414 CfTOmO0HKorjr8Zp7 141.142.220.118 59746 141.142.2.2 53 udp dns 420 38 183 SF F 0 Dd 1 66 1 211 -1300475168893988 Cab0vO1xNYSS2hJkle 141.142.220.118 45000 141.142.2.2 53 udp dns 384 38 89 SF F 0 Dd 1 66 1 117 -1300475168894422 Cx2FqO23omNawSNrxj 141.142.220.118 48479 141.142.2.2 53 udp dns 316 52 99 SF F 0 Dd 1 80 1 127 -1300475168894787 Cx3C534wEyF3OvvcQe 141.142.220.118 48128 141.142.2.2 53 udp dns 422 38 183 SF F 0 Dd 1 66 1 211 -1300475168901749 CUKS0W3HFYOnBqSE5e 141.142.220.118 56056 141.142.2.2 53 udp dns 402 36 131 SF F 0 Dd 1 64 1 159 -1300475168902195 CRrfvP2lalMAYOCLhj 141.142.220.118 55092 141.142.2.2 53 udp dns 374 36 198 SF F 0 Dd 1 64 1 226 -1300475169899438 CojBOU3CXcLHl1r6x1 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0 -1300475170862384 CJzVQRGJrX6V15ik7 141.142.220.226 137 141.142.220.255 137 udp dns 2613016 350 0 S0 F 0 D 7 546 0 0 -1300475171675372 ClAbxY1nmdjCuo0Le2 fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 100096 66 0 S0 F 0 D 2 162 0 0 -1300475171677081 CwG0BF1VXE0gWgs78 141.142.220.226 55131 224.0.0.252 5355 udp dns 100020 66 0 S0 F 0 D 2 122 0 0 -1300475173116749 CisNaL1Cm73CiNOmcg fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 99801 66 0 S0 F 0 D 2 162 0 0 -1300475173117362 CBQnJn22qN8TOeeZil 141.142.220.226 55671 224.0.0.252 5355 udp dns 99848 66 0 S0 F 0 D 2 122 0 0 -1300475173153679 CbEsuD3dgDDngdlbKf 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0 -1300475168859163 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 tcp http 215893 1130 734 S1 F 0 ShADad 6 1450 4 950 -1300475168652003 CsRx2w45OKnoww6xl4 141.142.220.118 35634 208.80.152.2 80 tcp 61328 463 350 OTH F 0 DdA 2 567 1 402 -1300475168895267 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 tcp http 227283 1178 734 S1 F 0 ShADad 6 1498 4 950 -1300475168902635 Cn78a440HlxuyZKs6f 141.142.220.118 35642 208.80.152.2 80 tcp http 120040 534 412 S1 F 0 ShADad 4 750 3 576 -1300475168892936 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 tcp http 229603 1148 734 S1 F 0 ShADad 6 1468 4 950 -1300475168855305 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 tcp http 218501 1171 733 S1 F 0 ShADad 6 1491 4 949 -1300475168892913 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 tcp http 220960 1137 733 S1 F 0 ShADad 6 1457 4 949 -1300475169780331 CUof3F2yAIid8QS3dk 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48 -1300475168724007 CRJuHdVW0XPVINV8a 141.142.220.118 48649 208.80.152.118 80 tcp http 119904 525 232 S1 F 0 ShADad 4 741 3 396 -1300475168855330 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp http 219720 1125 734 S1 F 0 ShADad 6 1445 4 950 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt deleted file mode 100644 index afb44e36eb..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt +++ /dev/null @@ -1,89 +0,0 @@ -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='conn' -ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents -1300475167.096535 CXWv6p3arKYeMETxOg 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0 -1300475167.097012 CjhGID4nQcgTWjvg4c fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp dns 0.000000 0 0 S0 F 0 D 1 199 0 0 -1300475167.099816 CCvvfg3TEfuqmmG4bh 141.142.220.50 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 179 0 0 -1300475168.853899 CPbrpk1qSsw6ESzHV4 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117 -1300475168.854378 C6pKV8GSxOnSLghOa 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127 -1300475168.854837 CIPOse170MGiRM1Qf4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211 -1300475168.857956 CMXxB5GvmoxJFXdTa 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 38 89 SF F 0 Dd 1 66 1 117 -1300475168.858306 Caby8b1slFea8xwSmb 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 52 99 SF F 0 Dd 1 80 1 127 -1300475168.858713 Che1bq3i2rO3KD1Syg 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 38 183 SF F 0 Dd 1 66 1 211 -1300475168.891644 CEle3f3zno26fFZkrh 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 38 89 SF F 0 Dd 1 66 1 117 -1300475168.892037 CwSkQu4eWZCH7OONC1 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 52 99 SF F 0 Dd 1 80 1 127 -1300475168.892414 CfTOmO0HKorjr8Zp7 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 38 183 SF F 0 Dd 1 66 1 211 -1300475168.893988 Cab0vO1xNYSS2hJkle 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 38 89 SF F 0 Dd 1 66 1 117 -1300475168.894422 Cx2FqO23omNawSNrxj 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 52 99 SF F 0 Dd 1 80 1 127 -1300475168.894787 Cx3C534wEyF3OvvcQe 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 38 183 SF F 0 Dd 1 66 1 211 -1300475168.901749 CUKS0W3HFYOnBqSE5e 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 36 131 SF F 0 Dd 1 64 1 159 -1300475168.902195 CRrfvP2lalMAYOCLhj 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 36 198 SF F 0 Dd 1 64 1 226 -1300475169.899438 CojBOU3CXcLHl1r6x1 141.142.220.44 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 85 0 0 -1300475170.862384 CJzVQRGJrX6V15ik7 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 F 0 D 7 546 0 0 -1300475171.675372 ClAbxY1nmdjCuo0Le2 fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 F 0 D 2 162 0 0 -1300475171.677081 CwG0BF1VXE0gWgs78 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 F 0 D 2 122 0 0 -1300475173.116749 CisNaL1Cm73CiNOmcg fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 F 0 D 2 162 0 0 -1300475173.117362 CBQnJn22qN8TOeeZil 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 F 0 D 2 122 0 0 -1300475173.153679 CbEsuD3dgDDngdlbKf 141.142.220.238 56641 141.142.220.255 137 udp dns 0.000000 0 0 S0 F 0 D 1 78 0 0 -1300475168.859163 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 tcp http 0.215893 1130 734 S1 F 0 ShADad 6 1450 4 950 -1300475168.652003 CsRx2w45OKnoww6xl4 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 463 350 OTH F 0 DdA 2 567 1 402 -1300475168.895267 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 tcp http 0.227284 1178 734 S1 F 0 ShADad 6 1498 4 950 -1300475168.902635 Cn78a440HlxuyZKs6f 141.142.220.118 35642 208.80.152.2 80 tcp http 0.120041 534 412 S1 F 0 ShADad 4 750 3 576 -1300475168.892936 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 F 0 ShADad 6 1468 4 950 -1300475168.855305 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 tcp http 0.218501 1171 733 S1 F 0 ShADad 6 1491 4 949 -1300475168.892913 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 tcp http 0.220961 1137 733 S1 F 0 ShADad 6 1457 4 949 -1300475169.780331 CUof3F2yAIid8QS3dk 141.142.220.235 6705 173.192.163.128 80 tcp 0.000000 0 0 OTH F 0 h 0 0 1 48 -1300475168.724007 CRJuHdVW0XPVINV8a 141.142.220.118 48649 208.80.152.118 80 tcp http 0.119905 525 232 S1 F 0 ShADad 4 741 3 396 -1300475168.855330 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp http 0.219720 1125 734 S1 F 0 ShADad 6 1445 4 950 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt deleted file mode 100644 index eec7031ba7..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt +++ /dev/null @@ -1,83 +0,0 @@ -# Extent Types ... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Extent, type='http' -ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types -1300475168.784020 CRJuHdVW0XPVINV8a 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.916018 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.916183 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.918358 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.952307 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.952296 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.954820 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.962687 Cn78a440HlxuyZKs6f 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.975934 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.976436 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.979264 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475169.014619 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475169.014593 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475169.014927 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 diff --git a/testing/btest/core/leaks/dataseries-rotate.bro b/testing/btest/core/leaks/dataseries-rotate.bro deleted file mode 100644 index f43823cd5f..0000000000 --- a/testing/btest/core/leaks/dataseries-rotate.bro +++ /dev/null @@ -1,36 +0,0 @@ -# -# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt -# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks -# -# @TEST-GROUP: leaks -# @TEST-GROUP: dataseries -# -# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/rotation.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES -# @TEST-EXEC: btest-bg-wait 15 - -module Test; - -export { - # Create a new ID for our log stream - redef enum Log::ID += { LOG }; - - # Define a record with all the columns the log file can have. - # (I'm using a subset of fields from ssh-ext for demonstration.) - type Log: record { - t: time; - id: conn_id; # Will be rolled out into individual columns. - } &log; -} - -redef Log::default_rotation_interval = 1hr; -redef Log::default_rotation_postprocessor_cmd = "echo"; - -event bro_init() -{ - Log::create_stream(Test::LOG, [$columns=Log]); -} - -event new_connection(c: connection) - { - Log::write(Test::LOG, [$t=network_time(), $id=c$id]); - } diff --git a/testing/btest/core/leaks/dataseries.bro b/testing/btest/core/leaks/dataseries.bro deleted file mode 100644 index c5556accf2..0000000000 --- a/testing/btest/core/leaks/dataseries.bro +++ /dev/null @@ -1,11 +0,0 @@ -# Needs perftools support. -# -# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt -# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks -# -# @TEST-GROUP: leaks -# @TEST-GROUP: dataseries -# -# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks -# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES -# @TEST-EXEC: btest-bg-wait 25 diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro deleted file mode 100644 index 1b05ca9b8f..0000000000 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro +++ /dev/null @@ -1,44 +0,0 @@ -# -# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt -# @TEST-GROUP: dataseries -# -# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES -# @TEST-EXEC: test -e ssh.ds.xml -# @TEST-EXEC: btest-diff ssh.ds.xml - -module SSH; - -redef LogDataSeries::dump_schema = T; - -# Haven't yet found a way to check for the effect of these. -redef LogDataSeries::compression = "bz2"; -redef LogDataSeries::extent_size = 1000; -redef LogDataSeries::num_threads = 5; - -# LogDataSeries::use_integer_for_time is tested separately. - -export { - redef enum Log::ID += { LOG }; - - type Log: record { - t: time; - id: conn_id; # Will be rolled out into individual columns. - status: string &optional; - country: string &default="unknown"; - } &log; -} - -event bro_init() -{ - Log::create_stream(SSH::LOG, [$columns=Log]); - - local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; - - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); - -} - diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro deleted file mode 100644 index ebc7a15002..0000000000 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro +++ /dev/null @@ -1,34 +0,0 @@ -# -# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt -# @TEST-GROUP: dataseries -# -# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out -# @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt --skip-index $i; done >>out -# @TEST-EXEC: btest-diff out - -module Test; - -export { - # Create a new ID for our log stream - redef enum Log::ID += { LOG }; - - # Define a record with all the columns the log file can have. - # (I'm using a subset of fields from ssh-ext for demonstration.) - type Log: record { - t: time; - id: conn_id; # Will be rolled out into individual columns. - } &log; -} - -redef Log::default_rotation_interval = 1hr; -redef Log::default_rotation_postprocessor_cmd = "echo"; - -event bro_init() -{ - Log::create_stream(Test::LOG, [$columns=Log]); -} - -event new_connection(c: connection) - { - Log::write(Test::LOG, [$t=network_time(), $id=c$id]); - } diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro deleted file mode 100644 index c030c58861..0000000000 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro +++ /dev/null @@ -1,35 +0,0 @@ -# -# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt -# @TEST-GROUP: dataseries -# -# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES -# @TEST-EXEC: ds2txt --skip-index ssh.ds >ssh.ds.txt -# @TEST-EXEC: btest-diff ssh.ds.txt - -module SSH; - -export { - redef enum Log::ID += { LOG }; - - type Log: record { - t: time; - id: conn_id; # Will be rolled out into individual columns. - status: string &optional; - country: string &default="unknown"; - } &log; -} - -event bro_init() -{ - Log::create_stream(SSH::LOG, [$columns=Log]); - - local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; - - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); - Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); - -} - diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro deleted file mode 100644 index 87d33a46d1..0000000000 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro +++ /dev/null @@ -1,9 +0,0 @@ -# -# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt -# @TEST-GROUP: dataseries -# -# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES -# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt -# @TEST-EXEC: btest-diff conn.ds.txt - -redef LogDataSeries::use_integer_for_time = T; diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro deleted file mode 100644 index 15e5ba16f1..0000000000 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro +++ /dev/null @@ -1,9 +0,0 @@ -# -# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt -# @TEST-GROUP: dataseries -# -# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES -# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt -# @TEST-EXEC: ds2txt --skip-index http.ds >http.ds.txt -# @TEST-EXEC: btest-diff conn.ds.txt -# @TEST-EXEC: btest-diff http.ds.txt