From 489899bcac2d5e86bcf29ccd291437675e256e75 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 8 Oct 2013 17:45:49 -0500 Subject: [PATCH 01/36] Update elasticsearch documentation Fixed some broken links and a couple minor typos. --- doc/frameworks/logging-elasticsearch.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/frameworks/logging-elasticsearch.rst b/doc/frameworks/logging-elasticsearch.rst index 6711d29543..3f1eac859d 100644 --- a/doc/frameworks/logging-elasticsearch.rst +++ b/doc/frameworks/logging-elasticsearch.rst @@ -31,12 +31,12 @@ Once extracted, start ElasticSearch with:: # ./bin/elasticsearch For more detailed information, refer to the ElasticSearch installation -documentation: http://www.elasticsearch.org/guide/reference/setup/installation.html +documentation: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html Compiling Bro with ElasticSearch Support ---------------------------------------- -First, ensure that you have libcurl installed the run configure.:: +First, ensure that you have libcurl installed then run configure:: # ./configure [...] @@ -51,9 +51,9 @@ First, ensure that you have libcurl installed the run configure.:: Activating ElasticSearch ------------------------ -The easiest way to enable ElasticSearch output is to load the tuning/logs-to- -elasticsearch.bro script. If you are using BroControl, the following line in -local.bro will enable it. +The easiest way to enable ElasticSearch output is to load the +tuning/logs-to-elasticsearch.bro script. If you are using BroControl, +the following line in local.bro will enable it: .. console:: @@ -76,7 +76,7 @@ A common problem encountered with ElasticSearch is too many files being held open. The ElasticSearch website has some suggestions on how to increase the open file limit. - - http://www.elasticsearch.org/tutorials/2011/04/06/too-many-open-files.html + - http://www.elasticsearch.org/tutorials/too-many-open-files/ TODO ---- From b1a6692ebfd93c3ec3cccbedf583b0e947f427d3 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 9 Oct 2013 17:33:40 -0500 Subject: [PATCH 02/36] Update installation instructions Added a few missing packages to the install instructions. Combined the two different sets of GeoIP install instructions into one location and updated it. Fixed a couple minor typos. --- doc/frameworks/geoip.rst | 71 ++++++++++++++++++++++++---------------- doc/install/install.rst | 53 ++++++------------------------ 2 files changed, 53 insertions(+), 71 deletions(-) diff --git a/doc/frameworks/geoip.rst b/doc/frameworks/geoip.rst index f54ea03096..98252d7184 100644 --- a/doc/frameworks/geoip.rst +++ b/doc/frameworks/geoip.rst @@ -11,10 +11,41 @@ GeoLocation to find the geographic location for an IP address. Bro has support for the `GeoIP library `__ at the policy script level beginning with release 1.3 to account for this - need. + need. To use this functionality, you need to first install the libGeoIP + software, and then install the GeoLite city database before building + Bro. .. contents:: +Install libGeoIP +---------------- + +* FreeBSD: + + .. console:: + + sudo pkg_add -r GeoIP + +* RPM/RedHat-based Linux: + + .. console:: + + sudo yum install GeoIP-devel + +* DEB/Debian-based Linux: + + .. console:: + + sudo apt-get install libgeoip-dev + +* Mac OS X: + + Vanilla OS X installations don't ship with libGeoIP, but if + installed from your preferred package management system (e.g. + MacPorts, Fink, or Homebrew), they should be automatically detected + and Bro will compile against them. + + GeoIPLite Database Installation ------------------------------------ @@ -22,39 +53,23 @@ A country database for GeoIPLite is included when you do the C API install, but for Bro, we are using the city database which includes cities and regions in addition to countries. -`Download `__ the geolitecity -binary database and follow the directions to install it. +`Download `__ the GeoLite city +binary database. -FreeBSD Quick Install ---------------------- + .. console:: -.. console:: - - pkg_add -r GeoIP wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz gunzip GeoLiteCity.dat.gz - mv GeoLiteCity.dat /usr/local/share/GeoIP/GeoIPCity.dat + +Next, the file needs to be put in the database directory. This directory +should already exist and will vary depending on which platform and package +you are using. For FreeBSD, use ``/usr/local/share/GeoIP``. For Linux, +use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one +already exists). - # Set your environment correctly before running Bro's configure script - export CFLAGS=-I/usr/local/include - export LDFLAGS=-L/usr/local/lib + .. console:: - -CentOS Quick Install --------------------- - -.. console:: - - yum install GeoIP-devel - - wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz - gunzip GeoLiteCity.dat.gz - mkdir -p /var/lib/GeoIP/ - mv GeoLiteCity.dat /var/lib/GeoIP/GeoIPCity.dat - - # Set your environment correctly before running Bro's configure script - export CFLAGS=-I/usr/local/include - export LDFLAGS=-L/usr/local/lib + mv GeoLiteCity.dat /GeoIPCity.dat Usage diff --git a/doc/install/install.rst b/doc/install/install.rst index a29be72472..99e7d7e3dd 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -32,6 +32,7 @@ before you begin: * Libmagic 5.04 or greater * Libz * Bash (for BroControl) + * Python (for BroControl) To build Bro from source, the following additional dependencies are required: @@ -47,7 +48,8 @@ To build Bro from source, the following additional dependencies are required: * zlib headers * Perl -To install the required dependencies, you can use: +To install the required dependencies, you can use (when done, make sure +that ``bash`` and ``python`` are in your ``PATH``): * RPM/RedHat-based Linux: @@ -70,10 +72,6 @@ To install the required dependencies, you can use: sudo pkg_add -r bash cmake swig bison python - Note that ``bash`` needs to be in ``PATH``, which by default it is - not. The FreeBSD package installs the binary into - ``/usr/local/bin``. - * Mac OS X: Compiling source code on Macs requires first downloading Xcode_, @@ -87,8 +85,8 @@ To install the required dependencies, you can use: preferred Mac OS X package management system (e.g. MacPorts_, Fink_, or Homebrew_). - Specifically for MacPorts, the ``swig``, ``swig-ruby``, ``swig-python`` - and ``file`` packages provide the required dependencies. + Specifically for MacPorts, the ``cmake``, ``swig``, ``swig-ruby``, + ``swig-python`` and ``file`` packages provide the required dependencies. Optional Dependencies @@ -98,45 +96,14 @@ Bro can make use of some optional libraries and tools if they are found at build time: * LibGeoIP (for geo-locating IP addresses) + * sendmail (enables Bro and BroControl to send mail) * gperftools (tcmalloc is used to improve memory and CPU usage) * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump) * Ruby executable, library, and headers (for Broccoli Ruby bindings) -LibGeoIP is probably the most interesting and can be easily installed -on most platforms: - -* RedHat Enterprise Linux: - - .. console:: - - sudo yum install geoip-devel sendmail - -* CentOS Linux: - - .. console:: - - sudo yum install GeoIP-devel sendmail - -* DEB/Debian-based Linux: - - .. console:: - - sudo apt-get install libgeoip-dev sendmail - -* FreeBSD using ports: - - .. console:: - - sudo pkg_add -r GeoIP - -* Mac OS X: - - Vanilla OS X installations don't ship with libGeoIP, but if - installed from your preferred package management system (e.g. - MacPorts, Fink, or Homebrew), they should be automatically detected - and Bro will compile against them. - -Additional steps may be needed to :ref:`get the right GeoIP database +LibGeoIP is probably the most interesting and can be installed +on most platforms by following the instructions for :ref:`installing +libGeoIP and the GeoIP database `. @@ -217,7 +184,7 @@ turn off unwanted auxiliary projects that would otherwise be installed automatically. Finally, use ``make install-aux`` to install some of the other programs that are in the ``aux/bro-aux`` directory. -OpenBSD users, please see our at `FAQ +OpenBSD users, please see our `FAQ `_ if you are having problems installing Bro. From bd47115c84a7ef29110b11809462f68093772df5 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 9 Oct 2013 23:41:37 -0500 Subject: [PATCH 03/36] Convert File Analysis examples to using btest sphinx Converted hard-coded examples in the File Analysis framework document to using btest sphinx. --- doc/frameworks/file-analysis.rst | 94 ++++--------------- doc/frameworks/file_analysis_01.bro | 20 ++++ doc/frameworks/file_analysis_02.bro | 11 +++ doc/frameworks/file_analysis_03.bro | 25 +++++ .../btest-doc.sphinx.file-analysis-01#1 | 17 ++++ .../btest-doc.sphinx.file-analysis-02#1 | 10 ++ .../btest-doc.sphinx.file-analysis-03#1 | 11 +++ .../output | 24 +++++ .../output | 15 +++ .../output | 29 ++++++ .../btest/doc/sphinx/file-analysis-01.btest | 1 + .../btest/doc/sphinx/file-analysis-02.btest | 1 + .../btest/doc/sphinx/file-analysis-03.btest | 2 + ...-doc_frameworks_file_analysis_01_bro.btest | 24 +++++ ...-doc_frameworks_file_analysis_02_bro.btest | 15 +++ ...-doc_frameworks_file_analysis_03_bro.btest | 29 ++++++ 16 files changed, 251 insertions(+), 77 deletions(-) create mode 100644 doc/frameworks/file_analysis_01.bro create mode 100644 doc/frameworks/file_analysis_02.bro create mode 100644 doc/frameworks/file_analysis_03.bro create mode 100644 testing/btest/Baseline/doc.sphinx.file-analysis-01/btest-doc.sphinx.file-analysis-01#1 create mode 100644 testing/btest/Baseline/doc.sphinx.file-analysis-02/btest-doc.sphinx.file-analysis-02#1 create mode 100644 testing/btest/Baseline/doc.sphinx.file-analysis-03/btest-doc.sphinx.file-analysis-03#1 create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_01_bro/output create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_03_bro/output create mode 100644 testing/btest/doc/sphinx/file-analysis-01.btest create mode 100644 testing/btest/doc/sphinx/file-analysis-02.btest create mode 100644 testing/btest/doc/sphinx/file-analysis-03.btest create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_01_bro.btest create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_03_bro.btest diff --git a/doc/frameworks/file-analysis.rst b/doc/frameworks/file-analysis.rst index 0a96a8efb7..50fb89317c 100644 --- a/doc/frameworks/file-analysis.rst +++ b/doc/frameworks/file-analysis.rst @@ -31,40 +31,13 @@ some information about the file such as which network :bro:see:`connection` and protocol are transporting the file, how many bytes have been transferred so far, and its MIME type. -.. code:: bro +Here's a simple example: - event connection_state_remove(c: connection) - { - print "connection_state_remove"; - print c$uid; - print c$id; - for ( s in c$service ) - print s; - } +.. btest-include:: ${DOC_ROOT}/frameworks/file_analysis_01.bro - event file_state_remove(f: fa_file) - { - print "file_state_remove"; - print f$id; - for ( cid in f$conns ) - { - print f$conns[cid]$uid; - print cid; - } - print f$source; - } +.. btest:: file-analysis-01 -might give output like:: - - file_state_remove - Cx92a0ym5R8 - REs2LQfVW2j - [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] - HTTP - connection_state_remove - REs2LQfVW2j - [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] - HTTP + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/get.trace ${DOC_ROOT}/frameworks/file_analysis_01.bro This doesn't perform any interesting analysis yet, but does highlight the similarity between analysis of connections and files. Connections @@ -90,27 +63,16 @@ will write the contents of the file out to the local file system). In the future there may be file analyzers that automatically attach to files based on heuristics, similar to the Dynamic Protocol Detection (DPD) framework for connections, but many will always require an -explicit attachment decision: +explicit attachment decision. -.. code:: bro +Here's a simple example of how to use the MD5 file analyzer to +calculate the MD5 of plain text files: - event file_new(f: fa_file) - { - print "new file", f$id; - if ( f?$mime_type && f$mime_type == "text/plain" ) - Files::add_analyzer(f, Files::ANALYZER_MD5); - } +.. btest-include:: ${DOC_ROOT}/frameworks/file_analysis_02.bro - event file_hash(f: fa_file, kind: string, hash: string) - { - print "file_hash", f$id, kind, hash; - } +.. btest:: file-analysis-02 -this script calculates MD5s for all plain text files and might give -output:: - - new file, Cx92a0ym5R8 - file_hash, Cx92a0ym5R8, md5, 397168fd09991a0e712254df7bc639ac + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/get.trace ${DOC_ROOT}/frameworks/file_analysis_02.bro Some file analyzers might have tunable parameters that need to be specified in the call to :bro:see:`Files::add_analyzer`: @@ -144,41 +106,19 @@ in the same way it analyzes files that it sees coming over traffic from a network interface it's monitoring. It only requires a call to :bro:see:`Input::add_analysis`: -.. code:: bro - - redef exit_only_after_terminate = T; - - event file_new(f: fa_file) - { - print "new file", f$id; - Files::add_analyzer(f, Files::ANALYZER_MD5); - } - - event file_state_remove(f: fa_file) - { - Input::remove(f$source); - terminate(); - } - - event file_hash(f: fa_file, kind: string, hash: string) - { - print "file_hash", f$id, kind, hash; - } - - event bro_init() - { - local source: string = "./myfile"; - Input::add_analysis([$source=source, $name=source]); - } +.. btest-include:: ${DOC_ROOT}/frameworks/file_analysis_03.bro Note that the "source" field of :bro:see:`fa_file` corresponds to the "name" field of :bro:see:`Input::AnalysisDescription` since that is what the input framework uses to uniquely identify an input stream. -The output of the above script may be:: +The output of the above script may be (assuming a file called "myfile" +exists): - new file, G1fS2xthS4l - file_hash, G1fS2xthS4l, md5, 54098b367d2e87b078671fad4afb9dbb +.. btest:: file-analysis-03 + + @TEST-EXEC: echo "Hello world" > myfile + @TEST-EXEC: btest-rst-cmd bro ${DOC_ROOT}/frameworks/file_analysis_03.bro Nothing that special, but it at least verifies the MD5 file analyzer saw all the bytes of the input file and calculated the checksum diff --git a/doc/frameworks/file_analysis_01.bro b/doc/frameworks/file_analysis_01.bro new file mode 100644 index 0000000000..a48f8184ad --- /dev/null +++ b/doc/frameworks/file_analysis_01.bro @@ -0,0 +1,20 @@ +event connection_state_remove(c: connection) + { + print "connection_state_remove"; + print c$uid; + print c$id; + for ( s in c$service ) + print s; + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + print f$id; + for ( cid in f$conns ) + { + print f$conns[cid]$uid; + print cid; + } + print f$source; + } diff --git a/doc/frameworks/file_analysis_02.bro b/doc/frameworks/file_analysis_02.bro new file mode 100644 index 0000000000..3ef32d865c --- /dev/null +++ b/doc/frameworks/file_analysis_02.bro @@ -0,0 +1,11 @@ +event file_new(f: fa_file) + { + print "new file", f$id; + if ( f?$mime_type && f$mime_type == "text/plain" ) + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } diff --git a/doc/frameworks/file_analysis_03.bro b/doc/frameworks/file_analysis_03.bro new file mode 100644 index 0000000000..4b1185e045 --- /dev/null +++ b/doc/frameworks/file_analysis_03.bro @@ -0,0 +1,25 @@ +redef exit_only_after_terminate = T; + +event file_new(f: fa_file) + { + print "new file", f$id; + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + Input::remove(f$source); + terminate(); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } + +event bro_init() + { + local source: string = "./myfile"; + Input::add_analysis([$source=source, $name=source]); + } diff --git a/testing/btest/Baseline/doc.sphinx.file-analysis-01/btest-doc.sphinx.file-analysis-01#1 b/testing/btest/Baseline/doc.sphinx.file-analysis-01/btest-doc.sphinx.file-analysis-01#1 new file mode 100644 index 0000000000..41dddf5500 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.file-analysis-01/btest-doc.sphinx.file-analysis-01#1 @@ -0,0 +1,17 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/get.trace file_analysis_01.bro + file_state_remove + FakNcS1Jfe01uljb3 + CXWv6p3arKYeMETxOg + [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] + HTTP + connection_state_remove + CXWv6p3arKYeMETxOg + [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] + HTTP + diff --git a/testing/btest/Baseline/doc.sphinx.file-analysis-02/btest-doc.sphinx.file-analysis-02#1 b/testing/btest/Baseline/doc.sphinx.file-analysis-02/btest-doc.sphinx.file-analysis-02#1 new file mode 100644 index 0000000000..ec727a39d5 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.file-analysis-02/btest-doc.sphinx.file-analysis-02#1 @@ -0,0 +1,10 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r http/get.trace file_analysis_02.bro + new file, FakNcS1Jfe01uljb3 + file_hash, FakNcS1Jfe01uljb3, md5, 397168fd09991a0e712254df7bc639ac + diff --git a/testing/btest/Baseline/doc.sphinx.file-analysis-03/btest-doc.sphinx.file-analysis-03#1 b/testing/btest/Baseline/doc.sphinx.file-analysis-03/btest-doc.sphinx.file-analysis-03#1 new file mode 100644 index 0000000000..9a01c7c4af --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.file-analysis-03/btest-doc.sphinx.file-analysis-03#1 @@ -0,0 +1,11 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro file_analysis_03.bro + new file, FZedLu4Ajcvge02jA8 + file_hash, FZedLu4Ajcvge02jA8, md5, f0ef7081e1539ac00ef5b761b4fb01b3 + file_state_remove + diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_01_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_01_bro/output new file mode 100644 index 0000000000..045fa5ff9f --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_01_bro/output @@ -0,0 +1,24 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_analysis_01.bro + +event connection_state_remove(c: connection) + { + print "connection_state_remove"; + print c$uid; + print c$id; + for ( s in c$service ) + print s; + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + print f$id; + for ( cid in f$conns ) + { + print f$conns[cid]$uid; + print cid; + } + print f$source; + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output new file mode 100644 index 0000000000..3b93ee757c --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output @@ -0,0 +1,15 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_analysis_02.bro + +event file_new(f: fa_file) + { + print "new file", f$id; + if ( f?$mime_type && f$mime_type == "text/plain" ) + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_03_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_03_bro/output new file mode 100644 index 0000000000..4084169945 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_03_bro/output @@ -0,0 +1,29 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_analysis_03.bro + +redef exit_only_after_terminate = T; + +event file_new(f: fa_file) + { + print "new file", f$id; + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + Input::remove(f$source); + terminate(); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } + +event bro_init() + { + local source: string = "./myfile"; + Input::add_analysis([$source=source, $name=source]); + } diff --git a/testing/btest/doc/sphinx/file-analysis-01.btest b/testing/btest/doc/sphinx/file-analysis-01.btest new file mode 100644 index 0000000000..6dac37ee38 --- /dev/null +++ b/testing/btest/doc/sphinx/file-analysis-01.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/get.trace ${DOC_ROOT}/frameworks/file_analysis_01.bro diff --git a/testing/btest/doc/sphinx/file-analysis-02.btest b/testing/btest/doc/sphinx/file-analysis-02.btest new file mode 100644 index 0000000000..ea359e1bca --- /dev/null +++ b/testing/btest/doc/sphinx/file-analysis-02.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/get.trace ${DOC_ROOT}/frameworks/file_analysis_02.bro diff --git a/testing/btest/doc/sphinx/file-analysis-03.btest b/testing/btest/doc/sphinx/file-analysis-03.btest new file mode 100644 index 0000000000..6ad81d6a76 --- /dev/null +++ b/testing/btest/doc/sphinx/file-analysis-03.btest @@ -0,0 +1,2 @@ +@TEST-EXEC: echo "Hello world" > myfile +@TEST-EXEC: btest-rst-cmd bro ${DOC_ROOT}/frameworks/file_analysis_03.bro diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_01_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_01_bro.btest new file mode 100644 index 0000000000..045fa5ff9f --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_01_bro.btest @@ -0,0 +1,24 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_analysis_01.bro + +event connection_state_remove(c: connection) + { + print "connection_state_remove"; + print c$uid; + print c$id; + for ( s in c$service ) + print s; + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + print f$id; + for ( cid in f$conns ) + { + print f$conns[cid]$uid; + print cid; + } + print f$source; + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest new file mode 100644 index 0000000000..3b93ee757c --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest @@ -0,0 +1,15 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_analysis_02.bro + +event file_new(f: fa_file) + { + print "new file", f$id; + if ( f?$mime_type && f$mime_type == "text/plain" ) + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_03_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_03_bro.btest new file mode 100644 index 0000000000..4084169945 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_03_bro.btest @@ -0,0 +1,29 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +file_analysis_03.bro + +redef exit_only_after_terminate = T; + +event file_new(f: fa_file) + { + print "new file", f$id; + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + Input::remove(f$source); + terminate(); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } + +event bro_init() + { + local source: string = "./myfile"; + Input::add_analysis([$source=source, $name=source]); + } From 1a36031874bdbec6de16b6fbc6a5af2a7501a6fa Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 00:53:13 -0500 Subject: [PATCH 04/36] Fix typos and formatting in the input framework docs --- scripts/base/frameworks/input/main.bro | 74 ++++++++++--------- .../base/frameworks/input/readers/ascii.bro | 4 +- .../frameworks/input/readers/benchmark.bro | 14 ++-- scripts/base/frameworks/input/readers/raw.bro | 10 +-- 4 files changed, 54 insertions(+), 48 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index e5d74cbc36..7ca6e1c72b 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -33,45 +33,45 @@ export { ## that contain types that are not supported (at the moment ## file and function). If true, the input framework will ## warn in these cases, but continue. If false, it will - ## abort. Defaults to false (abort) + ## abort. Defaults to false (abort). const accept_unsupported_types = F &redef; ## TableFilter description type used for the `table` method. type TableDescription: record { - ## Common definitions for tables and events + # Common definitions for tables and events ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; - ## Reader to use for this stream + ## Reader to use for this stream. reader: Reader &default=default_reader; - ## Read mode to use for this stream + ## Read mode to use for this stream. mode: Mode &default=default_mode; - ## Descriptive name. Used to remove a stream at a later time + ## Descriptive name. Used to remove a stream at a later time. name: string; # Special definitions for tables - ## Table which will receive the data read by the input framework + ## Table which will receive the data read by the input framework. destination: any; - ## Record that defines the values used as the index of the table + ## Record that defines the values used as the index of the table. idx: any; - ## Record that defines the values used as the elements of the table - ## If val is undefined, destination has to be a set. + ## Record that defines the values used as the elements of the table. + ## If this is undefined, then *destination* has to be a set. val: any &optional; - ## Defines if the value of the table is a record (default), or a single value. Val - ## can only contain one element when this is set to false. + ## Defines if the value of the table is a record (default), or a single value. + ## When this is set to false, then *val* can only contain one element. want_record: bool &default=T; ## The event that is raised each time a value is added to, changed in or removed ## from the table. The event will receive an Input::Event enum as the first - ## argument, the idx record as the second argument and the value (record) as the + ## argument, the *idx* record as the second argument and the value (record) as the ## third argument. ev: any &optional; # event containing idx, val as values. @@ -88,19 +88,19 @@ export { ## EventFilter description type used for the `event` method. type EventDescription: record { - ## Common definitions for tables and events + # Common definitions for tables and events ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; - ## Reader to use for this steam + ## Reader to use for this stream. reader: Reader &default=default_reader; - ## Read mode to use for this stream + ## Read mode to use for this stream. mode: Mode &default=default_mode; - ## Descriptive name. Used to remove a stream at a later time + ## Descriptive name. Used to remove a stream at a later time. name: string; # Special definitions for events @@ -108,8 +108,8 @@ export { ## Record describing the fields to be retrieved from the source input. fields: any; - ## If want_record if false, the event receives each value in fields as a separate argument. - ## If it is set to true (default), the event receives all fields in a single record value. + ## If this is false, the event receives each value in fields as a separate argument. + ## If this is set to true (default), the event receives all fields in a single record value. want_record: bool &default=T; ## The event that is raised each time a new line is received from the reader. @@ -122,23 +122,23 @@ export { config: table[string] of string &default=table(); }; - ## A file analyis input stream type used to forward input data to the + ## A file analysis input stream type used to forward input data to the ## file analysis framework. type AnalysisDescription: record { ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; - ## Reader to use for this steam. Compatible readers must be + ## Reader to use for this stream. Compatible readers must be ## able to accept a filter of a single string type (i.e. ## they read a byte stream). reader: Reader &default=Input::READER_BINARY; - ## Read mode to use for this stream + ## Read mode to use for this stream. mode: Mode &default=default_mode; ## Descriptive name that uniquely identifies the input source. - ## Can be used used to remove a stream at a later time. + ## Can be used to remove a stream at a later time. ## This will also be used for the unique *source* field of ## :bro:see:`fa_file`. Most of the time, the best choice for this ## field will be the same value as the *source* field. @@ -150,38 +150,44 @@ export { config: table[string] of string &default=table(); }; - ## Create a new table input from a given source. Returns true on success. + ## Create a new table input from a given source. ## ## description: `TableDescription` record describing the source. + ## + ## Returns: true on success. global add_table: function(description: Input::TableDescription) : bool; - ## Create a new event input from a given source. Returns true on success. + ## Create a new event input from a given source. ## - ## description: `TableDescription` record describing the source. + ## description: `EventDescription` record describing the source. + ## + ## Returns: true on success. global add_event: function(description: Input::EventDescription) : bool; ## Create a new file analysis input from a given source. Data read from ## the source is automatically forwarded to the file analysis framework. ## - ## description: A record describing the source + ## description: A record describing the source. ## - ## Returns: true on sucess. + ## Returns: true on success. global add_analysis: function(description: Input::AnalysisDescription) : bool; - ## Remove a input stream. Returns true on success and false if the named stream was - ## not found. + ## Remove an input stream. ## - ## id: string value identifying the stream to be removed + ## id: string value identifying the stream to be removed. + ## + ## Returns: true on success and false if the named stream was not found. global remove: function(id: string) : bool; ## Forces the current input to be checked for changes. - ## Returns true on success and false if the named stream was not found ## - ## id: string value identifying the stream + ## id: string value identifying the stream. + ## + ## Returns: true on success and false if the named stream was not found. global force_update: function(id: string) : bool; - ## Event that is called, when the end of a data source has been reached, including - ## after an update. + ## Event that is called when the end of a data source has been reached, + ## including after an update. global end_of_data: event(name: string, source:string); } diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index d46ab6f67a..1b486ddba0 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -6,11 +6,11 @@ module InputAscii; export { ## Separator between fields. - ## Please note that the separator has to be exactly one character long + ## Please note that the separator has to be exactly one character long. const separator = Input::separator &redef; ## Separator between set elements. - ## Please note that the separator has to be exactly one character long + ## Please note that the separator has to be exactly one character long. const set_separator = Input::set_separator &redef; ## String to use for empty fields. diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index fe44914271..0d79bd9163 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -1,23 +1,23 @@ -##! Interface for the ascii input reader. +##! Interface for the benchmark input reader. module InputBenchmark; export { - ## multiplication factor for each second + ## Multiplication factor for each second. const factor = 1.0 &redef; - ## spread factor between lines + ## Spread factor between lines. const spread = 0 &redef; - ## spreading where usleep = 1000000 / autospread * num_lines + ## Spreading where usleep = 1000000 / autospread * num_lines const autospread = 0.0 &redef; - ## addition factor for each heartbeat + ## Addition factor for each heartbeat. const addfactor = 0 &redef; - ## stop spreading at x lines per heartbeat + ## Stop spreading at x lines per heartbeat. const stopspreadat = 0 &redef; - ## 1 -> enable timed spreading + ## 1 -> enable timed spreading. const timedspread = 0.0 &redef; } diff --git a/scripts/base/frameworks/input/readers/raw.bro b/scripts/base/frameworks/input/readers/raw.bro index 9fa7d1a2fb..b1e0fb6831 100644 --- a/scripts/base/frameworks/input/readers/raw.bro +++ b/scripts/base/frameworks/input/readers/raw.bro @@ -4,14 +4,14 @@ module InputRaw; export { ## Separator between input records. - ## Please note that the separator has to be exactly one character long + ## Please note that the separator has to be exactly one character long. const record_separator = "\n" &redef; ## Event that is called when a process created by the raw reader exits. ## - ## name: name of the input stream - ## source: source of the input stream - ## exit_code: exit code of the program, or number of the signal that forced the program to exit - ## signal_exit: false when program exitted normally, true when program was forced to exit by a signal + ## name: name of the input stream. + ## source: source of the input stream. + ## exit_code: exit code of the program, or number of the signal that forced the program to exit. + ## signal_exit: false when program exited normally, true when program was forced to exit by a signal. global process_finished: event(name: string, source:string, exit_code:count, signal_exit:bool); } From 0d712b35d88210b5cf1ae551ab81ce934f00e8ca Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 12:23:34 -0500 Subject: [PATCH 05/36] Fix typos and a broken link in the file analysis doc --- scripts/base/frameworks/files/main.bro | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index c1883e037f..3cbb0f644b 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -50,9 +50,9 @@ export { ## Connection UIDS over which the file was transferred. conn_uids: set[string] &log; - ## An identification of the source of the file data. E.g. it may be - ## a network protocol over which it was transferred, or a local file - ## path which was read, or some other input source. + ## An identification of the source of the file data. E.g. it + ## may be a network protocol over which it was transferred, or a + ## local file path which was read, or some other input source. source: string &log &optional; ## A value to represent the depth of this file in relation @@ -79,12 +79,12 @@ export { ## If the source of this file is a network connection, this field ## indicates if the data originated from the local network or not as - ## determined by the configured bro:see:`Site::local_nets`. + ## determined by the configured :bro:see:`Site::local_nets`. local_orig: bool &log &optional; ## If the source of this file is a network connection, this field - ## indicates if the file is being sent by the originator of the connection - ## or the responder. + ## indicates if the file is being sent by the originator of the + ## connection or the responder. is_orig: bool &log &optional; ## Number of bytes provided to the file analysis engine for the file. @@ -116,15 +116,15 @@ export { ## The salt concatenated to unique file handle strings generated by ## :bro:see:`get_file_handle` before hashing them in to a file id ## (the *id* field of :bro:see:`fa_file`). - ## Provided to help mitigate the possiblility of manipulating parts of + ## Provided to help mitigate the possibility of manipulating parts of ## network connections that factor in to the file handle in order to ## generate two handles that would hash to the same file id. const salt = "I recommend changing this." &redef; ## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is ## used to determine the length of inactivity that is allowed for a file - ## before internal state related to it is cleaned up. When used within a - ## :bro:see:`file_timeout` handler, the analysis will delay timing out + ## before internal state related to it is cleaned up. When used within + ## a :bro:see:`file_timeout` handler, the analysis will delay timing out ## again for the period specified by *t*. ## ## f: the file. @@ -167,11 +167,12 @@ export { ## f: the file. ## ## Returns: true if analysis for the given file will be ignored for the - ## rest of it's contents, or false if analysis for the *id* + ## rest of its contents, or false if analysis for the *id* ## isn't currently active. global stop: function(f: fa_file): bool; - ## Translates an file analyzer enum value to a string with the analyzer's name. + ## Translates a file analyzer enum value to a string with the analyzer's + ## name. ## ## tag: The analyzer tag. ## @@ -183,7 +184,7 @@ export { ## ## f: The file to be described. ## - ## Returns a text description regarding metadata of the file. + ## Returns: a text description regarding metadata of the file. global describe: function(f: fa_file): string; type ProtoRegistration: record { @@ -198,7 +199,7 @@ export { &default=function(f: fa_file): string { return ""; }; }; - ## Register callbacks for protocols that work with the Files framework. + ## Register callbacks for protocols that work with the Files framework. ## The callbacks must uniquely identify a file and each protocol can ## only have a single callback registered for it. ## @@ -209,10 +210,10 @@ export { ## Returns: true if the protocol being registered was not previously registered. global register_protocol: function(tag: Analyzer::Tag, reg: ProtoRegistration): bool; - ## Register a callback for file analyzers to use if they need to do some manipulation - ## when they are being added to a file before the core code takes over. This is - ## unlikely to be interesting for users and should only be called by file analyzer - ## authors but it *not required*. + ## Register a callback for file analyzers to use if they need to do some + ## manipulation when they are being added to a file before the core code + ## takes over. This is unlikely to be interesting for users and should + ## only be called by file analyzer authors but is *not required*. ## ## tag: Tag for the file analyzer. ## From 7ddc670a026fd8e3331a45a664ad0d610866a339 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 12:46:11 -0500 Subject: [PATCH 06/36] Fix typos and formatting in the file analysis docs --- scripts/base/files/extract/main.bro | 4 ++-- scripts/base/files/unified2/main.bro | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/base/files/extract/main.bro b/scripts/base/files/extract/main.bro index 7030cac148..7095c0dcc7 100644 --- a/scripts/base/files/extract/main.bro +++ b/scripts/base/files/extract/main.bro @@ -8,7 +8,7 @@ export { const prefix = "./extract_files/" &redef; ## The default max size for extracted files (they won't exceed this - ## number of bytes), unlimited. + ## number of bytes). A value of zero means unlimited. const default_limit = 0 &redef; redef record Files::Info += { @@ -19,7 +19,7 @@ export { redef record Files::AnalyzerArgs += { ## The local filename to which to write an extracted file. ## This field is used in the core by the extraction plugin - ## to know where to write the file to. It's also optional + ## to know where to write the file to. It's also optional. extract_filename: string &optional; ## The maximum allowed file size in bytes of *extract_filename*. ## Once reached, a :bro:see:`file_extraction_limit` event is diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index a74c10e908..870f9335ae 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -42,17 +42,17 @@ export { sensor_id: count &log; ## Sig id for this generator. signature_id: count &log; - ## A string representation of the "signature_id" field if a sid_msg.map file was loaded. + ## A string representation of the *signature_id* field if a sid_msg.map file was loaded. signature: string &log &optional; ## Which generator generated the alert? generator_id: count &log; - ## A string representation of the "generator_id" field if a gen_msg.map file was loaded. + ## A string representation of the *generator_id* field if a gen_msg.map file was loaded. generator: string &log &optional; ## Sig revision for this id. signature_revision: count &log; ## Event classification. classification_id: count &log; - ## Descriptive classification string, + ## Descriptive classification string. classification: string &log &optional; ## Event priority. priority_id: count &log; From f21f45ac1a6ac9e0967d9d8e679445604beec017 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 14:39:44 -0500 Subject: [PATCH 07/36] Fix typos and formatting in the cluster framework docs Changed a Time Machine link to point to something useful. --- scripts/base/frameworks/cluster/main.bro | 9 +++++---- scripts/base/frameworks/cluster/nodes/worker.bro | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 4184ad6ded..0ff0f47fa7 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -39,7 +39,8 @@ export { ## The node type doing all the actual traffic analysis. WORKER, ## A node acting as a traffic recorder using the - ## `Time Machine `_ software. + ## `Time Machine `_ + ## software. TIME_MACHINE, }; @@ -58,7 +59,7 @@ export { ## Events raised by workers and handled by a manager. const worker2manager_events = /(TimeMachine::command|Drop::.*)/ &redef; - ## Events raised by workers and handled by proxies.. + ## Events raised by workers and handled by proxies. const worker2proxy_events = /EMPTY/ &redef; ## Events raised by TimeMachine instances and handled by a manager. @@ -73,14 +74,14 @@ export { ## Record type to indicate a node in a cluster. type Node: record { - ## Identifies the type of cluster node in this node's configuration. + ## Identifies the type of cluster node in this node's configuration. node_type: NodeType; ## The IP address of the cluster node. ip: addr; ## If the *ip* field is a non-global IPv6 address, this field ## can specify a particular :rfc:`4007` ``zone_id``. zone_id: string &default=""; - ## The port to which the this local node can connect when + ## The port to which this local node can connect when ## establishing communication. p: port; ## Identifier for the interface a worker is sniffing. diff --git a/scripts/base/frameworks/cluster/nodes/worker.bro b/scripts/base/frameworks/cluster/nodes/worker.bro index 61d5228c88..f876f8592e 100644 --- a/scripts/base/frameworks/cluster/nodes/worker.bro +++ b/scripts/base/frameworks/cluster/nodes/worker.bro @@ -19,6 +19,6 @@ redef Log::default_rotation_postprocessor_cmd = "delete-log"; ## Record all packets into trace file. ## ## Note that this only indicates that *if* we are recording packets, we want all -## of them (rather than just those the core deems sufficiently important). Setting -## this does not turn recording on. Use '-w ' for that. +## of them (rather than just those the core deems sufficiently important). +## Setting this does not turn recording on. Use '-w ' for that. redef record_all_packets = T; From e2c5a5c4a5a5c3dac61e2a800fc1d0ededc16b5e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 14:42:01 -0500 Subject: [PATCH 08/36] Fix typos and formatting in the communication framework doc Fixed a link that was pointing to the wrong "connect". --- .../base/frameworks/communication/main.bro | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index caf5119d9d..47b4120e54 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -42,10 +42,11 @@ export { type Info: record { ## The network time at which a communication event occurred. ts: time &log; - ## The peer name (if any) with which a communication event is concerned. + ## The peer name (if any) with which a communication event is + ## concerned. peer: string &log &optional; - ## Where the communication event message originated from, that is, - ## either from the scripting layer or inside the Bro process. + ## Where the communication event message originated from, that + ## is, either from the scripting layer or inside the Bro process. src_name: string &log &optional; ## .. todo:: currently unused. connected_peer_desc: string &log &optional; @@ -71,8 +72,8 @@ export { ## can specify a particular :rfc:`4007` ``zone_id``. zone_id: string &optional; - ## Port of the remote Bro communication endpoint if we are initiating - ## the connection based on the :bro:id:`connect` field. + ## Port of the remote Bro communication endpoint if we are + ## initiating the connection (based on the *connect* field). p: port &optional; ## When accepting a connection, the configuration only @@ -87,7 +88,7 @@ export { events: pattern &optional; ## Whether we are going to connect (rather than waiting - ## for the other sie to connect to us). + ## for the other side to connect to us). connect: bool &default = F; ## If disconnected, reconnect after this many seconds. @@ -103,13 +104,14 @@ export { request_logs: bool &default = F; ## When performing state synchronization, whether we consider - ## our state to be authoritative. If so, we will send the peer - ## our current set when the connection is set up. - ## (Only one side can be authoritative) + ## our state to be authoritative (only one side can be + ## authoritative). If so, we will send the peer our current + ## set when the connection is set up. auth: bool &default = F; ## If not set, no capture filter is sent. - ## If set to "", the default capture filter is sent. + ## If set to an empty string, then the default capture filter + ## is sent. capture_filter: string &optional; ## Whether to use SSL-based communication. From ff15e0832f3755eb69fb6a9779bdb8d26f58e96f Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 15:23:19 -0500 Subject: [PATCH 09/36] Fix typos and formatting in the intel framework docs --- scripts/base/frameworks/intel/cluster.bro | 4 +-- scripts/base/frameworks/intel/input.bro | 2 +- scripts/base/frameworks/intel/main.bro | 43 ++++++++++++----------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/scripts/base/frameworks/intel/cluster.bro b/scripts/base/frameworks/intel/cluster.bro index 8ed17109c1..7791c334d5 100644 --- a/scripts/base/frameworks/intel/cluster.bro +++ b/scripts/base/frameworks/intel/cluster.bro @@ -1,5 +1,5 @@ -##! Cluster transparency support for the intelligence framework. This is mostly oriented -##! toward distributing intelligence information across clusters. +##! Cluster transparency support for the intelligence framework. This is mostly +##! oriented toward distributing intelligence information across clusters. @load base/frameworks/cluster @load ./input diff --git a/scripts/base/frameworks/intel/input.bro b/scripts/base/frameworks/intel/input.bro index 6ab7c6a674..7b494dcd75 100644 --- a/scripts/base/frameworks/intel/input.bro +++ b/scripts/base/frameworks/intel/input.bro @@ -4,7 +4,7 @@ module Intel; export { ## Intelligence files that will be read off disk. The files are - ## reread everytime they are updated so updates much be atomic with + ## reread every time they are updated so updates must be atomic with ## "mv" instead of writing the file in place. const read_files: set[string] = {} &redef; } diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index b3dcfda00d..fb3a9a3613 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -1,6 +1,6 @@ ##! The intelligence framework provides a way to store and query IP addresses, ##! and strings (with a str_type). Metadata can -##! also be associated with the intelligence like for making more informed +##! also be associated with the intelligence, like for making more informed ##! decisions about matching and handling of intelligence. @load base/frameworks/notice @@ -14,7 +14,7 @@ export { type Type: enum { ## An IP address. ADDR, - ## A complete URL without the prefix "http://". + ## A complete URL without the prefix ``"http://"``. URL, ## Software name. SOFTWARE, @@ -24,21 +24,22 @@ export { DOMAIN, ## A user name. USER_NAME, - ## File hash which is non-hash type specific. It's up to the user to query - ## for any relevant hash types. + ## File hash which is non-hash type specific. It's up to the + ## user to query for any relevant hash types. FILE_HASH, - ## File names. Typically with protocols with definite indications - ## of a file name. + ## File name. Typically with protocols with definite + ## indications of a file name. FILE_NAME, ## Certificate SHA-1 hash. CERT_HASH, }; - ## Data about an :bro:type:`Intel::Item` + ## Data about an :bro:type:`Intel::Item`. type MetaData: record { - ## An arbitrary string value representing the data source. Typically, - ## the convention for this field will be the source name and feed name - ## separated by a hyphen. For example: "source1-c&c". + ## An arbitrary string value representing the data source. + ## Typically, the convention for this field will be the source + ## name and feed name separated by a hyphen. + ## For example: "source1-c&c". source: string; ## A freeform description for the data. desc: string &optional; @@ -81,7 +82,7 @@ export { where: Where &log; ## If the data was discovered within a connection, the - ## connection record should go into get to give context to the data. + ## connection record should go here to give context to the data. conn: connection &optional; ## If the data was discovered within a file, the file record @@ -106,10 +107,12 @@ export { ## this is the uid for the file. fuid: string &log &optional; ## A mime type if the intelligence hit is related to a file. - ## If the $f field is provided this will be automatically filled out. + ## If the $f field is provided this will be automatically filled + ## out. file_mime_type: string &log &optional; ## Frequently files can be "described" to give a bit more context. - ## If the $f field is provided this field will be automatically filled out. + ## If the $f field is provided this field will be automatically + ## filled out. file_desc: string &log &optional; ## Where the data was seen. @@ -125,13 +128,13 @@ export { ## it against known intelligence for matches. global seen: function(s: Seen); - ## Event to represent a match in the intelligence data from data that was seen. - ## On clusters there is no assurance as to where this event will be generated - ## so do not assume that arbitrary global state beyond the given data - ## will be available. + ## Event to represent a match in the intelligence data from data that + ## was seen. On clusters there is no assurance as to where this event + ## will be generated so do not assume that arbitrary global state beyond + ## the given data will be available. ## - ## This is the primary mechanism where a user will take actions based on data - ## within the intelligence framework. + ## This is the primary mechanism where a user will take actions based on + ## data within the intelligence framework. global match: event(s: Seen, items: set[Item]); global log_intel: event(rec: Info); @@ -140,7 +143,7 @@ export { # Internal handler for matches with no metadata available. global match_no_items: event(s: Seen); -# Internal events for cluster data distribution +# Internal events for cluster data distribution. global new_item: event(item: Item); global updated_item: event(item: Item); From ee921a419fc90c0132e4d1f4fbe2f2f731b4ce2f Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 16:10:00 -0500 Subject: [PATCH 10/36] Fix typos and formatting in the logging framework docs --- scripts/base/frameworks/logging/main.bro | 44 ++++++++++--------- .../frameworks/logging/postprocessors/scp.bro | 8 ++-- .../logging/postprocessors/sftp.bro | 8 ++-- .../base/frameworks/logging/writers/ascii.bro | 16 +++---- .../frameworks/logging/writers/dataseries.bro | 18 ++++---- .../logging/writers/elasticsearch.bro | 14 +++--- .../base/frameworks/logging/writers/none.bro | 2 +- .../frameworks/logging/writers/sqlite.bro | 2 +- 8 files changed, 57 insertions(+), 55 deletions(-) diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 0e398ac257..476d314523 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -1,6 +1,6 @@ ##! The Bro logging interface. ##! -##! See :doc:`/frameworks/logging` for a introduction to Bro's +##! See :doc:`/frameworks/logging` for an introduction to Bro's ##! logging framework. module Log; @@ -27,7 +27,7 @@ export { const set_separator = "," &redef; ## String to use for empty fields. This should be different from - ## *unset_field* to make the output non-ambigious. + ## *unset_field* to make the output unambiguous. ## Can be overwritten by individual writers. const empty_field = "(empty)" &redef; @@ -41,8 +41,8 @@ export { columns: any; ## Event that will be raised once for each log entry. - ## The event receives a single same parameter, an instance of type - ## ``columns``. + ## The event receives a single same parameter, an instance of + ## type ``columns``. ev: any &optional; }; @@ -114,7 +114,7 @@ export { ## ## The specific interpretation of the string is up to ## the used writer, and may for example be the destination - ## file name. Generally, filenames are expected to given + ## file name. Generally, filenames are expected to be given ## without any extensions; writers will add appropiate ## extensions automatically. ## @@ -126,34 +126,36 @@ export { path: string &optional; ## A function returning the output path for recording entries - ## matching this filter. This is similar to ``path`` yet allows + ## matching this filter. This is similar to *path* yet allows ## to compute the string dynamically. It is ok to return ## different strings for separate calls, but be careful: it's ## easy to flood the disk by returning a new string for each - ## connection ... + ## connection. ## ## id: The ID associated with the log stream. ## ## path: A suggested path value, which may be either the filter's - ## ``path`` if defined, else a previous result from the function. - ## If no ``path`` is defined for the filter, then the first call - ## to the function will contain an empty string. + ## ``path`` if defined, else a previous result from the + ## function. If no ``path`` is defined for the filter, + ## then the first call to the function will contain an + ## empty string. ## ## rec: An instance of the streams's ``columns`` type with its ## fields set to the values to be logged. ## - ## Returns: The path to be used for the filter, which will be subject - ## to the same automatic correction rules as the *path* - ## field of :bro:type:`Log::Filter` in the case of conflicts - ## with other filters trying to use the same writer/path pair. + ## Returns: The path to be used for the filter, which will be + ## subject to the same automatic correction rules as + ## the *path* field of :bro:type:`Log::Filter` in the + ## case of conflicts with other filters trying to use + ## the same writer/path pair. path_func: function(id: ID, path: string, rec: any): string &optional; ## Subset of column names to record. If not given, all ## columns are recorded. include: set[string] &optional; - ## Subset of column names to exclude from recording. If not given, - ## all columns are recorded. + ## Subset of column names to exclude from recording. If not + ## given, all columns are recorded. exclude: set[string] &optional; ## If true, entries are recorded locally. @@ -229,7 +231,7 @@ export { ## ## filter: A record describing the desired logging parameters. ## - ## Returns: True if the filter was sucessfully added, false if + ## Returns: True if the filter was successfully added, false if ## the filter was not added or the *filter* argument was not ## the correct type. ## @@ -277,7 +279,7 @@ export { ## ## Returns: True if the stream was found and no error occurred in writing ## to it or if the stream was disabled and nothing was written. - ## False if the stream was was not found, or the *columns* + ## False if the stream was not found, or the *columns* ## argument did not match what the stream was initially defined ## to handle, or one of the stream's filters has an invalid ## ``path_func``. @@ -286,8 +288,8 @@ export { global write: function(id: ID, columns: any) : bool; ## Sets the buffering status for all the writers of a given logging stream. - ## A given writer implementation may or may not support buffering and if it - ## doesn't then toggling buffering with this function has no effect. + ## A given writer implementation may or may not support buffering and if + ## it doesn't then toggling buffering with this function has no effect. ## ## id: The ID associated with a logging stream for which to ## enable/disable buffering. @@ -347,7 +349,7 @@ export { ## ## npath: The new path of the file (after already being rotated/processed ## by writer-specific postprocessor as defined in - ## :bro:id:`Log::default_rotation_postprocessors`. + ## :bro:id:`Log::default_rotation_postprocessors`). ## ## Returns: True when :bro:id:`Log::default_rotation_postprocessor_cmd` ## is empty or the system command given by it has been invoked diff --git a/scripts/base/frameworks/logging/postprocessors/scp.bro b/scripts/base/frameworks/logging/postprocessors/scp.bro index 3aadc5bbf3..d63520abe6 100644 --- a/scripts/base/frameworks/logging/postprocessors/scp.bro +++ b/scripts/base/frameworks/logging/postprocessors/scp.bro @@ -16,9 +16,9 @@ module Log; export { - ## Secure-copies the rotated-log to all the remote hosts + ## Secure-copies the rotated log to all the remote hosts ## defined in :bro:id:`Log::scp_destinations` and then deletes - ## the local copy of the rotated-log. It's not active when + ## the local copy of the rotated log. It's not active when ## reading from trace files. ## ## info: A record holding meta-information about the log file to be @@ -42,9 +42,9 @@ export { }; ## A table indexed by a particular log writer and filter path, that yields - ## a set remote destinations. The :bro:id:`Log::scp_postprocessor` + ## a set of remote destinations. The :bro:id:`Log::scp_postprocessor` ## function queries this table upon log rotation and performs a secure - ## copy of the rotated-log to each destination in the set. This + ## copy of the rotated log to each destination in the set. This ## table can be modified at run-time. global scp_destinations: table[Writer, string] of set[SCPDestination]; diff --git a/scripts/base/frameworks/logging/postprocessors/sftp.bro b/scripts/base/frameworks/logging/postprocessors/sftp.bro index 5a31853063..b7f6827026 100644 --- a/scripts/base/frameworks/logging/postprocessors/sftp.bro +++ b/scripts/base/frameworks/logging/postprocessors/sftp.bro @@ -16,9 +16,9 @@ module Log; export { - ## Securely transfers the rotated-log to all the remote hosts + ## Securely transfers the rotated log to all the remote hosts ## defined in :bro:id:`Log::sftp_destinations` and then deletes - ## the local copy of the rotated-log. It's not active when + ## the local copy of the rotated log. It's not active when ## reading from trace files. ## ## info: A record holding meta-information about the log file to be @@ -42,9 +42,9 @@ export { }; ## A table indexed by a particular log writer and filter path, that yields - ## a set remote destinations. The :bro:id:`Log::sftp_postprocessor` + ## a set of remote destinations. The :bro:id:`Log::sftp_postprocessor` ## function queries this table upon log rotation and performs a secure - ## transfer of the rotated-log to each destination in the set. This + ## transfer of the rotated log to each destination in the set. This ## table can be modified at run-time. global sftp_destinations: table[Writer, string] of set[SFTPDestination]; diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index da1cfbde87..e510874951 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -2,10 +2,10 @@ ##! to tweak the output format of ASCII logs. ##! ##! The ASCII writer supports currently one writer-specific filter option via -##! ``config``: setting ``tsv`` to the string ``T`` turns the output into into -##! "tab-separated-value" mode where only a single header row with the column names -##! is printed out as meta information, with no "# fields" prepended; no other meta -##! data gets included in that mode. +##! ``config``: setting ``tsv`` to the string ``T`` turns the output into +##! "tab-separated-value" mode where only a single header row with the column +##! names is printed out as meta information, with no "# fields" prepended; no +##! other meta data gets included in that mode. ##! ##! Example filter using this:: ##! @@ -19,9 +19,9 @@ export { ## into files. This is primarily for debugging purposes. const output_to_stdout = F &redef; - ## If true, include lines with log meta information such as column names with - ## types, the values of ASCII logging options that in use, and the time when the - ## file was opened and closes (the latter at the end). + ## If true, include lines with log meta information such as column names + ## with types, the values of ASCII logging options that are in use, and + ## the time when the file was opened and closed (the latter at the end). const include_meta = T &redef; ## Prefix for lines with meta information. @@ -34,7 +34,7 @@ export { const set_separator = Log::set_separator &redef; ## String to use for empty fields. This should be different from - ## *unset_field* to make the output non-ambigious. + ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; ## String to use for an unset &optional field. diff --git a/scripts/base/frameworks/logging/writers/dataseries.bro b/scripts/base/frameworks/logging/writers/dataseries.bro index e85d9c8c49..0b7b2f5a03 100644 --- a/scripts/base/frameworks/logging/writers/dataseries.bro +++ b/scripts/base/frameworks/logging/writers/dataseries.bro @@ -6,16 +6,16 @@ export { ## Compression to use with the DS output file. Options are: ## ## 'none' -- No compression. - ## 'lzf' -- LZF compression. Very quick, but leads to larger output files. - ## 'lzo' -- LZO compression. Very fast decompression times. - ## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output. - ## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output. + ## 'lzf' -- LZF compression (very quick, but leads to larger output files). + ## 'lzo' -- LZO compression (very fast decompression times). + ## 'gz' -- GZIP compression (slower than LZF, but also produces smaller output). + ## 'bz2' -- BZIP2 compression (slower than GZIP, but also produces smaller output). const compression = "gz" &redef; ## The extent buffer size. - ## Larger values here lead to better compression and more efficient writes, but - ## also increase the lag between the time events are received and the time they - ## are actually written to disk. + ## Larger values here lead to better compression and more efficient writes, + ## but also increase the lag between the time events are received and + ## the time they are actually written to disk. const extent_size = 65536 &redef; ## Should we dump the XML schema we use for this DS file to disk? @@ -43,8 +43,8 @@ export { } # Default function to postprocess a rotated DataSeries log file. It moves the -# rotated file to a new name that includes a timestamp with the opening time, and -# then runs the writer's default postprocessor command on it. +# rotated file to a new name that includes a timestamp with the opening time, +# and then runs the writer's default postprocessor command on it. function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool { # Move file to name including both opening and closing time. diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro index 1901759730..6292876bd0 100644 --- a/scripts/base/frameworks/logging/writers/elasticsearch.bro +++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro @@ -10,16 +10,16 @@ module LogElasticSearch; export { - ## Name of the ES cluster + ## Name of the ES cluster. const cluster_name = "elasticsearch" &redef; - ## ES Server + ## ES server. const server_host = "127.0.0.1" &redef; - ## ES Port + ## ES port. const server_port = 9200 &redef; - ## Name of the ES index + ## Name of the ES index. const index_prefix = "bro" &redef; ## The ES type prefix comes before the name of the related log. @@ -27,9 +27,9 @@ export { const type_prefix = "" &redef; ## The time before an ElasticSearch transfer will timeout. Note that - ## the fractional part of the timeout will be ignored. In particular, time - ## specifications less than a second result in a timeout value of 0, which - ## means "no timeout." + ## the fractional part of the timeout will be ignored. In particular, + ## time specifications less than a second result in a timeout value of + ## 0, which means "no timeout." const transfer_timeout = 2secs; ## The batch size is the number of messages that will be queued up before diff --git a/scripts/base/frameworks/logging/writers/none.bro b/scripts/base/frameworks/logging/writers/none.bro index 869d7246c7..5763b796a9 100644 --- a/scripts/base/frameworks/logging/writers/none.bro +++ b/scripts/base/frameworks/logging/writers/none.bro @@ -1,4 +1,4 @@ -##! Interface for the None log writer. Thiis writer is mainly for debugging. +##! Interface for the None log writer. This writer is mainly for debugging. module LogNone; diff --git a/scripts/base/frameworks/logging/writers/sqlite.bro b/scripts/base/frameworks/logging/writers/sqlite.bro index 0ad8946dcc..5df5e356c8 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.bro +++ b/scripts/base/frameworks/logging/writers/sqlite.bro @@ -11,7 +11,7 @@ export { const unset_field = Log::unset_field &redef; ## String to use for empty fields. This should be different from - ## *unset_field* to make the output non-ambigious. + ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; } From 3812716ace8b6585fc057996ce1e7d06f5354c24 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 20:44:31 -0500 Subject: [PATCH 11/36] Fix typos and formatting in the packet filter docs --- .../base/frameworks/packet-filter/main.bro | 39 ++++++++++--------- .../base/frameworks/packet-filter/utils.bro | 2 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro index 929b10fbe1..4a6e9c43b5 100644 --- a/scripts/base/frameworks/packet-filter/main.bro +++ b/scripts/base/frameworks/packet-filter/main.bro @@ -1,4 +1,4 @@ -##! This script supports how Bro sets it's BPF capture filter. By default +##! This script supports how Bro sets its BPF capture filter. By default ##! Bro sets a capture filter that allows all traffic. If a filter ##! is set on the command line, that filter takes precedence over the default ##! open filter and all filters defined in Bro scripts with the @@ -19,7 +19,7 @@ export { ## This notice is generated if a packet filter cannot be compiled. Compile_Failure, - ## Generated if a packet filter is fails to install. + ## Generated if a packet filter fails to install. Install_Failure, ## Generated when a notice takes too long to compile. @@ -33,8 +33,8 @@ export { ts: time &log; ## This is a string representation of the node that applied this - ## packet filter. It's mostly useful in the context of dynamically - ## changing filters on clusters. + ## packet filter. It's mostly useful in the context of + ## dynamically changing filters on clusters. node: string &log &optional; ## The packet filter that is being set. @@ -48,27 +48,28 @@ export { }; ## The BPF filter that is used by default to define what traffic should - ## be captured. Filters defined in :bro:id:`restrict_filters` will still - ## be applied to reduce the captured traffic. + ## be captured. Filters defined in :bro:id:`restrict_filters` will + ## still be applied to reduce the captured traffic. const default_capture_filter = "ip or not ip" &redef; - ## Filter string which is unconditionally or'ed to the beginning of every - ## dynamically built filter. + ## Filter string which is unconditionally or'ed to the beginning of + ## every dynamically built filter. const unrestricted_filter = "" &redef; - ## Filter string which is unconditionally and'ed to the beginning of every - ## dynamically built filter. This is mostly used when a custom filter is being - ## used but MPLS or VLAN tags are on the traffic. + ## Filter string which is unconditionally and'ed to the beginning of + ## every dynamically built filter. This is mostly used when a custom + ## filter is being used but MPLS or VLAN tags are on the traffic. const restricted_filter = "" &redef; ## The maximum amount of time that you'd like to allow for BPF filters to compile. ## If this time is exceeded, compensation measures may be taken by the framework - ## to reduce the filter size. This threshold being crossed also results in - ## the :bro:see:`PacketFilter::Too_Long_To_Compile_Filter` notice. + ## to reduce the filter size. This threshold being crossed also results + ## in the :bro:see:`PacketFilter::Too_Long_To_Compile_Filter` notice. const max_filter_compile_time = 100msec &redef; - ## Install a BPF filter to exclude some traffic. The filter should positively - ## match what is to be excluded, it will be wrapped in a "not". + ## Install a BPF filter to exclude some traffic. The filter should + ## positively match what is to be excluded, it will be wrapped in + ## a "not". ## ## filter_id: An arbitrary string that can be used to identify ## the filter. @@ -79,9 +80,9 @@ export { ## installed or not. global exclude: function(filter_id: string, filter: string): bool; - ## Install a temporary filter to traffic which should not be passed through - ## the BPF filter. The filter should match the traffic you don't want - ## to see (it will be wrapped in a "not" condition). + ## Install a temporary filter to traffic which should not be passed + ## through the BPF filter. The filter should match the traffic you + ## don't want to see (it will be wrapped in a "not" condition). ## ## filter_id: An arbitrary string that can be used to identify ## the filter. @@ -125,7 +126,7 @@ global dynamic_restrict_filters: table[string] of string = {}; # install the filter. global currently_building = F; -# Internal tracking for if the the filter being built has possibly been changed. +# Internal tracking for if the filter being built has possibly been changed. global filter_changed = F; global filter_plugins: set[FilterPlugin] = {}; diff --git a/scripts/base/frameworks/packet-filter/utils.bro b/scripts/base/frameworks/packet-filter/utils.bro index 7728ebf9f9..29b54229af 100644 --- a/scripts/base/frameworks/packet-filter/utils.bro +++ b/scripts/base/frameworks/packet-filter/utils.bro @@ -13,7 +13,7 @@ export { ## ## num_parts: The number of parts the traffic should be split into. ## - ## this_part: The part of the traffic this filter will accept. 0-based. + ## this_part: The part of the traffic this filter will accept (0-based). global sampling_filter: function(num_parts: count, this_part: count): string; ## Combines two valid BPF filter strings with a string based operator From 0753853726cfbe54d80579197b8608386bfa1844 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 23:15:12 -0500 Subject: [PATCH 12/36] Fix typos and formatting in the sumstats docs --- scripts/base/frameworks/sumstats/cluster.bro | 39 ++++++++++--------- scripts/base/frameworks/sumstats/main.bro | 16 ++++---- .../base/frameworks/sumstats/plugins/last.bro | 4 +- .../frameworks/sumstats/plugins/sample.bro | 7 ++-- .../base/frameworks/sumstats/plugins/topk.bro | 2 +- .../frameworks/sumstats/plugins/unique.bro | 2 +- 6 files changed, 36 insertions(+), 34 deletions(-) diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro index f2b937586d..ed51a95e30 100644 --- a/scripts/base/frameworks/sumstats/cluster.bro +++ b/scripts/base/frameworks/sumstats/cluster.bro @@ -1,6 +1,6 @@ ##! This implements transparent cluster support for the SumStats framework. ##! Do not load this file directly. It's only meant to be loaded automatically -##! and will be depending on if the cluster framework has been enabled. +##! and will be if the cluster framework has been enabled. ##! The goal of this script is to make sumstats calculation completely and ##! transparently automated when running on a cluster. @@ -10,31 +10,32 @@ module SumStats; export { - ## The percent of the full threshold value that needs to be met on a single worker - ## for that worker to send the value to its manager in order for it to request a - ## global view for that value. There is no requirement that the manager requests - ## a global view for the key since it may opt not to if it requested a global view - ## for the key recently. + ## The percent of the full threshold value that needs to be met on a + ## single worker for that worker to send the value to its manager in + ## order for it to request a global view for that value. There is no + ## requirement that the manager requests a global view for the key since + ## it may opt not to if it requested a global view for the key recently. const cluster_request_global_view_percent = 0.2 &redef; - ## This is to deal with intermediate update overload. A manager will only allow - ## this many intermediate update requests to the workers to be inflight at any - ## given time. Requested intermediate updates are currently thrown out and not - ## performed. In practice this should hopefully have a minimal effect. + ## This is to deal with intermediate update overload. A manager will + ## only allow this many intermediate update requests to the workers to + ## be inflight at any given time. Requested intermediate updates are + ## currently thrown out and not performed. In practice this should + ## hopefully have a minimal effect. const max_outstanding_global_views = 10 &redef; - ## Event sent by the manager in a cluster to initiate the collection of values for - ## a sumstat. + ## Event sent by the manager in a cluster to initiate the collection of + ## values for a sumstat. global cluster_ss_request: event(uid: string, ss_name: string, cleanup: bool); - ## Event sent by nodes that are collecting sumstats after receiving a request for - ## the sumstat from the manager. + ## Event sent by nodes that are collecting sumstats after receiving a + ## request for the sumstat from the manager. #global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool); - ## This event is sent by the manager in a cluster to initiate the collection of - ## a single key value from a sumstat. It's typically used to get intermediate - ## updates before the break interval triggers to speed detection of a value - ## crossing a threshold. + ## This event is sent by the manager in a cluster to initiate the + ## collection of a single key value from a sumstat. It's typically used + ## to get intermediate updates before the break interval triggers to + ## speed detection of a value crossing a threshold. global cluster_get_result: event(uid: string, ss_name: string, key: Key, cleanup: bool); ## This event is sent by nodes in response to a @@ -43,7 +44,7 @@ export { ## This is sent by workers to indicate that they crossed the percent ## of the current threshold by the percentage defined globally in - ## :bro:id:`SumStats::cluster_request_global_view_percent` + ## :bro:id:`SumStats::cluster_request_global_view_percent`. global cluster_key_intermediate_response: event(ss_name: string, key: SumStats::Key); ## This event is scheduled internally on workers to send result chunks. diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro index 657a0399c4..8dbdb61edd 100644 --- a/scripts/base/frameworks/sumstats/main.bro +++ b/scripts/base/frameworks/sumstats/main.bro @@ -51,8 +51,8 @@ export { ## would like to accept the data being inserted. pred: function(key: SumStats::Key, obs: SumStats::Observation): bool &optional; - ## A function to normalize the key. This can be used to aggregate or - ## normalize the entire key. + ## A function to normalize the key. This can be used to + ## aggregate or normalize the entire key. normalize_key: function(key: SumStats::Key): Key &optional; }; @@ -91,28 +91,28 @@ export { name: string; ## The interval at which this filter should be "broken" - ## and the '$epoch_result' callback called. The + ## and the *epoch_result* callback called. The ## results are also reset at this time so any threshold ## based detection needs to be set to a ## value that should be expected to happen within ## this epoch. epoch: interval; - ## The reducers for the SumStat + ## The reducers for the SumStat. reducers: set[Reducer]; ## Provide a function to calculate a value from the ## :bro:see:`SumStats::Result` structure which will be used ## for thresholding. - ## This is required if a $threshold value is given. + ## This is required if a *threshold* value is given. threshold_val: function(key: SumStats::Key, result: SumStats::Result): double &optional; ## The threshold value for calling the - ## $threshold_crossed callback. + ## *threshold_crossed* callback. threshold: double &optional; ## A series of thresholds for calling the - ## $threshold_crossed callback. + ## *threshold_crossed* callback. threshold_series: vector of double &optional; ## A callback that is called when a threshold is crossed. @@ -124,7 +124,7 @@ export { epoch_result: function(ts: time, key: SumStats::Key, result: SumStats::Result) &optional; ## A callback that will be called when a single collection - ## interval is completed. The ts value will be the time of + ## interval is completed. The *ts* value will be the time of ## when the collection started. epoch_finished: function(ts:time) &optional; }; diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index 24376f8a2b..430c2e375b 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -5,12 +5,12 @@ module SumStats; export { redef enum Calculation += { - ## Keep last X observations in a queue + ## Keep last X observations in a queue. LAST }; redef record Reducer += { - ## number of elements to keep. + ## Number of elements to keep. num_last_elements: count &default=0; }; diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index c270ef42fd..809d696896 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -4,7 +4,8 @@ module SumStats; export { redef enum Calculation += { - ## Get uniquely distributed random samples from the observation stream. + ## Get uniquely distributed random samples from the observation + ## stream. SAMPLE }; @@ -24,8 +25,8 @@ export { redef record ResultVal += { # Internal use only. This is not meant to be publically available - # and just a copy of num_samples from the Reducer. Needed for availability - # in the compose hook. + # and just a copy of num_samples from the Reducer. Needed for + # availability in the compose hook. num_samples: count &default=0; }; diff --git a/scripts/base/frameworks/sumstats/plugins/topk.bro b/scripts/base/frameworks/sumstats/plugins/topk.bro index cb90af962e..0ef0f01393 100644 --- a/scripts/base/frameworks/sumstats/plugins/topk.bro +++ b/scripts/base/frameworks/sumstats/plugins/topk.bro @@ -4,7 +4,7 @@ module SumStats; export { redef record Reducer += { - ## number of elements to keep in the top-k list + ## Number of elements to keep in the top-k list. topk_size: count &default=500; }; diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro index 60dec66ae4..abfbe3669d 100644 --- a/scripts/base/frameworks/sumstats/plugins/unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/unique.bro @@ -28,7 +28,7 @@ redef record ResultVal += { # Internal use only. This is not meant to be publically available # because we don't want to trust that we can inspect the values - # since we will like move to a probalistic data structure in the future. + # since we will likely move to a probabilistic data structure in the future. # TODO: in the future this will optionally be a hyperloglog structure unique_vals: set[Observation] &optional; }; From 960ddc22d18e836e4ccb3fa76aeee9efcb19d618 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 23:16:19 -0500 Subject: [PATCH 13/36] Fix typos and formatting in the software framework docs --- scripts/base/frameworks/software/main.bro | 33 ++++++++++++----------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 83002481c1..5080c9f888 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -1,5 +1,5 @@ ##! This script provides the framework for software version detection and -##! parsing but doesn't actually do any detection on it's own. It relys on +##! parsing but doesn't actually do any detection on it's own. It relies on ##! other protocol specific scripts to parse out software from the protocols ##! that they analyze. The entry point for providing new software detections ##! to this framework is through the :bro:id:`Software::found` function. @@ -23,15 +23,15 @@ export { ## A structure to represent the numeric version of software. type Version: record { - ## Major version number + ## Major version number. major: count &optional; - ## Minor version number + ## Minor version number. minor: count &optional; - ## Minor subversion number + ## Minor subversion number. minor2: count &optional; - ## Minor updates number + ## Minor updates number. minor3: count &optional; - ## Additional version string (e.g. "beta42") + ## Additional version string (e.g. "beta42"). addl: string &optional; } &log; @@ -41,7 +41,8 @@ export { ts: time &log &optional; ## The IP address detected running the software. host: addr &log; - ## The Port on which the software is running. Only sensible for server software. + ## The port on which the software is running. Only sensible for + ## server software. host_p: port &log &optional; ## The type of software detected (e.g. :bro:enum:`HTTP::SERVER`). software_type: Type &log &default=UNKNOWN; @@ -49,9 +50,9 @@ export { name: string &log &optional; ## Version of the software. version: Version &log &optional; - ## The full unparsed version string found because the version parsing - ## doesn't always work reliably in all cases and this acts as a - ## fallback in the logs. + ## The full unparsed version string found because the version + ## parsing doesn't always work reliably in all cases and this + ## acts as a fallback in the logs. unparsed_version: string &log &optional; ## This can indicate that this software being detected should @@ -59,13 +60,13 @@ export { ## default, only software that is "interesting" due to a change ## in version or it being currently unknown is sent to the ## logging framework. This can be set to T to force the record - ## to be sent to the logging framework if some amount of this tracking - ## needs to happen in a specific way to the software. + ## to be sent to the logging framework if some amount of this + ## tracking needs to happen in a specific way to the software. force_log: bool &default=F; }; ## Hosts whose software should be detected and tracked. - ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS + ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS. const asset_tracking = LOCAL_HOSTS &redef; ## Other scripts should call this function when they detect software. @@ -79,14 +80,14 @@ export { ## Compare two version records. ## ## Returns: -1 for v1 < v2, 0 for v1 == v2, 1 for v1 > v2. - ## If the numerical version numbers match, the addl string + ## If the numerical version numbers match, the *addl* string ## is compared lexicographically. global cmp_versions: function(v1: Version, v2: Version): int; ## Type to represent a collection of :bro:type:`Software::Info` records. ## It's indexed with the name of a piece of software such as "Firefox" - ## and it yields a :bro:type:`Software::Info` record with more information - ## about the software. + ## and it yields a :bro:type:`Software::Info` record with more + ## information about the software. type SoftwareSet: table[string] of Info; ## The set of software associated with an address. Data expires from From 50aca717d05b708d1058f202be8809d56c50e234 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 10 Oct 2013 23:18:27 -0500 Subject: [PATCH 14/36] Fix typos and formatting in various other framework docs --- scripts/base/frameworks/analyzer/main.bro | 18 +++++++++--------- scripts/base/frameworks/control/main.bro | 3 ++- scripts/base/frameworks/reporter/main.bro | 2 +- scripts/base/frameworks/tunnels/main.bro | 8 ++++---- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/base/frameworks/analyzer/main.bro b/scripts/base/frameworks/analyzer/main.bro index e266eb8c7a..39b0d573fd 100644 --- a/scripts/base/frameworks/analyzer/main.bro +++ b/scripts/base/frameworks/analyzer/main.bro @@ -5,8 +5,8 @@ ##! particular analyzer for new connections. ##! ##! Protocol analyzers are identified by unique tags of type -##! :bro:type:`Analyzer::Tag`, such as :bro:enum:`Analyzer::ANALYZER_HTTP` and -##! :bro:enum:`Analyzer::ANALYZER_HTTP`. These tags are defined internally by +##! :bro:type:`Analyzer::Tag`, such as :bro:enum:`Analyzer::ANALYZER_HTTP`. +##! These tags are defined internally by ##! the analyzers themselves, and documented in their analyzer-specific ##! description along with the events that they generate. @@ -15,8 +15,8 @@ module Analyzer; export { - ## If true, all available analyzers are initially disabled at startup. One - ## can then selectively enable them with + ## If true, all available analyzers are initially disabled at startup. + ## One can then selectively enable them with ## :bro:id:`Analyzer::enable_analyzer`. global disable_all = F &redef; @@ -45,7 +45,7 @@ export { ## ## ports: The set of well-known ports to associate with the analyzer. ## - ## Returns: True if the ports were sucessfully registered. + ## Returns: True if the ports were successfully registered. global register_for_ports: function(tag: Analyzer::Tag, ports: set[port]) : bool; ## Registers an individual well-known port for an analyzer. If a future @@ -57,7 +57,7 @@ export { ## ## p: The well-known port to associate with the analyzer. ## - ## Returns: True if the port was sucessfully registered. + ## Returns: True if the port was successfully registered. global register_for_port: function(tag: Analyzer::Tag, p: port) : bool; ## Returns a set of all well-known ports currently registered for a @@ -88,8 +88,8 @@ export { ## Returns: The analyzer tag corresponding to the name. global get_tag: function(name: string): Analyzer::Tag; - ## Schedules an analyzer for a future connection originating from a given IP - ## address and port. + ## Schedules an analyzer for a future connection originating from a + ## given IP address and port. ## ## orig: The IP address originating a connection in the future. ## 0.0.0.0 can be used as a wildcard to match any originator address. @@ -103,7 +103,7 @@ export { ## tout: A timeout interval after which the scheduling request will be ## discarded if the connection has not yet been seen. ## - ## Returns: True if succesful. + ## Returns: True if successful. global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port, analyzer: Analyzer::Tag, tout: interval) : bool; diff --git a/scripts/base/frameworks/control/main.bro b/scripts/base/frameworks/control/main.bro index 63e5f639a0..17003cb5a6 100644 --- a/scripts/base/frameworks/control/main.bro +++ b/scripts/base/frameworks/control/main.bro @@ -57,7 +57,8 @@ export { ## Returns the current net_stats. global net_stats_response: event(s: string); - ## Inform the remote Bro instance that it's configuration may have been updated. + ## Inform the remote Bro instance that it's configuration may have been + ## updated. global configuration_update_request: event(); ## This event is a wrapper and alias for the ## :bro:id:`Control::configuration_update_request` event. diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro index 891aebf6b2..3668348ef1 100644 --- a/scripts/base/frameworks/reporter/main.bro +++ b/scripts/base/frameworks/reporter/main.bro @@ -7,7 +7,7 @@ ##! :bro:see:`Reporter::errors_to_stderr`. ##! ##! Note that this framework deals with the handling of internally generated -##! reporter messages, for the interface in to actually creating interface +##! reporter messages, for the interface ##! into actually creating reporter messages from the scripting layer, use ##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`. diff --git a/scripts/base/frameworks/tunnels/main.bro b/scripts/base/frameworks/tunnels/main.bro index faa725b3f6..04207618d7 100644 --- a/scripts/base/frameworks/tunnels/main.bro +++ b/scripts/base/frameworks/tunnels/main.bro @@ -29,8 +29,8 @@ export { ## The unique identifier for the tunnel, which may correspond ## to a :bro:type:`connection`'s *uid* field for non-IP-in-IP tunnels. ## This is optional because there could be numerous connections - ## for payload proxies like SOCKS but we should treat it as a single - ## tunnel. + ## for payload proxies like SOCKS but we should treat it as a + ## single tunnel. uid: string &log &optional; ## The tunnel "connection" 4-tuple of endpoint addresses/ports. ## For an IP tunnel, the ports will be 0. @@ -76,8 +76,8 @@ export { ## connections before it is considered inactive/expired. const expiration_interval = 1hrs &redef; - ## Currently active tunnels. That is, tunnels for which new, encapsulated - ## connections have been seen in the interval indicated by + ## Currently active tunnels. That is, tunnels for which new, + ## encapsulated connections have been seen in the interval indicated by ## :bro:see:`Tunnel::expiration_interval`. global active: table[conn_id] of Info = table() &read_expire=expiration_interval &expire_func=expire; } From 60b2c5f1fe65820fdc9d38d4f3c4c4d16b5d6597 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 11 Oct 2013 00:19:37 -0500 Subject: [PATCH 15/36] Add README files for most Bro frameworks The text from these README files appears on the "Bro Script Packages" page after building the documentation. The text for these was mostly just copied from the existing docs. --- scripts/base/frameworks/analyzer/README | 3 +++ scripts/base/frameworks/cluster/README | 2 ++ scripts/base/frameworks/communication/README | 2 ++ scripts/base/frameworks/control/README | 3 +++ scripts/base/frameworks/dpd/README | 2 ++ scripts/base/frameworks/files/README | 3 +++ scripts/base/frameworks/input/README | 2 ++ scripts/base/frameworks/intel/README | 3 +++ scripts/base/frameworks/logging/README | 1 + scripts/base/frameworks/packet-filter/README | 1 + scripts/base/frameworks/software/README | 3 +++ scripts/base/frameworks/sumstats/README | 2 ++ scripts/base/frameworks/tunnels/README | 2 ++ 13 files changed, 29 insertions(+) create mode 100644 scripts/base/frameworks/analyzer/README create mode 100644 scripts/base/frameworks/cluster/README create mode 100644 scripts/base/frameworks/communication/README create mode 100644 scripts/base/frameworks/control/README create mode 100644 scripts/base/frameworks/dpd/README create mode 100644 scripts/base/frameworks/files/README create mode 100644 scripts/base/frameworks/input/README create mode 100644 scripts/base/frameworks/intel/README create mode 100644 scripts/base/frameworks/logging/README create mode 100644 scripts/base/frameworks/packet-filter/README create mode 100644 scripts/base/frameworks/software/README create mode 100644 scripts/base/frameworks/sumstats/README create mode 100644 scripts/base/frameworks/tunnels/README diff --git a/scripts/base/frameworks/analyzer/README b/scripts/base/frameworks/analyzer/README new file mode 100644 index 0000000000..2180fdd735 --- /dev/null +++ b/scripts/base/frameworks/analyzer/README @@ -0,0 +1,3 @@ +The analyzer framework allows to dynamically enable or disable Bro's +protocol analyzers, as well as to manage the well-known ports which +automatically activate a particular analyzer for new connections. diff --git a/scripts/base/frameworks/cluster/README b/scripts/base/frameworks/cluster/README new file mode 100644 index 0000000000..1bf9907d9b --- /dev/null +++ b/scripts/base/frameworks/cluster/README @@ -0,0 +1,2 @@ +The cluster framework provides for establishing and controlling a cluster +of Bro instances. diff --git a/scripts/base/frameworks/communication/README b/scripts/base/frameworks/communication/README new file mode 100644 index 0000000000..6f73c8c6a4 --- /dev/null +++ b/scripts/base/frameworks/communication/README @@ -0,0 +1,2 @@ +The communication framework facilitates connecting to remote Bro or +Broccoli instances to share state and transfer events. diff --git a/scripts/base/frameworks/control/README b/scripts/base/frameworks/control/README new file mode 100644 index 0000000000..ba6998d43c --- /dev/null +++ b/scripts/base/frameworks/control/README @@ -0,0 +1,3 @@ +The control framework provides the foundation for providing "commands" +that can be taken remotely at runtime to modify a running Bro instance +or collect information from the running instance. diff --git a/scripts/base/frameworks/dpd/README b/scripts/base/frameworks/dpd/README new file mode 100644 index 0000000000..d7f4d9c06a --- /dev/null +++ b/scripts/base/frameworks/dpd/README @@ -0,0 +1,2 @@ +The DPD (dynamic protocol detection) activates port-independent protocol +detection and selectively disables analyzers if protocol violations occur. diff --git a/scripts/base/frameworks/files/README b/scripts/base/frameworks/files/README new file mode 100644 index 0000000000..3788a60040 --- /dev/null +++ b/scripts/base/frameworks/files/README @@ -0,0 +1,3 @@ +The file analysis framework provides an interface for driving the analysis +of files, possibly independent of any network protocol over which they're +transported. diff --git a/scripts/base/frameworks/input/README b/scripts/base/frameworks/input/README new file mode 100644 index 0000000000..6f00f5f6eb --- /dev/null +++ b/scripts/base/frameworks/input/README @@ -0,0 +1,2 @@ +The input framework provides a way to read previously stored data either as +an event stream or into a Bro table. diff --git a/scripts/base/frameworks/intel/README b/scripts/base/frameworks/intel/README new file mode 100644 index 0000000000..56e00cfac1 --- /dev/null +++ b/scripts/base/frameworks/intel/README @@ -0,0 +1,3 @@ +The intelligence framework provides a way to store and query intelligence +data (such as IP addresses or strings). Metadata can also be associated +with the intelligence. diff --git a/scripts/base/frameworks/logging/README b/scripts/base/frameworks/logging/README new file mode 100644 index 0000000000..122b63d366 --- /dev/null +++ b/scripts/base/frameworks/logging/README @@ -0,0 +1 @@ +The logging framework provides a flexible key-value based logging interface. diff --git a/scripts/base/frameworks/packet-filter/README b/scripts/base/frameworks/packet-filter/README new file mode 100644 index 0000000000..536c1527db --- /dev/null +++ b/scripts/base/frameworks/packet-filter/README @@ -0,0 +1 @@ +The packet filter framework supports how Bro sets its BPF capture filter. diff --git a/scripts/base/frameworks/software/README b/scripts/base/frameworks/software/README new file mode 100644 index 0000000000..f22eb67aab --- /dev/null +++ b/scripts/base/frameworks/software/README @@ -0,0 +1,3 @@ +The software framework doesn't do software version detection and parsing +itself, but instead relies on other protocol specific scripts to parse out +software from the protocols they analyze. diff --git a/scripts/base/frameworks/sumstats/README b/scripts/base/frameworks/sumstats/README new file mode 100644 index 0000000000..12b0f7cd42 --- /dev/null +++ b/scripts/base/frameworks/sumstats/README @@ -0,0 +1,2 @@ +The summary statistics framework provides a way to summarize large streams +of data into simple reduced measurements. diff --git a/scripts/base/frameworks/tunnels/README b/scripts/base/frameworks/tunnels/README new file mode 100644 index 0000000000..56b648a4b5 --- /dev/null +++ b/scripts/base/frameworks/tunnels/README @@ -0,0 +1,2 @@ +The tunnels framework handles the tracking/logging of tunnels (e.g. Teredo, +AYIYA, or IP-in-IP such as 6to4 where "IP" is either IPv4 or IPv6). From b5af5892465e8ee227fea1618ac745ce0a0113fe Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 11 Oct 2013 16:31:53 -0500 Subject: [PATCH 16/36] Improvements to file analysis docs Fixed reference to wrong field name. Added documentation of a function arg. Added a couple references to other parts of the documentation. Explained how not specifying extraction filename results in automatic filename generation. Several other minor clarifications. --- scripts/base/files/extract/main.bro | 7 +++-- scripts/base/frameworks/files/main.bro | 28 +++++++++++-------- src/file_analysis/analyzer/extract/events.bif | 6 ++-- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/scripts/base/files/extract/main.bro b/scripts/base/files/extract/main.bro index 7095c0dcc7..765263a4d8 100644 --- a/scripts/base/files/extract/main.bro +++ b/scripts/base/files/extract/main.bro @@ -12,14 +12,17 @@ export { const default_limit = 0 &redef; redef record Files::Info += { - ## Local filenames of extracted file. + ## Local filename of extracted file. extracted: string &optional &log; }; redef record Files::AnalyzerArgs += { ## The local filename to which to write an extracted file. ## This field is used in the core by the extraction plugin - ## to know where to write the file to. It's also optional. + ## to know where to write the file to. If not specified, then + ## a filename in the format "extract--" is + ## automatically assigned (using the *source* and *id* + ## fields of :bro:see:`fa_file`). extract_filename: string &optional; ## The maximum allowed file size in bytes of *extract_filename*. ## Once reached, a :bro:see:`file_extraction_limit` event is diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index 3cbb0f644b..cf2c11be45 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -14,10 +14,11 @@ export { LOG }; - ## A structure which represents a desired type of file analysis. + ## A structure which parameterizes a type of file analysis. type AnalyzerArgs: record { ## An event which will be generated for all new file contents, - ## chunk-wise. Used when *tag* is + ## chunk-wise. Used when *tag* (in the + ## :bro:see:`Files::add_analyzer` function) is ## :bro:see:`Files::ANALYZER_DATA_EVENT`. chunk_event: event(f: fa_file, data: string, off: count) &optional; @@ -47,7 +48,7 @@ export { ## the data traveled to. rx_hosts: set[addr] &log; - ## Connection UIDS over which the file was transferred. + ## Connection UIDs over which the file was transferred. conn_uids: set[string] &log; ## An identification of the source of the file data. E.g. it @@ -64,9 +65,10 @@ export { ## A set of analysis types done during the file analysis. analyzers: set[string] &log; - ## A mime type provided by libmagic against the *bof_buffer*, or - ## in the cases where no buffering of the beginning of file occurs, - ## an initial guess of the mime type based on the first data seen. + ## A mime type provided by libmagic against the *bof_buffer* + ## field of :bro:see:`fa_file`, or in the cases where no + ## buffering of the beginning of file occurs, an initial + ## guess of the mime type based on the first data seen. mime_type: string &log &optional; ## A filename for the file if one is available from the source @@ -132,7 +134,7 @@ export { ## t: the amount of time the file can remain inactive before discarding. ## ## Returns: true if the timeout interval was set, or false if analysis - ## for the *id* isn't currently active. + ## for the file isn't currently active. global set_timeout_interval: function(f: fa_file, t: interval): bool; ## Adds an analyzer to the analysis of a given file. @@ -144,7 +146,7 @@ export { ## args: any parameters the analyzer takes. ## ## Returns: true if the analyzer will be added, or false if analysis - ## for the *id* isn't currently active or the *args* + ## for the file isn't currently active or the *args* ## were invalid for the analyzer type. global add_analyzer: function(f: fa_file, tag: Files::Tag, @@ -154,10 +156,12 @@ export { ## ## f: the file. ## + ## tag: the analyzer type. + ## ## args: the analyzer (type and args) to remove. ## ## Returns: true if the analyzer will be removed, or false if analysis - ## for the *id* isn't currently active. + ## for the file isn't currently active. global remove_analyzer: function(f: fa_file, tag: Files::Tag, args: AnalyzerArgs &default=AnalyzerArgs()): bool; @@ -167,12 +171,12 @@ export { ## f: the file. ## ## Returns: true if analysis for the given file will be ignored for the - ## rest of its contents, or false if analysis for the *id* + ## rest of its contents, or false if analysis for the file ## isn't currently active. global stop: function(f: fa_file): bool; - ## Translates a file analyzer enum value to a string with the analyzer's - ## name. + ## Translates a file analyzer enum value to a string with the + ## analyzer's name. ## ## tag: The analyzer tag. ## diff --git a/src/file_analysis/analyzer/extract/events.bif b/src/file_analysis/analyzer/extract/events.bif index dc16d066e6..1c08736416 100644 --- a/src/file_analysis/analyzer/extract/events.bif +++ b/src/file_analysis/analyzer/extract/events.bif @@ -1,6 +1,6 @@ ## This event is generated when a file extraction analyzer is about -## to exceed the maximum permitted file size allowed by -## *extract_size_limit* field of :bro:see:`Files::AnalyzerArgs`. +## to exceed the maximum permitted file size allowed by the +## *extract_limit* field of :bro:see:`Files::AnalyzerArgs`. ## The analyzer is automatically removed from file *f*. ## ## f: The file. @@ -13,7 +13,7 @@ ## ## offset: The offset at which a file chunk is about to be written. ## -## len:: The length of the file chunk about to be written. +## len: The length of the file chunk about to be written. ## ## .. bro:see:: Files::add_analyzer Files::ANALYZER_EXTRACT event file_extraction_limit%(f: fa_file, args: any, limit: count, offset: count, len: count%); From 72129ae7cf9770ba920108a6735b1057c8a595db Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 14 Oct 2013 01:24:04 -0500 Subject: [PATCH 17/36] Update FreeBSD install instructions Added perl to list of packages to install (it's not installed by default). --- doc/install/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/install.rst b/doc/install/install.rst index 99e7d7e3dd..86e92c23c6 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -70,7 +70,7 @@ that ``bash`` and ``python`` are in your ``PATH``): .. console:: - sudo pkg_add -r bash cmake swig bison python + sudo pkg_add -r bash cmake swig bison python perl * Mac OS X: From 3850399fa202ab8ae605454fe52ed6b1f1364c70 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 14 Oct 2013 17:19:29 -0500 Subject: [PATCH 18/36] Fix typos and formatting in the conn protocol docs --- scripts/base/protocols/conn/contents.bro | 8 ++-- scripts/base/protocols/conn/main.bro | 54 ++++++++++++------------ scripts/base/protocols/conn/polling.bro | 9 ++-- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/scripts/base/protocols/conn/contents.bro b/scripts/base/protocols/conn/contents.bro index 2e6b547ab1..f030c4a0c2 100644 --- a/scripts/base/protocols/conn/contents.bro +++ b/scripts/base/protocols/conn/contents.bro @@ -16,12 +16,12 @@ module Conn; export { - ## The prefix given to files containing extracted connections as they are - ## opened on disk. + ## The prefix given to files containing extracted connections as they + ## are opened on disk. const extraction_prefix = "contents" &redef; - ## If this variable is set to ``T``, then all contents of all connections - ## will be extracted. + ## If this variable is set to ``T``, then all contents of all + ## connections will be extracted. const default_extract = F &redef; } diff --git a/scripts/base/protocols/conn/main.bro b/scripts/base/protocols/conn/main.bro index 05e6170dc8..a904f1b230 100644 --- a/scripts/base/protocols/conn/main.bro +++ b/scripts/base/protocols/conn/main.bro @@ -1,7 +1,7 @@ ##! This script manages the tracking/logging of general information regarding ##! TCP, UDP, and ICMP traffic. For UDP and ICMP, "connections" are to ##! be interpreted using flow semantics (sequence of packets from a source -##! host/post to a destination host/port). Further, ICMP "ports" are to +##! host/port to a destination host/port). Further, ICMP "ports" are to ##! be interpreted as the source port meaning the ICMP message type and ##! the destination port being the ICMP message code. @@ -23,7 +23,7 @@ export { id: conn_id &log; ## The transport layer protocol of the connection. proto: transport_proto &log; - ## An identification of an application protocol being sent over the + ## An identification of an application protocol being sent over ## the connection. service: string &log &optional; ## How long the connection lasted. For 3-way or 4-way connection @@ -31,9 +31,10 @@ export { duration: interval &log &optional; ## The number of payload bytes the originator sent. For TCP ## this is taken from sequence numbers and might be inaccurate - ## (e.g., due to large connections) + ## (e.g., due to large connections). orig_bytes: count &log &optional; - ## The number of payload bytes the responder sent. See ``orig_bytes``. + ## The number of payload bytes the responder sent. See + ## *orig_bytes*. resp_bytes: count &log &optional; ## ========== =============================================== @@ -55,20 +56,20 @@ export { ## ========== =============================================== conn_state: string &log &optional; - ## If the connection is originated locally, this value will be T. If - ## it was originated remotely it will be F. In the case that the - ## :bro:id:`Site::local_nets` variable is undefined, this field will - ## be left empty at all times. + ## If the connection is originated locally, this value will be T. + ## If it was originated remotely it will be F. In the case that + ## the :bro:id:`Site::local_nets` variable is undefined, this + ## field will be left empty at all times. local_orig: bool &log &optional; - ## Indicates the number of bytes missed in content gaps, which is - ## representative of packet loss. A value other than zero will - ## normally cause protocol analysis to fail but some analysis may - ## have been completed prior to the packet loss. + ## Indicates the number of bytes missed in content gaps, which + ## is representative of packet loss. A value other than zero + ## will normally cause protocol analysis to fail but some + ## analysis may have been completed prior to the packet loss. missed_bytes: count &log &default=0; - ## Records the state history of connections as a string of letters. - ## The meaning of those letters is: + ## Records the state history of connections as a string of + ## letters. The meaning of those letters is: ## ## ====== ==================================================== ## Letter Meaning @@ -83,24 +84,25 @@ export { ## i inconsistent packet (e.g. SYN+RST bits both set) ## ====== ==================================================== ## - ## If the event comes from the originator, the letter is in upper-case; if it comes - ## from the responder, it's in lower-case. Multiple packets of the same type will - ## only be noted once (e.g. we only record one "d" in each direction, regardless of - ## how many data packets were seen.) + ## If the event comes from the originator, the letter is in + ## upper-case; if it comes from the responder, it's in + ## lower-case. Multiple packets of the same type will only be + ## noted once (e.g. we only record one "d" in each direction, + ## regardless of how many data packets were seen.) history: string &log &optional; ## Number of packets that the originator sent. - ## Only set if :bro:id:`use_conn_size_analyzer` = T + ## Only set if :bro:id:`use_conn_size_analyzer` = T. orig_pkts: count &log &optional; - ## Number of IP level bytes that the originator sent (as seen on the wire, - ## taken from IP total_length header field). - ## Only set if :bro:id:`use_conn_size_analyzer` = T + ## Number of IP level bytes that the originator sent (as seen on + ## the wire, taken from the IP total_length header field). + ## Only set if :bro:id:`use_conn_size_analyzer` = T. orig_ip_bytes: count &log &optional; ## Number of packets that the responder sent. - ## Only set if :bro:id:`use_conn_size_analyzer` = T + ## Only set if :bro:id:`use_conn_size_analyzer` = T. resp_pkts: count &log &optional; - ## Number og IP level bytes that the responder sent (as seen on the wire, - ## taken from IP total_length header field). - ## Only set if :bro:id:`use_conn_size_analyzer` = T + ## Number of IP level bytes that the responder sent (as seen on + ## the wire, taken from the IP total_length header field). + ## Only set if :bro:id:`use_conn_size_analyzer` = T. resp_ip_bytes: count &log &optional; ## If this connection was over a tunnel, indicate the ## *uid* values for any encapsulating parent connections diff --git a/scripts/base/protocols/conn/polling.bro b/scripts/base/protocols/conn/polling.bro index 45c09c8465..51d5dc9117 100644 --- a/scripts/base/protocols/conn/polling.bro +++ b/scripts/base/protocols/conn/polling.bro @@ -11,10 +11,11 @@ export { ## c: The connection to watch. ## ## callback: A callback function that takes as arguments the monitored - ## *connection*, and counter *cnt* that increments each time the - ## callback is called. It returns an interval indicating how long - ## in the future to schedule an event which will call the - ## callback. A negative return interval causes polling to stop. + ## *connection*, and counter *cnt* that increments each time + ## the callback is called. It returns an interval indicating + ## how long in the future to schedule an event which will call + ## the callback. A negative return interval causes polling + ## to stop. ## ## cnt: The initial value of a counter which gets passed to *callback*. ## From b3f79265cb227a7b484fa48712e2c73e9ce965e9 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 14 Oct 2013 17:20:44 -0500 Subject: [PATCH 19/36] Adjust line numbers to match changes in conn/main.bro --- doc/scripting/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index 8e503fcc97..12085c5382 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -810,7 +810,7 @@ example of the ``record`` data type in the earlier sections, the ``conn.log``, is shown by the excerpt below. .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro - :lines: 10-12,16,17,19,21,23,25,28,31,35,37,56,62,68,90,93,97,100,104,108,109,114 + :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,92,95,99,102,106,110-111,116 Looking at the structure of the definition, a new collection of data types is being defined as a type called ``Info``. Since this type From e36fcf0190106785bbedf8774ffc1d967391354e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 15 Oct 2013 16:19:06 -0500 Subject: [PATCH 20/36] Fix typos and formatting in the dhcp protocol docs Also added a link to an RFC and a Bro script (and corrected the name of the script being referenced). --- scripts/base/protocols/dhcp/consts.bro | 2 +- scripts/base/protocols/dhcp/main.bro | 9 +++++---- scripts/base/protocols/dhcp/utils.bro | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/base/protocols/dhcp/consts.bro b/scripts/base/protocols/dhcp/consts.bro index 1b2a271563..5afdfc9415 100644 --- a/scripts/base/protocols/dhcp/consts.bro +++ b/scripts/base/protocols/dhcp/consts.bro @@ -5,7 +5,7 @@ module DHCP; export { - ## Types of DHCP messages. See RFC 1533. + ## Types of DHCP messages. See :rfc:`1533`. const message_types = { [1] = "DHCP_DISCOVER", [2] = "DHCP_OFFER", diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro index 144e5a53e7..e5be50b1ab 100644 --- a/scripts/base/protocols/dhcp/main.bro +++ b/scripts/base/protocols/dhcp/main.bro @@ -3,7 +3,8 @@ ##! noisy on most networks, and focuses on the end-result: assigned leases. ##! ##! If you'd like to track known DHCP devices and to log the hostname -##! supplied by the client, see policy/protocols/dhcp/known-devices.bro +##! supplied by the client, see +##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`. @load ./utils.bro @@ -18,7 +19,7 @@ export { ## associated connection is observed. ts: time &log; ## A unique identifier of the connection over which DHCP is - ## occuring. + ## occurring. uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; @@ -28,7 +29,7 @@ export { assigned_ip: addr &log &optional; ## IP address lease interval. lease_time: interval &log &optional; - ## A random number choosen by the client for this transaction. + ## A random number chosen by the client for this transaction. trans_id: count &log; }; @@ -37,7 +38,7 @@ export { global log_dhcp: event(rec: Info); } -# Add the dhcp info to the connection record +# Add the dhcp info to the connection record. redef record connection += { dhcp: Info &optional; }; diff --git a/scripts/base/protocols/dhcp/utils.bro b/scripts/base/protocols/dhcp/utils.bro index ef4f244d8a..e49bfe6af9 100644 --- a/scripts/base/protocols/dhcp/utils.bro +++ b/scripts/base/protocols/dhcp/utils.bro @@ -3,11 +3,11 @@ module DHCP; export { - ## Reverse the octets of an IPv4 IP. + ## Reverse the octets of an IPv4 address. ## - ## ip: An :bro:type:`addr` IPv4 address. + ## ip: An IPv4 address. ## - ## Returns: A reversed addr. + ## Returns: A reversed IPv4 address. global reverse_ip: function(ip: addr): addr; } From bc337f3d5a9b77e075120da8b00d1d011e61dbfd Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 15 Oct 2013 16:44:28 -0500 Subject: [PATCH 21/36] Fix typos and formatting in the dns protocol docs --- scripts/base/protocols/dns/consts.bro | 7 +++-- scripts/base/protocols/dns/main.bro | 43 ++++++++++++++------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/scripts/base/protocols/dns/consts.bro b/scripts/base/protocols/dns/consts.bro index fbf4aba008..13af6c3e81 100644 --- a/scripts/base/protocols/dns/consts.bro +++ b/scripts/base/protocols/dns/consts.bro @@ -8,7 +8,8 @@ export { const EDNS = 41; ##< An OPT RR TYPE value described by EDNS. const ANY = 255; ##< A QTYPE value describing a request for all records. - ## Mapping of DNS query type codes to human readable string representation. + ## Mapping of DNS query type codes to human readable string + ## representation. const query_types = { [1] = "A", [2] = "NS", [3] = "MD", [4] = "MF", [5] = "CNAME", [6] = "SOA", [7] = "MB", [8] = "MG", @@ -64,8 +65,8 @@ export { [32768] = "DNS_SEC_OK", # accepts DNS Sec RRs } &default="?"; - ## Possible values of the CLASS field in resource records or QCLASS field - ## in query messages. + ## Possible values of the CLASS field in resource records or QCLASS + ## field in query messages. const classes = { [1] = "C_INTERNET", [2] = "C_CSNET", diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index bf47519cd8..0d23029ad7 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -22,8 +22,8 @@ export { id: conn_id &log; ## The transport layer protocol of the connection. proto: transport_proto &log; - ## A 16 bit identifier assigned by the program that generated the - ## DNS query. Also used in responses to match up replies to + ## A 16-bit identifier assigned by the program that generated + ## the DNS query. Also used in responses to match up replies to ## outstanding queries. trans_id: count &log &optional; ## The domain name that is the subject of the DNS query. @@ -40,17 +40,17 @@ export { rcode: count &log &optional; ## A descriptive name for the response code value. rcode_name: string &log &optional; - ## The Authoritative Answer bit for response messages specifies that - ## the responding name server is an authority for the domain name - ## in the question section. + ## The Authoritative Answer bit for response messages specifies + ## that the responding name server is an authority for the + ## domain name in the question section. AA: bool &log &default=F; ## The Truncation bit specifies that the message was truncated. TC: bool &log &default=F; ## The Recursion Desired bit in a request message indicates that ## the client wants recursive service for this query. RD: bool &log &default=F; - ## The Recursion Available bit in a response message indicates that - ## the name server supports recursive queries. + ## The Recursion Available bit in a response message indicates + ## that the name server supports recursive queries. RA: bool &log &default=F; ## A reserved field that is currently supposed to be zero in all ## queries and responses. @@ -58,19 +58,19 @@ export { ## The set of resource descriptions in the query answer. answers: vector of string &log &optional; ## The caching intervals of the associated RRs described by the - ## ``answers`` field. + ## *answers* field. TTLs: vector of interval &log &optional; ## The DNS query was rejected by the server. rejected: bool &log &default=F; - ## This value indicates if this request/response pair is ready to be - ## logged. + ## This value indicates if this request/response pair is ready + ## to be logged. ready: bool &default=F; - ## The total number of resource records in a reply message's answer - ## section. + ## The total number of resource records in a reply message's + ## answer section. total_answers: count &optional; - ## The total number of resource records in a reply message's answer, - ## authority, and additional sections. + ## The total number of resource records in a reply message's + ## answer, authority, and additional sections. total_replies: count &optional; }; @@ -78,9 +78,10 @@ export { ## record as it is sent to the logging framework. global log_dns: event(rec: Info); - ## This is called by the specific dns_*_reply events with a "reply" which - ## may not represent the full data available from the resource record, but - ## it's generally considered a summarization of the response(s). + ## This is called by the specific dns_*_reply events with a "reply" + ## which may not represent the full data available from the resource + ## record, but it's generally considered a summarization of the + ## responses. ## ## c: The connection record for which to fill in DNS reply data. ## @@ -95,7 +96,7 @@ export { ## This can be used if additional initialization logic needs to happen ## when creating a new session value. ## - ## c: The connection involved in the new session + ## c: The connection involved in the new session. ## ## msg: The DNS message header information. ## @@ -109,9 +110,9 @@ export { ## query/response which haven't completed yet. pending: table[count] of Queue::Queue; - ## This is the list of DNS responses that have completed based on the - ## number of responses declared and the number received. The contents - ## of the set are transaction IDs. + ## This is the list of DNS responses that have completed based + ## on the number of responses declared and the number received. + ## The contents of the set are transaction IDs. finished_answers: set[count]; }; } From 98181e829b10fb5f2264f4298ad33f33fa5f2794 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 15 Oct 2013 17:00:01 -0500 Subject: [PATCH 22/36] Fix typos and formatting in the ftp protocol docs --- scripts/base/protocols/ftp/gridftp.bro | 12 ++++++------ scripts/base/protocols/ftp/info.bro | 10 ++++++---- scripts/base/protocols/ftp/main.bro | 2 +- scripts/base/protocols/ftp/utils.bro | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/scripts/base/protocols/ftp/gridftp.bro b/scripts/base/protocols/ftp/gridftp.bro index 73bd656544..0e09db006d 100644 --- a/scripts/base/protocols/ftp/gridftp.bro +++ b/scripts/base/protocols/ftp/gridftp.bro @@ -4,20 +4,20 @@ ##! that successfully negotiate the GSSAPI method of an AUTH request ##! and for which the exchange involved an encoded TLS/SSL handshake, ##! indicating the GSI mechanism for GSSAPI was used. This analysis -##! is all supported internally, this script simple adds the "gridftp" +##! is all supported internally, this script simply adds the "gridftp" ##! label to the *service* field of the control channel's ##! :bro:type:`connection` record. ##! ##! GridFTP data channels are identified by a heuristic that relies on ##! the fact that default settings for GridFTP clients typically -##! mutally authenticate the data channel with TLS/SSL and negotiate a +##! mutually authenticate the data channel with TLS/SSL and negotiate a ##! NULL bulk cipher (no encryption). Connections with those ##! attributes are then polled for two minutes with decreasing frequency ##! to check if the transfer sizes are large enough to indicate a -##! GridFTP data channel that would be undesireable to analyze further +##! GridFTP data channel that would be undesirable to analyze further ##! (e.g. stop TCP reassembly). A side effect is that true connection ##! sizes are not logged, but at the benefit of saving CPU cycles that -##! otherwise go to analyzing the large (and likely benign) connections. +##! would otherwise go to analyzing the large (and likely benign) connections. @load ./info @load ./main @@ -59,8 +59,8 @@ export { ## been exceeded. This is called in a :bro:see:`ssl_established` event ## handler and by default looks for both a client and server certificate ## and for a NULL bulk cipher. One way in which this function could be - ## redefined is to make it also consider client/server certificate issuer - ## subjects. + ## redefined is to make it also consider client/server certificate + ## issuer subjects. ## ## c: The connection which may possibly be a GridFTP data channel. ## diff --git a/scripts/base/protocols/ftp/info.bro b/scripts/base/protocols/ftp/info.bro index f6fceb071e..a9db7ba6a0 100644 --- a/scripts/base/protocols/ftp/info.bro +++ b/scripts/base/protocols/ftp/info.bro @@ -18,7 +18,8 @@ export { orig_h: addr &log; ## The host that will be accepting the data connection. resp_h: addr &log; - ## The port at which the acceptor is listening for the data connection. + ## The port at which the acceptor is listening for the data + ## connection. resp_p: port &log; }; @@ -38,7 +39,8 @@ export { ## Argument for the command if one is given. arg: string &log &optional; - ## Libmagic "sniffed" file type if the command indicates a file transfer. + ## Libmagic "sniffed" file type if the command indicates a file + ## transfer. mime_type: string &log &optional; ## Size of the file if the command indicates a file transfer. file_size: count &log &optional; @@ -59,8 +61,8 @@ export { ## Command that is currently waiting for a response. cmdarg: CmdArg &optional; - ## Queue for commands that have been sent but not yet responded to - ## are tracked here. + ## Queue for commands that have been sent but not yet responded + ## to are tracked here. pending_commands: PendingCmds; ## Indicates if the session is in active or passive mode. diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 254dca7d42..9bc1f0d0f1 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -26,7 +26,7 @@ export { const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef; ## This record is to hold a parsed FTP reply code. For example, for the - ## 201 status code, the digits would be parsed as: x->2, y->0, z=>1. + ## 201 status code, the digits would be parsed as: x->2, y->0, z->1. type ReplyCode: record { x: count; y: count; diff --git a/scripts/base/protocols/ftp/utils.bro b/scripts/base/protocols/ftp/utils.bro index 313280b904..74aeaa1e03 100644 --- a/scripts/base/protocols/ftp/utils.bro +++ b/scripts/base/protocols/ftp/utils.bro @@ -11,14 +11,14 @@ export { ## ## rec: An :bro:type:`FTP::Info` record. ## - ## Returns: A URL, not prefixed by "ftp://". + ## Returns: A URL, not prefixed by ``"ftp://"``. global build_url: function(rec: Info): string; ## Creates a URL from an :bro:type:`FTP::Info` record. ## ## rec: An :bro:type:`FTP::Info` record. ## - ## Returns: A URL prefixed with "ftp://". + ## Returns: A URL prefixed with ``"ftp://"``. global build_url_ftp: function(rec: Info): string; ## Create an extremely shortened representation of a log line. From fe60404f0f44f8ac6e62ba1fe69a21a0ae59b7db Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 16 Oct 2013 13:13:53 -0500 Subject: [PATCH 23/36] Fix typos and formatting in the http protocol docs Also adjusted line numbers in scripting doc due to changes in http/main.bro --- doc/scripting/index.rst | 2 +- scripts/base/protocols/http/entities.bro | 6 ++++-- scripts/base/protocols/http/main.bro | 10 ++++++---- scripts/base/protocols/http/utils.bro | 16 ++++++++-------- ...de-scripts_base_protocols_http_main_bro.btest | 3 ++- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index 12085c5382..a24032f524 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -351,7 +351,7 @@ decrypted from HTTP streams is stored in excerpt from :doc:`/scripts/base/protocols/http/main` below. .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro - :lines: 8-10,19,20,118 + :lines: 8-10,19-21,120 Because the constant was declared with the ``&redef`` attribute, if we needed to turn this option on globally, we could do so by adding the diff --git a/scripts/base/protocols/http/entities.bro b/scripts/base/protocols/http/entities.bro index e9376a0c0c..0239e0fa85 100644 --- a/scripts/base/protocols/http/entities.bro +++ b/scripts/base/protocols/http/entities.bro @@ -28,9 +28,11 @@ export { ## The current entity. current_entity: Entity &optional; - ## Current number of MIME entities in the HTTP request message body. + ## Current number of MIME entities in the HTTP request message + ## body. orig_mime_depth: count &default=0; - ## Current number of MIME entities in the HTTP response message body. + ## Current number of MIME entities in the HTTP response message + ## body. resp_mime_depth: count &default=0; }; } diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index e103bc6b41..a164fcd6a6 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -16,7 +16,8 @@ export { EMPTY }; - ## This setting changes if passwords used in Basic-Auth are captured or not. + ## This setting changes if passwords used in Basic-Auth are captured or + ## not. const default_capture_password = F &redef; type Info: record { @@ -36,8 +37,8 @@ export { ## URI used in the request. uri: string &log &optional; ## Value of the "referer" header. The comment is deliberately - ## misspelled like the standard declares, but the name used here is - ## "referrer" spelled correctly. + ## misspelled like the standard declares, but the name used here + ## is "referrer" spelled correctly. referrer: string &log &optional; ## Value of the User-Agent header from the client. user_agent: string &log &optional; @@ -55,7 +56,8 @@ export { info_code: count &log &optional; ## Last seen 1xx informational reply message returned by the server. info_msg: string &log &optional; - ## Filename given in the Content-Disposition header sent by the server. + ## Filename given in the Content-Disposition header sent by the + ## server. filename: string &log &optional; ## A set of indicators of various attributes discovered and ## related to a particular request/response pair. diff --git a/scripts/base/protocols/http/utils.bro b/scripts/base/protocols/http/utils.bro index fe8c076780..3c75ae254b 100644 --- a/scripts/base/protocols/http/utils.bro +++ b/scripts/base/protocols/http/utils.bro @@ -6,8 +6,8 @@ module HTTP; export { - ## Given a string containing a series of key-value pairs separated by "=", - ## this function can be used to parse out all of the key names. + ## Given a string containing a series of key-value pairs separated + ## by "=", this function can be used to parse out all of the key names. ## ## data: The raw data, such as a URL or cookie value. ## @@ -17,20 +17,20 @@ export { ## Returns: A vector of strings containing the keys. global extract_keys: function(data: string, kv_splitter: pattern): string_vec; - ## Creates a URL from an :bro:type:`HTTP::Info` record. This should handle - ## edge cases such as proxied requests appropriately. + ## Creates a URL from an :bro:type:`HTTP::Info` record. This should + ## handle edge cases such as proxied requests appropriately. ## ## rec: An :bro:type:`HTTP::Info` record. ## - ## Returns: A URL, not prefixed by "http://". + ## Returns: A URL, not prefixed by ``"http://"``. global build_url: function(rec: Info): string; - ## Creates a URL from an :bro:type:`HTTP::Info` record. This should handle - ## edge cases such as proxied requests appropriately. + ## Creates a URL from an :bro:type:`HTTP::Info` record. This should + ## handle edge cases such as proxied requests appropriately. ## ## rec: An :bro:type:`HTTP::Info` record. ## - ## Returns: A URL prefixed with "http://". + ## Returns: A URL prefixed with ``"http://"``. global build_url_http: function(rec: Info): string; ## Create an extremely shortened representation of a log line. diff --git a/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest b/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest index c2e558bac8..e3f7a39429 100644 --- a/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest @@ -5,6 +5,7 @@ main.bro module HTTP; export { - ## This setting changes if passwords used in Basic-Auth are captured or not. + ## This setting changes if passwords used in Basic-Auth are captured or + ## not. const default_capture_password = F &redef; } From 876485033774d869ef539f47d031ebeb4b8247fa Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 16 Oct 2013 17:36:27 -0500 Subject: [PATCH 24/36] Fix typos and formatting in the ssl protocol docs Also changed one sentence (about setting the Mozilla root CA list) to make it more clear that the user doesn't need to do anything. --- scripts/base/protocols/ssl/consts.bro | 6 +++--- scripts/base/protocols/ssl/main.bro | 14 ++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 732e06b21f..55289a7419 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -23,7 +23,7 @@ export { } &default=function(i: count):string { return fmt("unknown-%d", i); }; ## Mapping between numeric codes and human readable strings for alert - ## descriptions.. + ## descriptions. const alert_descriptions: table[count] of string = { [0] = "close_notify", [10] = "unexpected_message", @@ -453,8 +453,8 @@ export { const TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF; ## This is a table of all known cipher specs. It can be used for - ## detecting unknown ciphers and for converting the cipher spec constants - ## into a human readable format. + ## detecting unknown ciphers and for converting the cipher spec + ## constants into a human readable format. const cipher_desc: table[count] of string = { [SSLv20_CK_RC4_128_EXPORT40_WITH_MD5] = "SSLv20_CK_RC4_128_EXPORT40_WITH_MD5", diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 898b6e9bbb..9eab0ca970 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -26,7 +26,8 @@ export { session_id: string &log &optional; ## Subject of the X.509 certificate offered by the server. subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the server. + ## Subject of the signer of the X.509 certificate offered by the + ## server. issuer_subject: string &log &optional; ## NotValidBefore field value from the server certificate. not_valid_before: time &log &optional; @@ -37,7 +38,8 @@ export { ## Subject of the X.509 certificate offered by the client. client_subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the client. + ## Subject of the signer of the X.509 certificate offered by the + ## client. client_issuer_subject: string &log &optional; ## Full binary server certificate stored in DER format. @@ -58,8 +60,8 @@ export { analyzer_id: count &optional; }; - ## The default root CA bundle. By loading the - ## mozilla-ca-list.bro script it will be set to Mozilla's root CA list. + ## The default root CA bundle. By default, the mozilla-ca-list.bro + ## script sets this to Mozilla's root CA list. const root_certs: table[string] of string = {} &redef; ## If true, detach the SSL analyzer from the connection to prevent @@ -67,8 +69,8 @@ export { ## (especially with large file transfers). const disable_analyzer_after_detection = T &redef; - ## Delays an SSL record for a specific token: the record will not be logged - ## as longs the token exists or until 15 seconds elapses. + ## Delays an SSL record for a specific token: the record will not be + ## logged as long as the token exists or until 15 seconds elapses. global delay_log: function(info: Info, token: string); ## Undelays an SSL record for a previously inserted token, allowing the From c224fbe7f844185703ae4512fa8018e43557c4df Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 16 Oct 2013 18:07:34 -0500 Subject: [PATCH 25/36] Fix typos and formatting in the other protocol docs --- scripts/base/protocols/dnp3/main.bro | 2 +- scripts/base/protocols/irc/main.bro | 4 ++-- scripts/base/protocols/modbus/main.bro | 6 +++--- scripts/base/protocols/smtp/main.bro | 15 ++++++++------- scripts/base/protocols/socks/main.bro | 8 +++++--- scripts/base/protocols/ssh/main.bro | 8 ++++---- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/scripts/base/protocols/dnp3/main.bro b/scripts/base/protocols/dnp3/main.bro index f9c35ed8c0..3e5eede462 100644 --- a/scripts/base/protocols/dnp3/main.bro +++ b/scripts/base/protocols/dnp3/main.bro @@ -10,7 +10,7 @@ export { type Info: record { ## Time of the request. ts: time &log; - ## Unique identifier for the connnection. + ## Unique identifier for the connection. uid: string &log; ## Identifier for the connection. id: conn_id &log; diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro index a57fc95448..d861e88ae9 100644 --- a/scripts/base/protocols/irc/main.bro +++ b/scripts/base/protocols/irc/main.bro @@ -15,9 +15,9 @@ export { uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; - ## Nick name given for the connection. + ## Nickname given for the connection. nick: string &log &optional; - ## User name given for the connection. + ## Username given for the connection. user: string &log &optional; ## Command given by the client. diff --git a/scripts/base/protocols/modbus/main.bro b/scripts/base/protocols/modbus/main.bro index d484e7582b..707c2e47a7 100644 --- a/scripts/base/protocols/modbus/main.bro +++ b/scripts/base/protocols/modbus/main.bro @@ -10,7 +10,7 @@ export { type Info: record { ## Time of the request. ts: time &log; - ## Unique identifier for the connnection. + ## Unique identifier for the connection. uid: string &log; ## Identifier for the connection. id: conn_id &log; @@ -20,8 +20,8 @@ export { exception: string &log &optional; }; - ## Event that can be handled to access the Modbus record as it is sent on - ## to the logging framework. + ## Event that can be handled to access the Modbus record as it is sent + ## on to the logging framework. global log_modbus: event(rec: Info); } diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index f54fcdd4c8..fb3095c121 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -14,8 +14,8 @@ export { uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; - ## A count to represent the depth of this message transaction in a single - ## connection where multiple messages were transferred. + ## A count to represent the depth of this message transaction in + ## a single connection where multiple messages were transferred. trans_depth: count &log; ## Contents of the Helo header. helo: string &log &optional; @@ -37,7 +37,7 @@ export { in_reply_to: string &log &optional; ## Contents of the Subject header. subject: string &log &optional; - ## Contents of the X-Origininating-IP header. + ## Contents of the X-Originating-IP header. x_originating_ip: addr &log &optional; ## Contents of the first Received header. first_received: string &log &optional; @@ -50,7 +50,8 @@ export { ## Value of the User-Agent header from the client. user_agent: string &log &optional; - ## Indicates if the "Received: from" headers should still be processed. + ## Indicates if the "Received: from" headers should still be + ## processed. process_received_from: bool &default=T; ## Indicates if client activity has been seen, but not yet logged. has_client_activity: bool &default=F; @@ -58,9 +59,9 @@ export { type State: record { helo: string &optional; - ## Count the number of individual messages transmitted during this - ## SMTP session. Note, this is not the number of recipients, but the - ## number of message bodies transferred. + ## Count the number of individual messages transmitted during + ## this SMTP session. Note, this is not the number of + ## recipients, but the number of message bodies transferred. messages_transferred: count &default=0; pending_messages: set[Info] &optional; diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index f697b355c1..713161d442 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -9,17 +9,19 @@ export { type Info: record { ## Time when the proxy connection was first detected. ts: time &log; - ## Unique ID for the tunnel - may correspond to connection uid or be non-existent. + ## Unique ID for the tunnel - may correspond to connection uid + ## or be non-existent. uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; ## Protocol version of SOCKS. version: count &log; - ## Username for the proxy if extracted from the network.. + ## Username for the proxy if extracted from the network. user: string &log &optional; ## Server status for the attempt at using the proxy. status: string &log &optional; - ## Client requested SOCKS address. Could be an address, a name or both. + ## Client requested SOCKS address. Could be an address, a name + ## or both. request: SOCKS::Address &log &optional; ## Client requested port. request_p: port &log &optional; diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index 53b61f00d8..5b452d432c 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -25,8 +25,8 @@ export { uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; - ## Indicates if the login was heuristically guessed to be "success", - ## "failure", or "undetermined". + ## Indicates if the login was heuristically guessed to be + ## "success", "failure", or "undetermined". status: string &log &default="undetermined"; ## Direction of the connection. If the client was a local host ## logging into an external host, this would be OUTBOUND. INBOUND @@ -39,8 +39,8 @@ export { server: string &log &optional; ## Amount of data returned from the server. This is currently ## the only measure of the success heuristic and it is logged to - ## assist analysts looking at the logs to make their own determination - ## about the success on a case-by-case basis. + ## assist analysts looking at the logs to make their own + ## determination about the success on a case-by-case basis. resp_size: count &log &default=0; ## Indicate if the SSH session is done being watched. From 92dc8e5880c3b1cae715f19931351e05480d720d Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 16 Oct 2013 20:41:56 -0500 Subject: [PATCH 26/36] Fix typos and formatting in the base/utils docs --- scripts/base/utils/addrs.bro | 10 ++++-- scripts/base/utils/conn-ids.bro | 8 ++--- scripts/base/utils/dir.bro | 4 +-- scripts/base/utils/directions-and-hosts.bro | 16 +++++++--- scripts/base/utils/exec.bro | 20 ++++++------ scripts/base/utils/paths.bro | 26 +++++++++------ scripts/base/utils/patterns.bro | 18 +++++++---- scripts/base/utils/queue.bro | 8 ++--- scripts/base/utils/site.bro | 2 +- scripts/base/utils/strings.bro | 35 +++++++++++++-------- scripts/base/utils/thresholds.bro | 20 +++++++----- scripts/base/utils/time.bro | 5 ++- scripts/base/utils/urls.bro | 4 +-- 13 files changed, 107 insertions(+), 69 deletions(-) diff --git a/scripts/base/utils/addrs.bro b/scripts/base/utils/addrs.bro index 08efd5281a..e2031e3efa 100644 --- a/scripts/base/utils/addrs.bro +++ b/scripts/base/utils/addrs.bro @@ -28,7 +28,9 @@ const ip_addr_regex = /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec ## Checks if all elements of a string array are a valid octet value. +## ## octets: an array of strings to check for valid octet values. +## ## Returns: T if every element is between 0 and 255, inclusive, else F. function has_valid_octets(octets: string_array): bool { @@ -43,7 +45,9 @@ function has_valid_octets(octets: string_array): bool } ## Checks if a string appears to be a valid IPv4 or IPv6 address. +## ## ip_str: the string to check for valid IP formatting. +## ## Returns: T if the string is a valid IPv4 or IPv6 address format. function is_valid_ip(ip_str: string): bool { @@ -84,8 +88,10 @@ function is_valid_ip(ip_str: string): bool } ## Extracts all IP (v4 or v6) address strings from a given string. +## ## input: a string that may contain an IP address anywhere within it. -## Returns: an array containing all valid IP address strings found in input. +## +## Returns: an array containing all valid IP address strings found in *input*. function find_ip_addresses(input: string): string_array { local parts = split_all(input, ip_addr_regex); @@ -105,7 +111,7 @@ function find_ip_addresses(input: string): string_array ## ## a: the address to make suitable for URI inclusion. ## -## Returns: the string representation of *a* suitable for URI inclusion. +## Returns: the string representation of the address suitable for URI inclusion. function addr_to_uri(a: addr): string { if ( is_v4_addr(a) ) diff --git a/scripts/base/utils/conn-ids.bro b/scripts/base/utils/conn-ids.bro index 3a10263a95..6601b665e5 100644 --- a/scripts/base/utils/conn-ids.bro +++ b/scripts/base/utils/conn-ids.bro @@ -4,17 +4,17 @@ module GLOBAL; export { ## Takes a conn_id record and returns a string representation with the - ## the general data flow appearing to be from the connection originator + ## general data flow appearing to be from the connection originator ## on the left to the responder on the right. global id_string: function(id: conn_id): string; ## Takes a conn_id record and returns a string representation with the - ## the general data flow appearing to be from the connection responder + ## general data flow appearing to be from the connection responder ## on the right to the originator on the left. global reverse_id_string: function(id: conn_id): string; - ## Calls :bro:id:`id_string` or :bro:id:`reverse_id_string` if the second - ## argument is T or F, respectively. + ## Calls :bro:id:`id_string` or :bro:id:`reverse_id_string` if the + ## second argument is T or F, respectively. global directed_id_string: function(id: conn_id, is_orig: bool): string; } diff --git a/scripts/base/utils/dir.bro b/scripts/base/utils/dir.bro index 1ade4a47f7..e64af43b15 100644 --- a/scripts/base/utils/dir.bro +++ b/scripts/base/utils/dir.bro @@ -11,8 +11,8 @@ export { ## Register a directory to monitor with a callback that is called ## every time a previously unseen file is seen. If a file is deleted - ## and seen to be gone, the file is available for being seen again in - ## the future. + ## and seen to be gone, then the file is available for being seen again + ## in the future. ## ## dir: The directory to monitor for files. ## diff --git a/scripts/base/utils/directions-and-hosts.bro b/scripts/base/utils/directions-and-hosts.bro index a88c4827a6..dab35411a6 100644 --- a/scripts/base/utils/directions-and-hosts.bro +++ b/scripts/base/utils/directions-and-hosts.bro @@ -1,8 +1,8 @@ @load ./site type Direction: enum { - ## The connection originator is not within the locally-monitored network, - ## but the other endpoint is. + ## The connection originator is not within the locally-monitored + ## network, but the other endpoint is. INBOUND, ## The connection originator is within the locally-monitored network, ## but the other endpoint is not. @@ -16,8 +16,11 @@ type Direction: enum { ## Checks whether a given connection is of a given direction with respect ## to the locally-monitored network. +## ## id: a connection record containing the originator/responder hosts. -## d: a direction with respect to the locally-monitored network +## +## d: a direction with respect to the locally-monitored network. +## ## Returns: T if the two connection endpoints match the given direction, else F. function id_matches_direction(id: conn_id, d: Direction): bool { @@ -46,8 +49,11 @@ type Host: enum { }; ## Checks whether a given host (IP address) matches a given host type. -## ip: address of a host -## h: a host type +## +## ip: address of a host. +## +## h: a host type. +## ## Returns: T if the given host matches the given type, else F. function addr_matches_host(ip: addr, h: Host): bool { diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro index d505b424c7..12f5a0087b 100644 --- a/scripts/base/utils/exec.bro +++ b/scripts/base/utils/exec.bro @@ -6,16 +6,16 @@ module Exec; export { type Command: record { - ## The command line to execute. Use care to avoid injection attacks. - ## I.e. if the command uses untrusted/variable data, sanitize - ## it with str_shell_escape(). + ## The command line to execute. Use care to avoid injection + ## attacks (i.e., if the command uses untrusted/variable data, + ## sanitize it with :bro:see:`str_shell_escape`). cmd: string; - ## Provide standard in to the program as a string. + ## Provide standard input to the program as a string. stdin: string &default=""; - ## If additional files are required to be read in as part of the output - ## of the command they can be defined here. + ## If additional files are required to be read in as part of the + ## output of the command they can be defined here. read_files: set[string] &optional; - # The unique id for tracking executors. + ## The unique id for tracking executors. uid: string &default=unique_id(""); }; @@ -24,7 +24,7 @@ export { exit_code: count &default=0; ## True if the command was terminated with a signal. signal_exit: bool &default=F; - ## Each line of standard out. + ## Each line of standard output. stdout: vector of string &optional; ## Each line of standard error. stderr: vector of string &optional; @@ -39,11 +39,11 @@ export { ## ## cmd: The command to run. Use care to avoid injection attacks! ## - ## returns: A record representing the full results from the + ## Returns: A record representing the full results from the ## external program execution. global run: function(cmd: Command): Result; - ## The system directory for temp files. + ## The system directory for temporary files. const tmp_dir = "/tmp" &redef; } diff --git a/scripts/base/utils/paths.bro b/scripts/base/utils/paths.bro index f8ad384ea7..ce083eb6d0 100644 --- a/scripts/base/utils/paths.bro +++ b/scripts/base/utils/paths.bro @@ -4,9 +4,12 @@ const absolute_path_pat = /(\/|[A-Za-z]:[\\\/]).*/; ## Given an arbitrary string, extracts a single, absolute path (directory ## with filename). -## TODO: Make this work on Window's style directories. -## input: a string that may contain an absolute path -## Returns: the first absolute path found in input string, else an empty string +## +## .. todo:: Make this work on Window's style directories. +## +## input: a string that may contain an absolute path. +## +## Returns: the first absolute path found in input string, else an empty string. function extract_path(input: string): string { const dir_pattern = /(\/|[A-Za-z]:[\\\/])([^\"\ ]|(\\\ ))*/; @@ -20,8 +23,10 @@ function extract_path(input: string): string ## Compresses a given path by removing '..'s and the parent directory it ## references and also removing dual '/'s and extraneous '/./'s. -## dir: a path string, either relative or absolute -## Returns: a compressed version of the input path +## +## dir: a path string, either relative or absolute. +## +## Returns: a compressed version of the input path. function compress_path(dir: string): string { const cdup_sep = /((\/)*([^\/]|\\\/)+)?((\/)+\.\.(\/)*)/; @@ -56,10 +61,13 @@ function compress_path(dir: string): string } ## Constructs a path to a file given a directory and a file name. -## dir: the directory in which the file lives -## file_name: the name of the file +## +## dir: the directory in which the file lives. +## +## file_name: the name of the file. +## ## Returns: the concatenation of the directory path and file name, or just -## the file name if it's already an absolute path +## the file name if it's already an absolute path. function build_path(dir: string, file_name: string): string { return (file_name == absolute_path_pat) ? @@ -67,7 +75,7 @@ function build_path(dir: string, file_name: string): string } ## Returns a compressed path to a file given a directory and file name. -## See :bro:id`build_path` and :bro:id:`compress_path`. +## See :bro:id:`build_path` and :bro:id:`compress_path`. function build_path_compressed(dir: string, file_name: string): string { return compress_path(build_path(dir, file_name)); diff --git a/scripts/base/utils/patterns.bro b/scripts/base/utils/patterns.bro index f2a242b271..957e19a14b 100644 --- a/scripts/base/utils/patterns.bro +++ b/scripts/base/utils/patterns.bro @@ -5,11 +5,14 @@ module GLOBAL; ## Given a pattern as a string with two tildes (~~) contained in it, it will ## return a pattern with string set's elements OR'd together where the ## double-tilde was given (this function only works at or before init time). -## ss: a set of strings to OR together +## +## ss: a set of strings to OR together. +## ## pat: the pattern containing a "~~" in it. If a literal backslash is ## included, it needs to be escaped with another backslash due to Bro's ## string parsing reducing it to a single backslash upon rendering. -## Returns: the input pattern with "~~" replaced by OR'd elements of input set +## +## Returns: the input pattern with "~~" replaced by OR'd elements of input set. function set_to_regex(ss: set[string], pat: string): pattern { local i: count = 0; @@ -38,10 +41,13 @@ type PatternMatchResult: record { ## For example: ``match_pattern("foobar", /o*[a-k]/)`` returns ## ``[matched=T, str=f, off=1]``, because the *first* match is for ## zero o's followed by an [a-k], but ``match_pattern("foobar", /o+[a-k]/)`` -## returns ``[matched=T, str=oob, off=2]`` -## s: a string to match against -## p: a pattern to match -## Returns: a record indicating the match status +## returns ``[matched=T, str=oob, off=2]``. +## +## s: a string to match against. +## +## p: a pattern to match. +## +## Returns: a record indicating the match status. function match_pattern(s: string, p: pattern): PatternMatchResult { local a = split_n(s, p, T, 1); diff --git a/scripts/base/utils/queue.bro b/scripts/base/utils/queue.bro index 64202c54bc..5bad2382c4 100644 --- a/scripts/base/utils/queue.bro +++ b/scripts/base/utils/queue.bro @@ -42,8 +42,8 @@ export { ## Returns: The value at the end of the queue. global peek: function(q: Queue): any; - ## Merge two queue's together. If any settings are applied - ## to the queues, the settings from q1 are used for the new + ## Merge two queues together. If any settings are applied + ## to the queues, the settings from *q1* are used for the new ## merged queue. ## ## q1: The first queue. Settings are taken from here. @@ -64,8 +64,8 @@ export { ## ## q: The queue. ## - ## ret: A vector containing the - ## current contents of q as the type of ret. + ## ret: A vector containing the current contents of the queue + ## as the type of ret. global get_vector: function(q: Queue, ret: vector of any); } diff --git a/scripts/base/utils/site.bro b/scripts/base/utils/site.bro index 55ee0e5ed1..9edefe6a8e 100644 --- a/scripts/base/utils/site.bro +++ b/scripts/base/utils/site.bro @@ -67,7 +67,7 @@ export { ## The function inspects :bro:id:`Site::neighbor_zones`. global is_neighbor_name: function(name: string): bool; - ## Function that returns a common separated list of email addresses + ## Function that returns a comma-separated list of email addresses ## that are considered administrators for the IP address provided as ## an argument. ## The function inspects :bro:id:`Site::local_admins`. diff --git a/scripts/base/utils/strings.bro b/scripts/base/utils/strings.bro index 560ba6d160..8a3f03faaa 100644 --- a/scripts/base/utils/strings.bro +++ b/scripts/base/utils/strings.bro @@ -1,6 +1,6 @@ ##! Functions to assist with small string analysis and manipulation that can -##! be implemented as Bro functions and don't need to be implemented as built -##! in functions. +##! be implemented as Bro functions and don't need to be implemented as built-in +##! functions. ## Returns true if the given string is at least 25% composed of 8-bit ## characters. @@ -9,10 +9,13 @@ function is_string_binary(s: string): bool return |gsub(s, /[\x00-\x7f]/, "")| * 100 / |s| >= 25; } -## Joins a set of string together, with elements delimited by a constant string. -## ss: a set of strings to join -## j: the string used to join set elements -## Returns: a string composed of the all elements of the set, delimited by the +## Join a set of strings together, with elements delimited by a constant string. +## +## ss: a set of strings to join. +## +## j: the string used to join set elements. +## +## Returns: a string composed of all elements of the set, delimited by the ## joining string. function join_string_set(ss: set[string], j: string): string { @@ -30,9 +33,12 @@ function join_string_set(ss: set[string], j: string): string } ## Given a string, returns an escaped version. -## s: a string to escape -## chars: a string containing all the characters that need to be escaped -## Returns: a string with all occurrences of any character in ``chars`` escaped +## +## s: a string to escape. +## +## chars: a string containing all the characters that need to be escaped. +## +## Returns: a string with all occurrences of any character in *chars* escaped ## using ``\``, and any literal ``\`` characters likewise escaped. function string_escape(s: string, chars: string): string { @@ -42,10 +48,13 @@ function string_escape(s: string, chars: string): string return s; } -## Cut a number of character from the end of the given string. -## s: a string to trim -## tail_len: the number of characters to remove from end of string -## Returns: the string in ``s`` with ``tail_len`` characters removed from end +## Cut a number of characters from the end of the given string. +## +## s: a string to trim. +## +## tail_len: the number of characters to remove from the end of the string. +## +## Returns: the given string with *tail_len* characters removed from the end. function cut_tail(s: string, tail_len: count): string { if ( tail_len > |s| ) diff --git a/scripts/base/utils/thresholds.bro b/scripts/base/utils/thresholds.bro index 14bcc3b50b..31d1d3e84f 100644 --- a/scripts/base/utils/thresholds.bro +++ b/scripts/base/utils/thresholds.bro @@ -11,8 +11,9 @@ export { type TrackCount: record { ## The counter for the number of times something has happened. n: count &default=0; - ## The index of the vector where the counter currently is. This is - ## used to track which threshold is currently being watched for. + ## The index of the vector where the counter currently is. This + ## is used to track which threshold is currently being watched + ## for. index: count &default=0; }; @@ -24,15 +25,18 @@ export { ## This will check if a :bro:type:`TrackCount` variable has crossed any ## thresholds in a given set. - ## v: a vector holding counts that represent thresholds + ## + ## v: a vector holding counts that represent thresholds. + ## ## tracker: the record being used to track event counter and currently - ## monitored threshold value - ## Returns: T if a threshold has been crossed, else F + ## monitored threshold value. + ## + ## Returns: T if a threshold has been crossed, else F. global check_threshold: function(v: vector of count, tracker: TrackCount): bool; - ## This will use the :bro:id:`default_notice_thresholds` variable to check - ## a :bro:type:`TrackCount` variable to see if it has crossed another - ## threshold. + ## This will use the :bro:id:`default_notice_thresholds` variable to + ## check a :bro:type:`TrackCount` variable to see if it has crossed + ## another threshold. global default_check_threshold: function(tracker: TrackCount): bool; } diff --git a/scripts/base/utils/time.bro b/scripts/base/utils/time.bro index 2e3788e681..c173c82878 100644 --- a/scripts/base/utils/time.bro +++ b/scripts/base/utils/time.bro @@ -1,7 +1,6 @@ -## Given an interval, returns a string of the form 3m34s to -## give a minimalized human readable string for the minutes -## and seconds represented by the interval. +## Given an interval, returns a string representing the minutes and seconds +## in the interval (for example, "3m34s"). function duration_to_mins_secs(dur: interval): string { local dur_count = double_to_count(interval_to_double(dur)); diff --git a/scripts/base/utils/urls.bro b/scripts/base/utils/urls.bro index d704e72013..8ef9ed7e2d 100644 --- a/scripts/base/utils/urls.bro +++ b/scripts/base/utils/urls.bro @@ -1,4 +1,4 @@ -## Functions for URL handling. +##! Functions for URL handling. ## A regular expression for matching and extracting URLs. const url_regex = /^([a-zA-Z\-]{3,5})(:\/\/[^\/?#"'\r\n><]*)([^?#"'\r\n><]*)([^[:blank:]\r\n"'><]*|\??[^"'\r\n><]*)/ &redef; @@ -22,4 +22,4 @@ function find_all_urls_without_scheme(s: string): string_set } return return_urls; - } \ No newline at end of file + } From 8c3adc9df62c1e3cf2746bf338514e683a6c2616 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 17 Oct 2013 01:04:20 -0500 Subject: [PATCH 27/36] Fix typos and formatting in the BiFs docs --- src/bro.bif | 21 +++---- src/probabilistic/bloom-filter.bif | 23 ++++---- src/probabilistic/cardinality-counter.bif | 21 +++---- src/probabilistic/top-k.bif | 68 ++++++++++++----------- src/strings.bif | 2 + 5 files changed, 73 insertions(+), 62 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index f699c1e162..24dff3c77c 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1,10 +1,11 @@ ##! A collection of built-in functions that implement a variety of things ##! such as general programming algorithms, string processing, math functions, -##! introspection, type conversion, file/directory manipulation, packet filtering, -##! inter-process communication and controlling protocol analyzer behavior. +##! introspection, type conversion, file/directory manipulation, packet +##! filtering, interprocess communication and controlling protocol analyzer +##! behavior. ##! -##! You'll find most of Bro's built-in functions that aren't protocol-specific in -##! this file. +##! You'll find most of Bro's built-in functions that aren't protocol-specific +##! in this file. %%{ // C segment #include @@ -615,7 +616,7 @@ function md5_hmac%(...%): string ## Constructs an MD5 handle to enable incremental hash computation. You can ## feed data to the returned opaque value with :bro:id:`md5_hash_update` and ## eventually need to call :bro:id:`md5_hash_finish` to finish the computation -## and get the hash digest as result. +## and get the hash digest. ## ## For example, when computing incremental MD5 values of transferred files in ## multiple concurrent HTTP connections, one keeps an optional handle in the @@ -640,12 +641,12 @@ function md5_hash_init%(%): opaque of md5 ## Constructs an SHA1 handle to enable incremental hash computation. You can ## feed data to the returned opaque value with :bro:id:`sha1_hash_update` and ## finally need to call :bro:id:`sha1_hash_finish` to finish the computation -## and get the hash digest as result. +## and get the hash digest. ## ## For example, when computing incremental SHA1 values of transferred files in ## multiple concurrent HTTP connections, one keeps an optional handle in the ## HTTP session record. Then, one would call -## ``c$http$sha1_handle = sha1_hash_init()`` ## once before invoking +## ``c$http$sha1_handle = sha1_hash_init()`` once before invoking ## ``sha1_hash_update(c$http$sha1_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`sha1_hash_finish` returns the final hash value. @@ -665,12 +666,12 @@ function sha1_hash_init%(%): opaque of sha1 ## Constructs an SHA256 handle to enable incremental hash computation. You can ## feed data to the returned opaque value with :bro:id:`sha256_hash_update` and ## finally need to call :bro:id:`sha256_hash_finish` to finish the computation -## and get the hash digest as result. +## and get the hash digest. ## ## For example, when computing incremental SHA256 values of transferred files in ## multiple concurrent HTTP connections, one keeps an optional handle in the ## HTTP session record. Then, one would call -## ``c$http$sha256_handle = sha256_hash_init()`` ## once before invoking +## ``c$http$sha256_handle = sha256_hash_init()`` once before invoking ## ``sha256_hash_update(c$http$sha256_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`sha256_hash_finish` returns the final hash value. @@ -1532,7 +1533,7 @@ function log10%(d: double%): double # # =========================================================================== -## Determines whether *c* has been received externally. For example, +## Determines whether a connection has been received externally. For example, ## Broccoli or the Time Machine can send packets to Bro via a mechanism that is ## one step lower than sending events. This function checks whether the packets ## of a connection stem from one of these external *packet sources*. diff --git a/src/probabilistic/bloom-filter.bif b/src/probabilistic/bloom-filter.bif index f76a10a900..26865f160d 100644 --- a/src/probabilistic/bloom-filter.bif +++ b/src/probabilistic/bloom-filter.bif @@ -24,8 +24,8 @@ module GLOBAL; ## name: A name that uniquely identifies and seeds the Bloom filter. If empty, ## the filter will use :bro:id:`global_hash_seed` if that's set, and ## otherwise use a local seed tied to the current Bro process. Only -## filters with the same seed can be merged with -## :bro:id:`bloomfilter_merge` . +## filters with the same seed can be merged with +## :bro:id:`bloomfilter_merge`. ## ## Returns: A Bloom filter handle. ## @@ -50,8 +50,9 @@ function bloomfilter_basic_init%(fp: double, capacity: count, %} ## Creates a basic Bloom filter. This function serves as a low-level -## alternative to bloomfilter_basic_init where the user has full control over -## the number of hash functions and cells in the underlying bit vector. +## alternative to :bro:id:`bloomfilter_basic_init` where the user has full +## control over the number of hash functions and cells in the underlying bit +## vector. ## ## k: The number of hash functions to use. ## @@ -61,7 +62,7 @@ function bloomfilter_basic_init%(fp: double, capacity: count, ## the filter will use :bro:id:`global_hash_seed` if that's set, and ## otherwise use a local seed tied to the current Bro process. Only ## filters with the same seed can be merged with -## :bro:id:`bloomfilter_merge` . +## :bro:id:`bloomfilter_merge`. ## ## Returns: A Bloom filter handle. ## @@ -97,7 +98,7 @@ function bloomfilter_basic_init2%(k: count, cells: count, ## counting Bloom filter, we refer to the Bloom filter literature ## here for choosing an appropiate value. ## -## max: The maximum counter value associated with each each element +## max: The maximum counter value associated with each element ## described by *w = ceil(log_2(max))* bits. Each bit in the underlying ## counter vector becomes a cell of size *w* bits. ## @@ -105,7 +106,7 @@ function bloomfilter_basic_init2%(k: count, cells: count, ## the filter will use :bro:id:`global_hash_seed` if that's set, and ## otherwise use a local seed tied to the current Bro process. Only ## filters with the same seed can be merged with -## :bro:id:`bloomfilter_merge` . +## :bro:id:`bloomfilter_merge`. ## ## Returns: A Bloom filter handle. ## @@ -187,9 +188,9 @@ function bloomfilter_lookup%(bf: opaque of bloomfilter, x: any%): count return new Val(0, TYPE_COUNT); %} -## Removes all elements from a Bloom filter. This function resets all bits in the -## underlying bitvector back to 0 but does not change the parameterization of the -## Bloom filter, such as the element type and the hasher seed. +## Removes all elements from a Bloom filter. This function resets all bits in +## the underlying bitvector back to 0 but does not change the parameterization +## of the Bloom filter, such as the element type and the hasher seed. ## ## bf: The Bloom filter handle. ## @@ -242,6 +243,8 @@ function bloomfilter_merge%(bf1: opaque of bloomfilter, ## state. This is for debugging/testing purposes only. ## ## bf: The Bloom filter handle. +## +## Returns: a string with a representation of a Bloom filter's internal state. function bloomfilter_internal_state%(bf: opaque of bloomfilter%): string %{ BloomFilterVal* bfv = static_cast(bf); diff --git a/src/probabilistic/cardinality-counter.bif b/src/probabilistic/cardinality-counter.bif index f1113d73fc..aea9f296f5 100644 --- a/src/probabilistic/cardinality-counter.bif +++ b/src/probabilistic/cardinality-counter.bif @@ -8,11 +8,12 @@ using namespace probabilistic; module GLOBAL; -## Initializes a probabilistic cardinality counter that uses the HyperLogLog algorithm. +## Initializes a probabilistic cardinality counter that uses the HyperLogLog +## algorithm. ## ## err: the desired error rate (e.g. 0.01). ## -## confidence: the desirec confidence for the error rate (e.g., 0.95). +## confidence: the desired confidence for the error rate (e.g., 0.95). ## ## Returns: a HLL cardinality handle. ## @@ -30,9 +31,9 @@ function hll_cardinality_init%(err: double, confidence: double%): opaque of card ## ## handle: the HLL handle. ## -## elem: the element to add +## elem: the element to add. ## -## Returns: true on success +## Returns: true on success. ## ## .. bro:see:: hll_cardinality_estimate hll_cardinality_merge_into ## hll_cardinality_init hll_cardinality_copy @@ -61,11 +62,11 @@ function hll_cardinality_add%(handle: opaque of cardinality, elem: any%): bool ## .. note:: The same restrictions as for Bloom filter merging apply, ## see :bro:id:`bloomfilter_merge`. ## -## handle1: the first HLL handle, which will contain the merged result +## handle1: the first HLL handle, which will contain the merged result. ## -## handle2: the second HLL handle, which will be merged into the first +## handle2: the second HLL handle, which will be merged into the first. ## -## Returns: true on success +## Returns: true on success. ## ## .. bro:see:: hll_cardinality_estimate hll_cardinality_add ## hll_cardinality_init hll_cardinality_copy @@ -98,7 +99,7 @@ function hll_cardinality_merge_into%(handle1: opaque of cardinality, handle2: op ## Estimate the current cardinality of an HLL cardinality counter. ## -## handle: the HLL handle +## handle: the HLL handle. ## ## Returns: the cardinality estimate. Returns -1.0 if the counter is empty. ## @@ -116,9 +117,9 @@ function hll_cardinality_estimate%(handle: opaque of cardinality%): double ## Copy a HLL cardinality counter. ## -## handle: cardinality counter to copy +## handle: cardinality counter to copy. ## -## Returns: copy of handle +## Returns: copy of handle. ## ## .. bro:see:: hll_cardinality_estimate hll_cardinality_merge_into hll_cardinality_add ## hll_cardinality_init diff --git a/src/probabilistic/top-k.bif b/src/probabilistic/top-k.bif index 2725fe0422..e3dc891552 100644 --- a/src/probabilistic/top-k.bif +++ b/src/probabilistic/top-k.bif @@ -6,7 +6,7 @@ ## Creates a top-k data structure which tracks *size* elements. ## -## size: number of elements to track +## size: number of elements to track. ## ## Returns: Opaque pointer to the data structure. ## @@ -24,9 +24,9 @@ function topk_init%(size: count%): opaque of topk ## the top-k data structure. All following values have to be of the same ## type. ## -## handle: the TopK handle +## handle: the TopK handle. ## -## value: observed value +## value: observed value. ## ## .. bro:see:: topk_init topk_get_top topk_count topk_epsilon ## topk_size topk_sum topk_merge topk_merge_prune @@ -41,11 +41,11 @@ function topk_add%(handle: opaque of topk, value: any%): any ## Get the first *k* elements of the top-k data structure. ## -## handle: the TopK handle +## handle: the TopK handle. ## -## k: number of elements to return +## k: number of elements to return. ## -## Returns: vector of the first k elements +## Returns: vector of the first k elements. ## ## .. bro:see:: topk_init topk_add topk_count topk_epsilon ## topk_size topk_sum topk_merge topk_merge_prune @@ -56,16 +56,17 @@ function topk_get_top%(handle: opaque of topk, k: count%): any return h->GetTopK(k); %} -## Get an overestimated count of how often value has been encountered. +## Get an overestimated count of how often a value has been encountered. ## -## .. note:: value has to be part of the currently tracked elements, otherwise -## 0 will be returned and an error message will be added to reporter. +## .. note:: The value has to be part of the currently tracked elements, +## otherwise 0 will be returned and an error message will be added to +## reporter. ## -## handle: the TopK handle +## handle: the TopK handle. ## ## value: Value to look up count for. ## -## Returns: Overestimated number for how often the element has been encountered +## Returns: Overestimated number for how often the element has been encountered. ## ## .. bro:see:: topk_init topk_add topk_get_top topk_epsilon ## topk_size topk_sum topk_merge topk_merge_prune @@ -78,13 +79,14 @@ function topk_count%(handle: opaque of topk, value: any%): count ## Get the maximal overestimation for count. ## -## .. note:: Same restrictions as for :bro:id:`topk_count` apply. +## .. note:: Same restrictions as for :bro:id:`topk_count` apply. ## -## handle: the TopK handle +## handle: the TopK handle. ## ## value: Value to look up epsilon for. ## -## Returns: Number which represents the maximal overesimation for the count of this element. +## Returns: Number which represents the maximal overestimation for the count of +## this element. ## ## .. bro:see:: topk_init topk_add topk_get_top topk_count ## topk_size topk_sum topk_merge topk_merge_prune @@ -95,14 +97,15 @@ function topk_epsilon%(handle: opaque of topk, value: any%): count return new Val(h->GetEpsilon(value), TYPE_COUNT); %} -## Get the number of elements this data structure is supposed to track (given on init). +## Get the number of elements this data structure is supposed to track (given +## on init). ## -## .. note ::Note that the actual number of elements in the data structure can be lower -## or higher (due to non-pruned merges) than this. +## .. note:: Note that the actual number of elements in the data structure can +## be lower or higher (due to non-pruned merges) than this. ## -## handle: the TopK handle +## handle: the TopK handle. ## -## Returns: size given during initialization +## Returns: size given during initialization. ## ## .. bro:see:: topk_init topk_add topk_get_top topk_count topk_epsilon ## topk_sum topk_merge topk_merge_prune @@ -115,13 +118,14 @@ function topk_size%(handle: opaque of topk%): count ## Get the sum of all counts of all elements in the data structure. ## -## .. note:: This is equal to the number of all inserted objects if the data structure -## never has been pruned. Do not use after calling topk_merge_prune (will throw a -## warning message if used afterwards) +## .. note:: This is equal to the number of all inserted objects if the data +## structure never has been pruned. Do not use after +## calling :bro:id:`topk_merge_prune` (will throw a warning message if used +## afterwards). ## -## handle: the TopK handle +## handle: the TopK handle. ## -## Returns: sum of all counts +## Returns: sum of all counts. ## ## .. bro:see:: topk_init topk_add topk_get_top topk_count topk_epsilon ## topk_size topk_merge topk_merge_prune @@ -132,10 +136,10 @@ function topk_sum%(handle: opaque of topk%): count return new Val(h->GetSum(), TYPE_COUNT); %} -## Merge the second topk data structure into the first. +## Merge the second top-k data structure into the first. ## -## .. note:: This does not remove any elements, the resulting data structure can be -## bigger than the maximum size given on initialization. +## .. note:: This does not remove any elements, the resulting data structure +## can be bigger than the maximum size given on initialization. ## ## .. bro:see:: topk_init topk_add topk_get_top topk_count topk_epsilon ## topk_size topk_sum topk_merge_prune @@ -152,16 +156,16 @@ function topk_merge%(handle1: opaque of topk, handle2: opaque of topk%): any return 0; %} -## Merge the second topk data structure into the first and prunes the final data -## structure back to the size given on initialization. +## Merge the second top-k data structure into the first and prunes the final +## data structure back to the size given on initialization. ## ## .. note:: Use with care and only when being aware of the restrictions this -## entails. Do not call :bro:id:`topk_size` or :bro:id:`topk_add` afterwards, +## entails. Do not call :bro:id:`topk_size` or :bro:id:`topk_add` afterwards, ## results will probably not be what you expect. ## -## handle1: the TopK handle in which the second TopK structure is merged +## handle1: the TopK handle in which the second TopK structure is merged. ## -## handle2: the TopK handle in which is merged into the first TopK structure +## handle2: the TopK handle in which is merged into the first TopK structure. ## ## .. bro:see:: topk_init topk_add topk_get_top topk_count topk_epsilon ## topk_size topk_sum topk_merge diff --git a/src/strings.bif b/src/strings.bif index 233071d318..f50eb1f89b 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1052,6 +1052,8 @@ function find_last%(str: string, re: pattern%) : string ## ## data_str: The string to dump in hex format. ## +## Returns: The hex dump of the given string. +## ## .. bro:see:: string_to_ascii_hex bytestring_to_hexstr ## ## .. note:: Based on Netdude's hex editor code. From 9d2668a3f46019c959e064a7024bc4fc833b1b21 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 17 Oct 2013 01:50:42 -0500 Subject: [PATCH 28/36] Fix incorrect uses of reST directives These did not generate any warning messages unfortunately, so had to find them using grep. --- scripts/base/init-bare.bro | 22 +++++++++---------- scripts/policy/misc/known-devices.bro | 4 ++-- .../protocols/modbus/known-masters-slaves.bro | 6 ++--- .../policy/protocols/modbus/track-memmap.bro | 2 +- .../protocols/ssl/extract-certs-pem.bro | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index fe3b84a93b..e499c7cc7d 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -888,13 +888,13 @@ const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef; ## For services without an a handler, these sets define originator-side ports that ## still trigger reassembly. ## -## .. :bro:see:: tcp_reassembler_ports_resp +## .. bro:see:: tcp_reassembler_ports_resp const tcp_reassembler_ports_orig: set[port] = {} &redef; ## For services without an a handler, these sets define responder-side ports that ## still trigger reassembly. ## -## .. :bro:see:: tcp_reassembler_ports_orig +## .. bro:see:: tcp_reassembler_ports_orig const tcp_reassembler_ports_resp: set[port] = {} &redef; ## Defines destination TCP ports for which the contents of the originator stream @@ -987,7 +987,7 @@ const table_incremental_step = 5000 &redef; ## When expiring table entries, wait this amount of time before checking the next ## chunk of entries. ## -## .. :bro:see:: table_expire_interval table_incremental_step +## .. bro:see:: table_expire_interval table_incremental_step const table_expire_delay = 0.01 secs &redef; ## Time to wait before timing out a DNS request. @@ -1676,7 +1676,7 @@ global secondary_filters: table[string] of event(filter: string, pkt: pkt_hdr) ## Maximum length of payload passed to discarder functions. ## -## .. :bro:see:: discarder_check_tcp discarder_check_udp discarder_check_icmp +## .. bro:see:: discarder_check_tcp discarder_check_udp discarder_check_icmp ## discarder_check_ip global discarder_maxlen = 128 &redef; @@ -1689,7 +1689,7 @@ global discarder_maxlen = 128 &redef; ## ## Returns: True if the packet should not be analyzed any further. ## -## .. :bro:see:: discarder_check_tcp discarder_check_udp discarder_check_icmp +## .. bro:see:: discarder_check_tcp discarder_check_udp discarder_check_icmp ## discarder_maxlen ## ## .. note:: This is very low-level functionality and potentially expensive. @@ -1707,7 +1707,7 @@ global discarder_check_ip: function(p: pkt_hdr): bool; ## ## Returns: True if the packet should not be analyzed any further. ## -## .. :bro:see:: discarder_check_ip discarder_check_udp discarder_check_icmp +## .. bro:see:: discarder_check_ip discarder_check_udp discarder_check_icmp ## discarder_maxlen ## ## .. note:: This is very low-level functionality and potentially expensive. @@ -1725,7 +1725,7 @@ global discarder_check_tcp: function(p: pkt_hdr, d: string): bool; ## ## Returns: True if the packet should not be analyzed any further. ## -## .. :bro:see:: discarder_check_ip discarder_check_tcp discarder_check_icmp +## .. bro:see:: discarder_check_ip discarder_check_tcp discarder_check_icmp ## discarder_maxlen ## ## .. note:: This is very low-level functionality and potentially expensive. @@ -1741,7 +1741,7 @@ global discarder_check_udp: function(p: pkt_hdr, d: string): bool; ## ## Returns: True if the packet should not be analyzed any further. ## -## .. :bro:see:: discarder_check_ip discarder_check_tcp discarder_check_udp +## .. bro:see:: discarder_check_ip discarder_check_tcp discarder_check_udp ## discarder_maxlen ## ## .. note:: This is very low-level functionality and potentially expensive. @@ -1935,11 +1935,11 @@ export { ## .. .. bro:see:: return_data_max return_data_first_only const return_data = F &redef; - ## If bro:id:`NFS3::return_data` is true, how much data should be returned at + ## If :bro:id:`NFS3::return_data` is true, how much data should be returned at ## most. const return_data_max = 512 &redef; - ## If bro:id:`NFS3::return_data` is true, whether to *only* return data if the read + ## If :bro:id:`NFS3::return_data` is true, whether to *only* return data if the read ## or write offset is 0, i.e., only return data for the beginning of the file. const return_data_first_only = T &redef; @@ -2827,7 +2827,7 @@ const report_gaps_for_partial = F &redef; ## Flag to prevent Bro from exiting automatically when input is exhausted. ## Normally Bro terminates when all packets sources have gone dry ## and communication isn't enabled. If this flag is set, Bro's main loop will -## instead keep idleing until :bro:see::`terminate` is explicitly called. +## instead keep idleing until :bro:see:`terminate` is explicitly called. ## ## This is mainly for testing purposes when termination behaviour needs to be ## controlled for reproducing results. diff --git a/scripts/policy/misc/known-devices.bro b/scripts/policy/misc/known-devices.bro index f4776a990b..a7c0b314b7 100644 --- a/scripts/policy/misc/known-devices.bro +++ b/scripts/policy/misc/known-devices.bro @@ -3,11 +3,11 @@ ##! The log that is output provides an easy way to determine a count of the devices ##! in use on a network per day. ##! -##! ..note:: +##! .. note:: ##! ##! This script will not generate any logs on its own, it needs to be ##! supplied with information from elsewhere, such as -##! :doc:`policy/protocols/dhcp/known-devices-and-hostnames/scripts/. +##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`. module Known; diff --git a/scripts/policy/protocols/modbus/known-masters-slaves.bro b/scripts/policy/protocols/modbus/known-masters-slaves.bro index b1544a48dd..84ea7ea35a 100644 --- a/scripts/policy/protocols/modbus/known-masters-slaves.bro +++ b/scripts/policy/protocols/modbus/known-masters-slaves.bro @@ -1,8 +1,8 @@ ##! Script for tracking known Modbus masters and slaves. ##! -##! .. todo: This script needs a lot of work. What might be more interesting is to track -##! master/slave relationships based on commands sent and successful (non-exception) -##! responses. +##! .. todo:: This script needs a lot of work. What might be more interesting +##! is to track master/slave relationships based on commands sent and +##! successful (non-exception) responses. @load base/protocols/modbus diff --git a/scripts/policy/protocols/modbus/track-memmap.bro b/scripts/policy/protocols/modbus/track-memmap.bro index e2001e66bf..6ba788f4ed 100644 --- a/scripts/policy/protocols/modbus/track-memmap.bro +++ b/scripts/policy/protocols/modbus/track-memmap.bro @@ -1,7 +1,7 @@ ##! This script tracks the memory map of holding (read/write) registers and logs ##! changes as they are discovered. ##! -##! .. todo: Not all register reads and write functions are being supported yet. +##! .. todo:: Not all register read and write functions are supported yet. @load base/protocols/modbus @load base/utils/directions-and-hosts diff --git a/scripts/policy/protocols/ssl/extract-certs-pem.bro b/scripts/policy/protocols/ssl/extract-certs-pem.bro index 0a412a119d..d67310f502 100644 --- a/scripts/policy/protocols/ssl/extract-certs-pem.bro +++ b/scripts/policy/protocols/ssl/extract-certs-pem.bro @@ -2,7 +2,7 @@ ##! after being converted to PEM files. The certificates will be stored in ##! a single file, one for local certificates and one for remote certificates. ##! -##! ..note:: +##! .. note:: ##! ##! - It doesn't work well on a cluster because each worker will write its ##! own certificate files and no duplicate checking is done across From 8f2336f531e3225ac8a8f8f78afa0b511a6930ed Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 17 Oct 2013 12:47:32 -0500 Subject: [PATCH 29/36] Add README files for base/protocols The text from these README files appears on the "Bro Script Packages" page after building the documentation. --- scripts/base/protocols/conn/README | 1 + scripts/base/protocols/dhcp/README | 1 + scripts/base/protocols/dnp3/README | 1 + scripts/base/protocols/dns/README | 1 + scripts/base/protocols/ftp/README | 1 + scripts/base/protocols/http/README | 1 + scripts/base/protocols/irc/README | 1 + scripts/base/protocols/modbus/README | 1 + scripts/base/protocols/pop3/README | 1 + scripts/base/protocols/smtp/README | 1 + scripts/base/protocols/socks/README | 1 + scripts/base/protocols/ssh/README | 1 + scripts/base/protocols/ssl/README | 1 + scripts/base/protocols/syslog/README | 1 + 14 files changed, 14 insertions(+) create mode 100644 scripts/base/protocols/conn/README create mode 100644 scripts/base/protocols/dhcp/README create mode 100644 scripts/base/protocols/dnp3/README create mode 100644 scripts/base/protocols/dns/README create mode 100644 scripts/base/protocols/ftp/README create mode 100644 scripts/base/protocols/http/README create mode 100644 scripts/base/protocols/irc/README create mode 100644 scripts/base/protocols/modbus/README create mode 100644 scripts/base/protocols/pop3/README create mode 100644 scripts/base/protocols/smtp/README create mode 100644 scripts/base/protocols/socks/README create mode 100644 scripts/base/protocols/ssh/README create mode 100644 scripts/base/protocols/ssl/README create mode 100644 scripts/base/protocols/syslog/README diff --git a/scripts/base/protocols/conn/README b/scripts/base/protocols/conn/README new file mode 100644 index 0000000000..320dc50b4c --- /dev/null +++ b/scripts/base/protocols/conn/README @@ -0,0 +1 @@ +Support for connection (TCP, UDP, or ICMP) analysis. diff --git a/scripts/base/protocols/dhcp/README b/scripts/base/protocols/dhcp/README new file mode 100644 index 0000000000..507f065d46 --- /dev/null +++ b/scripts/base/protocols/dhcp/README @@ -0,0 +1 @@ +Support for Dynamic Host Configuration Protocol (DHCP) analysis. diff --git a/scripts/base/protocols/dnp3/README b/scripts/base/protocols/dnp3/README new file mode 100644 index 0000000000..c14a84c225 --- /dev/null +++ b/scripts/base/protocols/dnp3/README @@ -0,0 +1 @@ +Support for Distributed Network Protocol (DNP3) analysis. diff --git a/scripts/base/protocols/dns/README b/scripts/base/protocols/dns/README new file mode 100644 index 0000000000..bcece32e26 --- /dev/null +++ b/scripts/base/protocols/dns/README @@ -0,0 +1 @@ +Support for Domain Name System (DNS) protocol analysis. diff --git a/scripts/base/protocols/ftp/README b/scripts/base/protocols/ftp/README new file mode 100644 index 0000000000..7e035451a7 --- /dev/null +++ b/scripts/base/protocols/ftp/README @@ -0,0 +1 @@ +Support for File Transfer Protocol (FTP) analysis. diff --git a/scripts/base/protocols/http/README b/scripts/base/protocols/http/README new file mode 100644 index 0000000000..ebe2899a8d --- /dev/null +++ b/scripts/base/protocols/http/README @@ -0,0 +1 @@ +Support for Hypertext Transfer Protocol (HTTP) analysis. diff --git a/scripts/base/protocols/irc/README b/scripts/base/protocols/irc/README new file mode 100644 index 0000000000..03fdda21e1 --- /dev/null +++ b/scripts/base/protocols/irc/README @@ -0,0 +1 @@ +Support for Internet Relay Chat (IRC) protocol analysis. diff --git a/scripts/base/protocols/modbus/README b/scripts/base/protocols/modbus/README new file mode 100644 index 0000000000..dd75bd4eba --- /dev/null +++ b/scripts/base/protocols/modbus/README @@ -0,0 +1 @@ +Support for Modbus protocol analysis. diff --git a/scripts/base/protocols/pop3/README b/scripts/base/protocols/pop3/README new file mode 100644 index 0000000000..92eae3af34 --- /dev/null +++ b/scripts/base/protocols/pop3/README @@ -0,0 +1 @@ +Support for POP3 (Post Office Protocol) protocol analysis. diff --git a/scripts/base/protocols/smtp/README b/scripts/base/protocols/smtp/README new file mode 100644 index 0000000000..f647b6ca62 --- /dev/null +++ b/scripts/base/protocols/smtp/README @@ -0,0 +1 @@ +Support for Simple Mail Transfer Protocol (SMTP) analysis. diff --git a/scripts/base/protocols/socks/README b/scripts/base/protocols/socks/README new file mode 100644 index 0000000000..9b11e54d8a --- /dev/null +++ b/scripts/base/protocols/socks/README @@ -0,0 +1 @@ +Support for Socket Secure (SOCKS) protocol analysis. diff --git a/scripts/base/protocols/ssh/README b/scripts/base/protocols/ssh/README new file mode 100644 index 0000000000..c3f68d543f --- /dev/null +++ b/scripts/base/protocols/ssh/README @@ -0,0 +1 @@ +Support for Secure Shell (SSH) protocol analysis. diff --git a/scripts/base/protocols/ssl/README b/scripts/base/protocols/ssl/README new file mode 100644 index 0000000000..8fa99fd553 --- /dev/null +++ b/scripts/base/protocols/ssl/README @@ -0,0 +1 @@ +Support for Secure Sockets Layer (SSL) protocol analysis. diff --git a/scripts/base/protocols/syslog/README b/scripts/base/protocols/syslog/README new file mode 100644 index 0000000000..4592d82756 --- /dev/null +++ b/scripts/base/protocols/syslog/README @@ -0,0 +1 @@ +Support for Syslog protocol analysis. From 363cfb850639a01d6f9b871fba7cd8ddc796d749 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 12:24:40 -0700 Subject: [PATCH 30/36] rename the dbname configuration option to tablename. Sorry for this - I noticed that I named this option quite unfortunately while writing the documentation. The patch also removes the dbname configuration option from the sqlite input reader - it was not used there at all anymore (and I did not notice that). --- src/input/readers/SQLite.cc | 12 +----------- src/logging/writers/SQLite.cc | 14 +++++++------- .../.stderr | 2 +- .../scripts/base/frameworks/input/sqlite/basic.bro | 1 - .../scripts/base/frameworks/input/sqlite/error.bro | 2 -- .../scripts/base/frameworks/input/sqlite/port.bro | 1 - .../scripts/base/frameworks/input/sqlite/types.bro | 1 - 7 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/input/readers/SQLite.cc b/src/input/readers/SQLite.cc index e600788b16..eff34593d8 100644 --- a/src/input/readers/SQLite.cc +++ b/src/input/readers/SQLite.cc @@ -78,18 +78,8 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading: string fullpath(info.source); fullpath.append(".sqlite"); - string dbname; - map::const_iterator it = info.config.find("dbname"); - if ( it == info.config.end() ) - { - MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to source %s", info.source)); - dbname = info.source; - } - else - dbname = it->second; - string query; - it = info.config.find("query"); + map::const_iterator it = info.config.find("query"); if ( it == info.config.end() ) { Error(Fmt("No query specified when setting up SQLite data source. Aborting.", info.source)); diff --git a/src/logging/writers/SQLite.cc b/src/logging/writers/SQLite.cc index 76e5b84dfb..37e3134659 100644 --- a/src/logging/writers/SQLite.cc +++ b/src/logging/writers/SQLite.cc @@ -124,16 +124,16 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, string fullpath(info.path); fullpath.append(".sqlite"); - string dbname; + string tablename; - map::const_iterator it = info.config.find("dbname"); + map::const_iterator it = info.config.find("tablename"); if ( it == info.config.end() ) { - MsgThread::Info(Fmt("dbname configuration option not found. Defaulting to path %s", info.path)); - dbname = info.path; + MsgThread::Info(Fmt("tablename configuration option not found. Defaulting to path %s", info.path)); + tablename = info.path; } else - dbname = it->second; + tablename = it->second; if ( checkError(sqlite3_open_v2( fullpath.c_str(), @@ -145,7 +145,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, NULL)) ) return false; - string create = "CREATE TABLE IF NOT EXISTS " + dbname + " (\n"; + string create = "CREATE TABLE IF NOT EXISTS " + tablename + " (\n"; //"id SERIAL UNIQUE NOT NULL"; // SQLite has rowids, we do not need a counter here. for ( unsigned int i = 0; i < num_fields; ++i ) @@ -193,7 +193,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, // create the prepared statement that will be re-used forever... string insert = "VALUES ("; - string names = "INSERT INTO " + dbname + " ( "; + string names = "INSERT INTO " + tablename + " ( "; for ( unsigned int i = 0; i < num_fields; i++ ) { diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr index 96565881d3..2857d22ca9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.error/.stderr @@ -1,3 +1,3 @@ -ssh/Log::WRITER_SQLITE: dbname configuration option not found. Defaulting to path ssh +ssh/Log::WRITER_SQLITE: tablename configuration option not found. Defaulting to path ssh error: ssh/Log::WRITER_SQLITE: SQLite call failed: table ssh has no column named f error: ssh/Log::WRITER_SQLITE: terminating thread diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro b/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro index aa3a75ae4b..07906dab91 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/basic.bro @@ -89,7 +89,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from conn;", - ["dbname"] = "conn" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/error.bro b/testing/btest/scripts/base/frameworks/input/sqlite/error.bro index c6712df99a..08938e6df5 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/error.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/error.bro @@ -83,12 +83,10 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from ssh;", - ["dbname"] = "ssh" }; local config_strings2: table[string] of string = { ["query"] = "select b, g, h from ssh;", - ["dbname"] = "ssh" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/port.bro b/testing/btest/scripts/base/frameworks/input/sqlite/port.bro index c2949b5b3e..6fc18139fe 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/port.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/port.bro @@ -39,7 +39,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select port as p, proto from port;", - ["dbname"] = "port" }; outfile = open("../out"); diff --git a/testing/btest/scripts/base/frameworks/input/sqlite/types.bro b/testing/btest/scripts/base/frameworks/input/sqlite/types.bro index ddcbefa67f..42f8717c12 100644 --- a/testing/btest/scripts/base/frameworks/input/sqlite/types.bro +++ b/testing/btest/scripts/base/frameworks/input/sqlite/types.bro @@ -77,7 +77,6 @@ event bro_init() { local config_strings: table[string] of string = { ["query"] = "select * from ssh;", - ["dbname"] = "ssh" }; outfile = open("../out"); From 4aa363b0d2c5e38ad04b0c7409538c4fc13e3b2c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 14:47:15 -0700 Subject: [PATCH 31/36] add check that the SQLite reader is only used in MANUAL reading mode --- src/input/readers/SQLite.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/input/readers/SQLite.cc b/src/input/readers/SQLite.cc index eff34593d8..d02cc17fc7 100644 --- a/src/input/readers/SQLite.cc +++ b/src/input/readers/SQLite.cc @@ -54,7 +54,7 @@ void SQLite::DoClose() } } -bool SQLite::checkError( int code ) +bool SQLite::checkError(int code) { if ( code != SQLITE_OK && code != SQLITE_DONE ) { @@ -73,6 +73,12 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading: return false; } + if ( Info().mode != MODE_MANUAL ) + { + Error("SQLite only supports manual reading mode."); + return false; + } + started = false; string fullpath(info.source); From 5c2b2195f741af13a4cb3d98e38303e765f51b63 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 15:22:52 -0700 Subject: [PATCH 32/36] First try at sqlite reader/writer documentation --- doc/frameworks/input.rst | 11 +- doc/frameworks/logging-input-sqlite.rst | 236 ++++++++++++++++++++++++ doc/frameworks/logging.rst | 1 + 3 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 doc/frameworks/logging-input-sqlite.rst diff --git a/doc/frameworks/input.rst b/doc/frameworks/input.rst index aca5091972..ef40756a26 100644 --- a/doc/frameworks/input.rst +++ b/doc/frameworks/input.rst @@ -1,4 +1,6 @@ +.. _framework-input: + =============== Input Framework =============== @@ -260,8 +262,13 @@ to optimize the speed of the input framework. It can generate arbitrary amounts of semi-random data in all Bro data types supported by the input framework. -In the future, the input framework will get support for new data sources -like, for example, different databases. +Currently, Bro supports the following readers in addition to the +aforementioned ones: + +.. toctree:: + :maxdepth: 1 + + logging-input-sqlite Add_table options ----------------- diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst new file mode 100644 index 0000000000..2a1373a232 --- /dev/null +++ b/doc/frameworks/logging-input-sqlite.rst @@ -0,0 +1,236 @@ + +============================================ +Logging to and reading from SQLite Databases +============================================ + +.. rst-class:: opening + + Starting with version 2.2, Bro features a SQLite logging writer + as well as a SQLite input reader. SQLite is a simple, file-based, + widely used SQL database system. Using SQLite allows Bro to write + and access data in a format that is easy to use in interchange with + other applications. Due to the transactional nature of SQLite, + databases can be used by several applications simultaneously. Hence, + they can, for example, be used to make data that changes regularly available + to Bro on a continuing basis. + +.. contents:: + +Warning +======= + +In contrast to the ASCII plugins, the SQLite plugins have not yet +seen extensive use in production environments. While we are not aware +of any issues with them at the moment, we urge to caution when using them +in production environments. There could be lingering issues which only occur +when the plugins are used with high amounts of data or in high-load environments. + +Logging Data into SQLite Databases +================================== + +Logging support for SQLite is available in all Bro installations starting with +version 2.2. There is no need to load any additional scripts or for any compile-time +configurations. + +Sending data from existing logging streams to SQLite is rather straightforward. You +have to define a filter which specifies SQLite as the writer. + +The following example code adds SQLite as a filter for the connection log: + +.. code:: bro + + event bro_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } + +Bro will create the database file ``/var/db/conn.sqlite``, if it does not already exist. +It will also create a table with the name ``conn`` (if it does not exist) and start +appending connection information to the table. + +At the moment, SQLite databases are not rotated the same way ASCII log-files are. You +have to take care to create them in an adequate location. + +Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a +log stream, you can remove the default filter: + +.. code:: bro + + Log::remove_filter(Conn::LOG, "default"); + +If you examine the resulting SQLite database, the schema will contain the same fields +that are present in the ASCII log files:: + + # sqlite3 /var/db/conn.sqlite + + SQLite version 3.8.0.2 2013-09-03 17:11:13 + Enter ".help" for instructions + Enter SQL statements terminated with a ";" + sqlite> .schema + CREATE TABLE conn ( + 'ts' double precision, + 'uid' text, + 'id.orig_h' text, + 'id.orig_p' integer, + ... + +To create a custom SQLite log file, you have to create a new log stream that contains +just the information you want to commit to the database. Please refer to the +:ref:`framework-logging` documentation. + +Reading Data from SQLite Databases +================================== + +Like logging support, support for reading data from SQLite databases is built into Bro starting +with version 2.2. + +Just as with the text-based input readers (please refer to the :ref:`framework-input` +documentation for them), the SQLite reader can be used to read data - in this case the result of +SQL queries - into tables or into events. + +Reading data into Tables +------------------------ + +To read data from a SQLite database, we first have to provide Bro with the information, how +the resulting data will be structured. For this example, we expect that we have a SQLite database, +which contains host IP addresses and the user accounts that are allowed to log into a specific +machine. + +The SQLite commands to create the schema are as follows:: + + create table machines_to_users ( + host text unique not null, + users text not null); + + insert into machines_to_users values ('192.168.17.1', 'bernhard,matthias,seth'); + insert into machines_to_users values ('192.168.17.2', 'bernhard'); + insert into machines_to_users values ('192.168.17.3', 'seth,matthias'); + +After creating a file called ``hosts.sqlite`` with this content, we can read the resulting table +into Bro: + +.. code:: bro + + type Idx: record { + host: addr; + }; + + type Val: record { + users: set[string]; + }; + + global hostslist: table[addr] of Val = table(); + + event bro_init() { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + + event Input::end_of_data(name: string, source: string) { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } + +Afterwards, that table can be used to check logins into hosts against the available +userlist. + +Reading data to Events +---------------------- + +The second mode is to use the SQLite reader to output the input data to events. Typically there +are two reasons to do this. First, when the structure of the input data is too complicated +for a direct table import. In this case, the data can be read into an event which can then +create the necessary data structures in Bro in scriptland. + +The second reason is, that the dataset is too big to hold it in memory. In this case, the checks +can be performed on-demand, when Bro encounters a situation where it needs additional information. + +An example for this would be an internal huge database with malware hashes. Live database queries +could be used to check the sporadically happening downloads against the database. + +The SQLite commands to create the schema are as follows:: + + create table malware_hashes ( + hash text unique not null, + description text not null); + + insert into malware_hashes values ('86f7e437faa5a7fce15d1ddcb9eaeaea377667b8', 'malware a'); + insert into malware_hashes values ('e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98', 'malware b'); + insert into malware_hashes values ('84a516841ba77a5b4648de2cd0dfcb30ea46dbb4', 'malware c'); + insert into malware_hashes values ('3c363836cf4e16666669a25da280a1865c2d2874', 'malware d'); + insert into malware_hashes values ('58e6b3a414a1e090dfc6029add0f3555ccba127f', 'malware e'); + insert into malware_hashes values ('4a0a19218e082a343a1b17e5333409af9d98f0f5', 'malware f'); + insert into malware_hashes values ('54fd1711209fb1c0781092374132c66e79e2241b', 'malware g'); + insert into malware_hashes values ('27d5482eebd075de44389774fce28c69f45c8a75', 'malware h'); + insert into malware_hashes values ('73f45106968ff8dc51fba105fa91306af1ff6666', 'ftp-trace'); + + +The following code uses the file-analysis framework to get the sha1 hashes of files that are +transmitted over the network. For each hash, a SQL-query is run against SQLite. If the query +returns with a result, we had a hit against our malware-database and output the matching hash. + +.. code:: bro + + @load frameworks/files/hash-all-files + + type Val: record { + hash: string; + description: string; + }; + + event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hot with hash %s, description %s", r$hash, r$description); + } + + global malware_source = "/var/db/malware"; + + event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $reader=Input::READER_SQLITE, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ) + ]); + } + } + + event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } + diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index 93cbf24e84..eb64df4ec3 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -387,3 +387,4 @@ Bro supports the following output formats other than ASCII: logging-dataseries logging-elasticsearch + logging-input-sqlite From dc685bbef3c3a51b2595481b38abe74676706fad Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 15:52:00 -0700 Subject: [PATCH 33/36] and restructure it a bit --- doc/frameworks/logging-input-sqlite.rst | 33 +++++++++++++------------ 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 2a1373a232..7c8c848c5f 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -19,9 +19,9 @@ Logging to and reading from SQLite Databases Warning ======= -In contrast to the ASCII plugins, the SQLite plugins have not yet +In contrast to the ASCII reader and writer, the SQLite plugins have not yet seen extensive use in production environments. While we are not aware -of any issues with them at the moment, we urge to caution when using them +of any issues with them, we urge to caution when using them in production environments. There could be lingering issues which only occur when the plugins are used with high amounts of data or in high-load environments. @@ -59,13 +59,6 @@ appending connection information to the table. At the moment, SQLite databases are not rotated the same way ASCII log-files are. You have to take care to create them in an adequate location. -Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a -log stream, you can remove the default filter: - -.. code:: bro - - Log::remove_filter(Conn::LOG, "default"); - If you examine the resulting SQLite database, the schema will contain the same fields that are present in the ASCII log files:: @@ -81,10 +74,18 @@ that are present in the ASCII log files:: 'id.orig_h' text, 'id.orig_p' integer, ... - + +Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a +log stream, you can remove the default filter: + +.. code:: bro + + Log::remove_filter(Conn::LOG, "default"); + + To create a custom SQLite log file, you have to create a new log stream that contains just the information you want to commit to the database. Please refer to the -:ref:`framework-logging` documentation. +:ref:`framework-logging` documentation on how to create custom log streams. Reading Data from SQLite Databases ================================== @@ -93,8 +94,8 @@ Like logging support, support for reading data from SQLite databases is built in with version 2.2. Just as with the text-based input readers (please refer to the :ref:`framework-input` -documentation for them), the SQLite reader can be used to read data - in this case the result of -SQL queries - into tables or into events. +documentation for them and for basic information on how to use the input-framework), the SQLite reader +can be used to read data - in this case the result of SQL queries - into tables or into events. Reading data into Tables ------------------------ @@ -202,7 +203,7 @@ returns with a result, we had a hit against our malware-database and output the event line(description: Input::EventDescription, tpe: Input::Event, r: Val) { - print fmt("malware-hot with hash %s, description %s", r$hash, r$description); + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); } global malware_source = "/var/db/malware"; @@ -220,10 +221,10 @@ returns with a result, we had a hit against our malware-database and output the $fields=Val, $ev=line, $want_record=T, - $reader=Input::READER_SQLITE, $config=table( ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) - ) + ), + $reader=Input::READER_SQLITE ]); } } From 613a04d1761d0b5c3fbfbe7531bffdff2a41911e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 17 Oct 2013 16:00:22 -0700 Subject: [PATCH 34/36] and provide a bit of motivation to try the last example. --- doc/frameworks/logging-input-sqlite.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 7c8c848c5f..370c8d8573 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -235,3 +235,5 @@ returns with a result, we had a hit against our malware-database and output the Input::remove(name); } +If you run this script against the trace in ``testing/btest/Traces/ftp/ipv4.trace``, you +will get one hit. From 1759e0d68788e463301178e2e07103e847f226f6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Oct 2013 13:59:50 -0700 Subject: [PATCH 35/36] Updating test baseline. --- .../output | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output index c2e558bac8..e3f7a39429 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output @@ -5,6 +5,7 @@ main.bro module HTTP; export { - ## This setting changes if passwords used in Basic-Auth are captured or not. + ## This setting changes if passwords used in Basic-Auth are captured or + ## not. const default_capture_password = F &redef; } From 8bfb81ca6fdd1238504b29a6a866170cd0211be6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Oct 2013 14:09:04 -0700 Subject: [PATCH 36/36] Moving the SQLite examples into separate Bro files to turn them into sphinx-btest tests. It's hard to make sure these are actually working as expected, but the tests now at least make sure things parse correctly. --- CHANGES | 5 + VERSION | 2 +- doc/frameworks/logging-input-sqlite.rst | 97 +++---------------- doc/frameworks/sqlite-conn-filter.bro | 12 +++ doc/frameworks/sqlite-read-events.bro | 40 ++++++++ doc/frameworks/sqlite-read-table.bro | 35 +++++++ .../output | 16 +++ .../output | 44 +++++++++ .../output | 39 ++++++++ ...oc_frameworks_sqlite-conn-filter_bro.btest | 16 +++ ...oc_frameworks_sqlite-read-events_bro.btest | 44 +++++++++ ...doc_frameworks_sqlite-read-table_bro.btest | 39 ++++++++ .../doc/sphinx/sqlite-conn-filter-check.btest | 2 + .../doc/sphinx/sqlite-read-events-check.btest | 2 + .../doc/sphinx/sqlite-read-table-check.btest | 2 + 15 files changed, 309 insertions(+), 86 deletions(-) create mode 100644 doc/frameworks/sqlite-conn-filter.bro create mode 100644 doc/frameworks/sqlite-read-events.bro create mode 100644 doc/frameworks/sqlite-read-table.bro create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-conn-filter_bro/output create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-events_bro/output create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-table_bro/output create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_sqlite-conn-filter_bro.btest create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-events_bro.btest create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-table_bro.btest create mode 100644 testing/btest/doc/sphinx/sqlite-conn-filter-check.btest create mode 100644 testing/btest/doc/sphinx/sqlite-read-events-check.btest create mode 100644 testing/btest/doc/sphinx/sqlite-read-table-check.btest diff --git a/CHANGES b/CHANGES index 57c5ff7a40..0da3362974 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.2-beta-114 | 2013-10-18 14:17:57 -0700 + + * Moving the SQLite examples into separate Bro files to turn them + into sphinx-btest tests. (Robin Sommer) + 2.2-beta-112 | 2013-10-18 13:47:13 -0700 * A larger chunk of documentation fixes and cleanup. (Daniel Thayer) diff --git a/VERSION b/VERSION index fb80125ef8..8acc30c190 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-beta-112 +2.2-beta-114 diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 47ee1fa23f..15df91b8c6 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -37,20 +37,12 @@ have to define a filter which specifies SQLite as the writer. The following example code adds SQLite as a filter for the connection log: -.. code:: bro +.. btest-include:: ${DOC_ROOT}/frameworks/sqlite-conn-filter.bro - event bro_init() - { - local filter: Log::Filter = - [ - $name="sqlite", - $path="/var/db/conn", - $config=table(["tablename"] = "conn"), - $writer=Log::WRITER_SQLITE - ]; +.. btest:: sqlite-conn-filter-check - Log::add_filter(Conn::LOG, filter); - } + # Make sure this parses correctly at least. + @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-conn-filter.bro Bro will create the database file ``/var/db/conn.sqlite``, if it does not already exist. It will also create a table with the name ``conn`` (if it does not exist) and start @@ -118,41 +110,12 @@ The SQLite commands to create the schema are as follows:: After creating a file called ``hosts.sqlite`` with this content, we can read the resulting table into Bro: -.. code:: bro +.. btest-include:: ${DOC_ROOT}/frameworks/sqlite-read-table.bro - type Idx: record { - host: addr; - }; +.. btest:: sqlite-read-table-check - type Val: record { - users: set[string]; - }; - - global hostslist: table[addr] of Val = table(); - - event bro_init() { - Input::add_table([$source="/var/db/hosts", - $name="hosts", - $idx=Idx, - $val=Val, - $destination=hostslist, - $reader=Input::READER_SQLITE, - $config=table(["query"] = "select * from machines_to_users;") - ]); - - Input::remove("hosts"); - } - - event Input::end_of_data(name: string, source: string) { - if ( name != "hosts" ) - return; - - # now all data is in the table - print "Hosts list has been successfully imported"; - - # List the users of one host. - print hostslist[192.168.17.1]$users; - } + # Make sure this parses correctly at least. + @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-table.bro Afterwards, that table can be used to check logins into hosts against the available userlist. @@ -192,48 +155,12 @@ The following code uses the file-analysis framework to get the sha1 hashes of fi transmitted over the network. For each hash, a SQL-query is run against SQLite. If the query returns with a result, we had a hit against our malware-database and output the matching hash. -.. code:: bro +.. btest-include:: ${DOC_ROOT}/frameworks/sqlite-read-events.bro - @load frameworks/files/hash-all-files +.. btest:: sqlite-read-events-check - type Val: record { - hash: string; - description: string; - }; - - event line(description: Input::EventDescription, tpe: Input::Event, r: Val) - { - print fmt("malware-hit with hash %s, description %s", r$hash, r$description); - } - - global malware_source = "/var/db/malware"; - - event file_hash(f: fa_file, kind: string, hash: string) - { - - # check all sha1 hashes - if ( kind=="sha1" ) - { - Input::add_event( - [ - $source=malware_source, - $name=hash, - $fields=Val, - $ev=line, - $want_record=T, - $config=table( - ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) - ), - $reader=Input::READER_SQLITE - ]); - } - } - - event Input::end_of_data(name: string, source:string) - { - if ( source == malware_source ) - Input::remove(name); - } + # Make sure this parses correctly at least. + @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-events.bro If you run this script against the trace in ``testing/btest/Traces/ftp/ipv4.trace``, you will get one hit. diff --git a/doc/frameworks/sqlite-conn-filter.bro b/doc/frameworks/sqlite-conn-filter.bro new file mode 100644 index 0000000000..c7927d2e60 --- /dev/null +++ b/doc/frameworks/sqlite-conn-filter.bro @@ -0,0 +1,12 @@ +event bro_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } diff --git a/doc/frameworks/sqlite-read-events.bro b/doc/frameworks/sqlite-read-events.bro new file mode 100644 index 0000000000..f14513e0cb --- /dev/null +++ b/doc/frameworks/sqlite-read-events.bro @@ -0,0 +1,40 @@ +@load frameworks/files/hash-all-files + +type Val: record { + hash: string; + description: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); + } + +global malware_source = "/var/db/malware"; + +event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ), + $reader=Input::READER_SQLITE + ]); + } + } + +event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } diff --git a/doc/frameworks/sqlite-read-table.bro b/doc/frameworks/sqlite-read-table.bro new file mode 100644 index 0000000000..f969fc52c7 --- /dev/null +++ b/doc/frameworks/sqlite-read-table.bro @@ -0,0 +1,35 @@ +type Idx: record { + host: addr; +}; + +type Val: record { + users: set[string]; +}; + +global hostslist: table[addr] of Val = table(); + +event bro_init() + { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + +event Input::end_of_data(name: string, source: string) + { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-conn-filter_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-conn-filter_bro/output new file mode 100644 index 0000000000..dc42f0bce1 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-conn-filter_bro/output @@ -0,0 +1,16 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sqlite-conn-filter.bro + +event bro_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-events_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-events_bro/output new file mode 100644 index 0000000000..6703c4ca7e --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-events_bro/output @@ -0,0 +1,44 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sqlite-read-events.bro + +@load frameworks/files/hash-all-files + +type Val: record { + hash: string; + description: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); + } + +global malware_source = "/var/db/malware"; + +event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ), + $reader=Input::READER_SQLITE + ]); + } + } + +event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-table_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-table_bro/output new file mode 100644 index 0000000000..dea06055ea --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sqlite-read-table_bro/output @@ -0,0 +1,39 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sqlite-read-table.bro + +type Idx: record { + host: addr; +}; + +type Val: record { + users: set[string]; +}; + +global hostslist: table[addr] of Val = table(); + +event bro_init() + { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + +event Input::end_of_data(name: string, source: string) + { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-conn-filter_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-conn-filter_bro.btest new file mode 100644 index 0000000000..dc42f0bce1 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-conn-filter_bro.btest @@ -0,0 +1,16 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sqlite-conn-filter.bro + +event bro_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-events_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-events_bro.btest new file mode 100644 index 0000000000..6703c4ca7e --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-events_bro.btest @@ -0,0 +1,44 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sqlite-read-events.bro + +@load frameworks/files/hash-all-files + +type Val: record { + hash: string; + description: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); + } + +global malware_source = "/var/db/malware"; + +event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ), + $reader=Input::READER_SQLITE + ]); + } + } + +event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-table_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-table_bro.btest new file mode 100644 index 0000000000..dea06055ea --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sqlite-read-table_bro.btest @@ -0,0 +1,39 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sqlite-read-table.bro + +type Idx: record { + host: addr; +}; + +type Val: record { + users: set[string]; +}; + +global hostslist: table[addr] of Val = table(); + +event bro_init() + { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + +event Input::end_of_data(name: string, source: string) + { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } diff --git a/testing/btest/doc/sphinx/sqlite-conn-filter-check.btest b/testing/btest/doc/sphinx/sqlite-conn-filter-check.btest new file mode 100644 index 0000000000..c34319428f --- /dev/null +++ b/testing/btest/doc/sphinx/sqlite-conn-filter-check.btest @@ -0,0 +1,2 @@ +# Make sure this parses correctly at least. +@TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-conn-filter.bro diff --git a/testing/btest/doc/sphinx/sqlite-read-events-check.btest b/testing/btest/doc/sphinx/sqlite-read-events-check.btest new file mode 100644 index 0000000000..7a0f291882 --- /dev/null +++ b/testing/btest/doc/sphinx/sqlite-read-events-check.btest @@ -0,0 +1,2 @@ +# Make sure this parses correctly at least. +@TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-events.bro diff --git a/testing/btest/doc/sphinx/sqlite-read-table-check.btest b/testing/btest/doc/sphinx/sqlite-read-table-check.btest new file mode 100644 index 0000000000..f696e30f1a --- /dev/null +++ b/testing/btest/doc/sphinx/sqlite-read-table-check.btest @@ -0,0 +1,2 @@ +# Make sure this parses correctly at least. +@TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-table.bro