From bf08770764f00d9d15246338464d8e3a0990c8d7 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 4 Mar 2015 13:31:13 -0600 Subject: [PATCH 01/15] Correct a minor typo in the docs --- src/strings.bif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/strings.bif b/src/strings.bif index b8d21cb04a..1f09667064 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -698,7 +698,7 @@ function split_n%(str: string, re: pattern, ## ## Returns: An array of strings where, if *incl_sep* is true, each two ## successive elements correspond to a substring in *str* of the part -## not matching *re* (event-indexed) and the part that matches *re* +## not matching *re* (even-indexed) and the part that matches *re* ## (odd-indexed). ## ## .. bro:see:: split_string split_string1 split_string_all str_split From 8841d0ae771b3d201ded29e921d8ef3fbe02fbaa Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 19 Mar 2015 16:01:28 -0500 Subject: [PATCH 02/15] Minor improvements to logging framework documentation --- scripts/base/frameworks/logging/main.bro | 51 ++++++++++--------- .../base/frameworks/logging/writers/ascii.bro | 14 ++--- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index d4d5c0244e..aecc552146 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -6,9 +6,10 @@ module Log; export { - ## Type that defines an ID unique to each log stream. Scripts creating new log - ## streams need to redef this enum to add their own specific log ID. The log ID - ## implicitly determines the default name of the generated log file. + ## Type that defines an ID unique to each log stream. Scripts creating new + ## log streams need to redef this enum to add their own specific log ID. + ## The log ID implicitly determines the default name of the generated log + ## file. type Log::ID: enum { ## Dummy place-holder. UNKNOWN @@ -20,25 +21,24 @@ export { ## If true, remote logging is by default enabled for all filters. const enable_remote_logging = T &redef; - ## Default writer to use if a filter does not specify - ## anything else. + ## Default writer to use if a filter does not specify anything else. const default_writer = WRITER_ASCII &redef; - ## Default separator between fields for logwriters. - ## Can be overwritten by individual writers. + ## Default separator to use between fields. + ## Individual writers can use a different value. const separator = "\t" &redef; - ## Separator between set elements. - ## Can be overwritten by individual writers. + ## Default separator to use between elements of a set. + ## Individual writers can use a different value. const set_separator = "," &redef; - ## String to use for empty fields. This should be different from - ## *unset_field* to make the output unambiguous. - ## Can be overwritten by individual writers. + ## Default string to use for empty fields. This should be different + ## from *unset_field* to make the output unambiguous. + ## Individual writers can use a different value. const empty_field = "(empty)" &redef; - ## String to use for an unset &optional field. - ## Can be overwritten by individual writers. + ## Default string to use for an unset &optional field. + ## Individual writers can use a different value. const unset_field = "-" &redef; ## Type defining the content of a logging stream. @@ -63,7 +63,7 @@ export { ## If no ``path`` is defined for the filter, then the first call ## to the function will contain an empty string. ## - ## rec: An instance of the streams's ``columns`` type with its + ## rec: An instance of the stream's ``columns`` type with its ## fields set to the values to be logged. ## ## Returns: The path to be used for the filter. @@ -81,7 +81,8 @@ export { terminating: bool; ##< True if rotation occured due to Bro shutting down. }; - ## Default rotation interval. Zero disables rotation. + ## Default rotation interval to use for filters that do not specify + ## an interval. Zero disables rotation. ## ## Note that this is overridden by the BroControl LogRotationInterval ## option. @@ -116,8 +117,8 @@ export { ## Indicates whether a log entry should be recorded. ## If not given, all entries are recorded. ## - ## rec: An instance of the streams's ``columns`` type with its - ## fields set to the values to logged. + ## rec: An instance of the stream's ``columns`` type with its + ## fields set to the values to be logged. ## ## Returns: True if the entry is to be recorded. pred: function(rec: any): bool &optional; @@ -125,10 +126,10 @@ export { ## Output path for recording entries matching this ## filter. ## - ## The specific interpretation of the string is up to - ## the used writer, and may for example be the destination + ## The specific interpretation of the string is up to the + ## logging writer, and may for example be the destination ## file name. Generally, filenames are expected to be given - ## without any extensions; writers will add appropiate + ## without any extensions; writers will add appropriate ## extensions automatically. ## ## If this path is found to conflict with another filter's @@ -153,7 +154,7 @@ export { ## then the first call to the function will contain an ## empty string. ## - ## rec: An instance of the streams's ``columns`` type with its + ## rec: An instance of the stream's ``columns`` type with its ## fields set to the values to be logged. ## ## Returns: The path to be used for the filter, which will be @@ -177,7 +178,7 @@ export { ## If true, entries are passed on to remote peers. log_remote: bool &default=enable_remote_logging; - ## Rotation interval. + ## Rotation interval. Zero disables rotation. interv: interval &default=default_rotation_interval; ## Callback function to trigger for rotated files. If not set, the @@ -207,9 +208,9 @@ export { ## Removes a logging stream completely, stopping all the threads. ## - ## id: The ID enum to be associated with the new logging stream. + ## id: The ID associated with the logging stream. ## - ## Returns: True if a new stream was successfully removed. + ## Returns: True if the stream was successfully removed. ## ## .. bro:see:: Log::create_stream global remove_stream: function(id: ID) : bool; diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 5f59229f7f..c10c86145e 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -1,15 +1,15 @@ ##! Interface for the ASCII log writer. Redefinable options are available ##! to tweak the output format of ASCII logs. ##! -##! The ASCII writer supports currently one writer-specific filter option via -##! ``config``: setting ``tsv`` to the string ``T`` turns the output into +##! The ASCII writer currently supports one writer-specific per-filter config +##! option: setting ``tsv`` to the string ``T`` turns the output into ##! "tab-separated-value" mode where only a single header row with the column ##! names is printed out as meta information, with no "# fields" prepended; no -##! other meta data gets included in that mode. +##! other meta data gets included in that mode. Example filter using this:: ##! -##! Example filter using this:: -##! -##! local my_filter: Log::Filter = [$name = "my-filter", $writer = Log::WRITER_ASCII, $config = table(["tsv"] = "T")]; +##! local f: Log::Filter = [$name = "my-filter", +##! $writer = Log::WRITER_ASCII, +##! $config = table(["tsv"] = "T")]; ##! module LogAscii; @@ -29,6 +29,8 @@ export { ## Format of timestamps when writing out JSON. By default, the JSON ## formatter will use double values for timestamps which represent the ## number of seconds from the UNIX epoch. + ## + ## This option is also available as a per-filter ``$config`` option. const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef; ## If true, include lines with log meta information such as column names From fcaf1d9c95e8ee0c89003cf380a3f3f5e2deaa1a Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 25 May 2015 13:08:03 -0500 Subject: [PATCH 03/15] Update install documentation and fix some typos --- doc/install/guidelines.rst | 10 ++++++---- doc/install/install.rst | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/install/guidelines.rst b/doc/install/guidelines.rst index af33b8fee1..d1e1777165 100644 --- a/doc/install/guidelines.rst +++ b/doc/install/guidelines.rst @@ -8,10 +8,12 @@ How to Upgrade If you're doing an upgrade install (rather than a fresh install), there's two suggested approaches: either install Bro using the same installation prefix directory as before, or pick a new prefix and copy -local customizations over. Regardless of which approach you choose, -if you are using BroControl, then after upgrading Bro you will need to -run "broctl check" (to verify that your new configuration is OK) -and "broctl install" to complete the upgrade process. +local customizations over. + +Regardless of which approach you choose, if you are using BroControl, then +before doing the upgrade you should stop all running Bro processes with the +"broctl stop" command. After the upgrade is complete then you will need +to run "broctl deploy". In the following we summarize general guidelines for upgrading, see the :ref:`release-notes` for version-specific information. diff --git a/doc/install/install.rst b/doc/install/install.rst index 18bacd7758..ab3f6cafc8 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -46,8 +46,7 @@ To build Bro from source, the following additional dependencies are required: * zlib headers * Perl -To install the required dependencies, you can use (when done, make sure -that ``bash`` and ``python`` are in your ``PATH``): +To install the required dependencies, you can use: * RPM/RedHat-based Linux: @@ -68,13 +67,17 @@ that ``bash`` and ``python`` are in your ``PATH``): .. console:: - sudo pkg_add -r bash cmake swig bison python perl py27-sqlite3 + sudo pkg install bash cmake swig bison python perl py27-sqlite3 + + Note that in older versions of FreeBSD, you might have to use the + "pkg_add -r" command instead of "pkg install". * Mac OS X: - Compiling source code on Macs requires first downloading Xcode_, - then going through its "Preferences..." -> "Downloads" menus to - install the "Command Line Tools" component. + Compiling source code on Macs requires first installing Xcode_ (in older + versions of Xcode, you would then need to go through its + "Preferences..." -> "Downloads" menus to install the "Command Line Tools" + component). OS X comes with all required dependencies except for CMake_ and SWIG_. Distributions of these dependencies can likely be obtained from your @@ -94,7 +97,6 @@ build time: * curl (used by a Bro script that implements active HTTP) * gperftools (tcmalloc is used to improve memory and CPU usage) * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump) - * Ruby executable, library, and headers (for Broccoli Ruby bindings) LibGeoIP is probably the most interesting and can be installed on most platforms by following the instructions for :ref:`installing @@ -119,8 +121,8 @@ platforms for binary releases and for installation instructions. Linux based binary installations are usually performed by adding information about the Bro packages to the respective system packaging - tool. Theen the usual system utilities such as ``apt``, ``yum`` - or ``zyppper`` are used to perforn the installation. By default, + tool. Then the usual system utilities such as ``apt``, ``yum`` + or ``zypper`` are used to perform the installation. By default, installations of binary packages will go into ``/opt/bro``. * MacOS Disk Image with Installer @@ -131,7 +133,7 @@ platforms for binary releases and for installation instructions. The primary install prefix for binary packages is ``/opt/bro``. Installing from Source -========================== +====================== Bro releases are bundled into source packages for convenience and are available on the `bro downloads page`_. Alternatively, the latest From e02ad1711c1b76bb02c1cc7051e16e8128e2c344 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 27 May 2015 16:23:02 -0500 Subject: [PATCH 04/15] Add link to broctl doc from the quickstart doc --- doc/quickstart/index.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/quickstart/index.rst b/doc/quickstart/index.rst index 616c94c261..ba9896e19d 100644 --- a/doc/quickstart/index.rst +++ b/doc/quickstart/index.rst @@ -24,9 +24,10 @@ Managing Bro with BroControl BroControl is an interactive shell for easily operating/managing Bro installations on a single system or even across multiple systems in a traffic-monitoring cluster. This section explains how to use BroControl -to manage a stand-alone Bro installation. For instructions on how to -configure a Bro cluster, see the :doc:`Cluster Configuration -<../configuration/index>` documentation. +to manage a stand-alone Bro installation. For a complete reference on +BroControl, see the :doc:`BroControl <../components/broctl/README>` +documentation. For instructions on how to configure a Bro cluster, +see the :doc:`Cluster Configuration <../configuration/index>` documentation. A Minimal Starting Configuration -------------------------------- From 7cf04c9f3a1d425a6bc5c8fa9895b65fc9ef21c3 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 28 May 2015 17:52:32 -0500 Subject: [PATCH 05/15] Improve logging framework doc Reorganized the content to be easier to follow, added a few more examples, fixed some ugly formatting (removed scrollbars that make the examples difficult to read). --- doc/frameworks/logging-input-sqlite.rst | 4 +- doc/frameworks/logging.rst | 618 ++++++++++++++---------- 2 files changed, 371 insertions(+), 251 deletions(-) diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 15df91b8c6..6f5e867686 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -67,8 +67,8 @@ that are present in the ASCII log files:: 'id.orig_p' integer, ... -Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a -log stream, you can remove the default filter: +Note that the ASCII ``conn.log`` will still be created. To prevent this file +from being created, you can remove the default filter: .. code:: bro diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index 765d7bed23..7cb21da36b 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -19,195 +19,128 @@ Terminology Bro's logging interface is built around three main abstractions: - Log streams - A stream corresponds to a single log. It defines the set of - fields that a log consists of with their names and fields. - Examples are the ``conn`` for recording connection summaries, + Streams + A log stream corresponds to a single log. It defines the set of + fields that a log consists of with their names and types. + Examples are the ``conn`` stream for recording connection summaries, and the ``http`` stream for recording HTTP activity. Filters Each stream has a set of filters attached to it that determine what information gets written out. By default, each stream has - one default filter that just logs everything directly to disk - with an automatically generated file name. However, further - filters can be added to record only a subset, split a stream - into different outputs, or to even duplicate the log to - multiple outputs. If all filters are removed from a stream, - all output is disabled. + one default filter that just logs everything directly to disk. + However, additional filters can be added to record only a subset + of the log records, write to different outputs, or set a custom + rotation interval. If all filters are removed from a stream, + then output is disabled for that stream. Writers - A writer defines the actual output format for the information - being logged. At the moment, Bro comes with only one type of - writer, which produces tab separated ASCII files. In the - future we will add further writers, like for binary output and - direct logging into a database. + Each filter has a writer. A writer defines the actual output + format for the information being logged. The default writer is + the ASCII writer, which produces tab-separated ASCII files. Other + writers are available, like for binary output or direct logging + into a database. -Basics -====== +There are several different ways to customize Bro's logging: you can create +a new log stream, you can extend an existing log with new fields, you +can apply filters to an existing log stream, or you can customize the output +format by setting log writer options. All of these approaches are +described in this document. -The data fields that a stream records are defined by a record type -specified when it is created. Let's look at the script generating Bro's -connection summaries as an example, -:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record -:bro:type:`Conn::Info` that lists all the fields that go into -``conn.log``, each marked with a ``&log`` attribute indicating that it -is part of the information written out. To write a log record, the -script then passes an instance of :bro:type:`Conn::Info` to the logging -framework's :bro:id:`Log::write` function. +Streams +======= -By default, each stream automatically gets a filter named ``default`` -that generates the normal output by recording all record fields into a -single output file. +In order to log data to a new log stream, all of the following needs to be +done: -In the following, we summarize ways in which the logging can be -customized. We continue using the connection summaries as our example -to work with. +- A :bro:type:`record` type must be defined which consists of all the + fields that will be logged (by convention, the name of this record type is + usually "Info"). +- A log stream ID (an :bro:type:`enum` with type name "Log::ID") must be + defined that uniquely identifies the new log stream. +- A log stream must be created using the :bro:id:`Log::create_stream` function. +- When the data to be logged becomes available, the :bro:id:`Log::write` + function must be called. -Filtering ---------- - -To create a new output file for an existing stream, you can add a -new filter. A filter can, e.g., restrict the set of fields being -logged: +In the following example, we create a new module "Foo" which creates +a new log stream. .. code:: bro - event bro_init() + module Foo; + + export { + # Create an ID for our new stream. By convention, this is + # called "LOG". + redef enum Log::ID += { LOG }; + + # Define the record type that will contain the data to log. + # By convention, the type is called "Info". + # All fields to log must have the &log attribute. + # If a field might not contain any data, use the + # &optional attribute. + type Info: record { + ts: time &log; + id: conn_id &log; + service: string &log &optional; + }; + } + + # Optionally, we can add a new field to the connection record so that + # the data we are logging (our "Info" record) will be easily + # accessible in a variety of event handlers. + # By convention, the name of this new field is the lowercase name of + # the module. + redef record connection += { + foo: Info &optional; + }; + + # This event is handled at a priority higher than zero so that if + # users modify this stream in another script, they can do so at the + # default priority of zero. + event bro_init() &priority=5 { - # Add a new filter to the Conn::LOG stream that logs only - # timestamp and originator address. - local filter: Log::Filter = [$name="orig-only", $path="origs", $include=set("ts", "id.orig_h")]; - Log::add_filter(Conn::LOG, filter); + # Create the stream. This adds a default filter automatically. + Log::create_stream(Foo::LOG, [$columns=Info, $path="foo"]); } -Note the fields that are set for the filter: - - ``name`` - A mandatory name for the filter that can later be used - to manipulate it further. - - ``path`` - The filename for the output file, without any extension (which - may be automatically added by the writer). Default path values - are generated by taking the stream's ID and munging it slightly. - :bro:enum:`Conn::LOG` is converted into ``conn``, - :bro:enum:`PacketFilter::LOG` is converted into - ``packet_filter``, and :bro:enum:`Known::CERTS_LOG` is - converted into ``known_certs``. - - ``include`` - A set limiting the fields to the ones given. The names - correspond to those in the :bro:type:`Conn::Info` record, with - sub-records unrolled by concatenating fields (separated with - dots). - -Using the code above, you will now get a new log file ``origs.log`` -that looks like this:: - - #separator \x09 - #path origs - #fields ts id.orig_h - #types time addr - 1128727430.350788 141.42.64.125 - 1128727435.450898 141.42.64.125 - -If you want to make this the only log file for the stream, you can -remove the default filter (which, conveniently, has the name -``default``): +At this point, the only thing missing is a call to the :bro:id:`Log::write` +function to send data to the logging framework. The actual event handler +where this should take place will depend on where your data becomes available. +In this example, the connection_established event provides our data, and we +also store a copy of the data being logged into the :bro:type:`connection` +record: .. code:: bro - event bro_init() + event connection_established(c: connection) { - # Remove the filter called "default". - Log::remove_filter(Conn::LOG, "default"); + local rec: Foo::Info = [$ts=network_time(), $id=c$id]; + + # Store a copy of the data in the connection record so other + # event handlers can access it. + c$foo = rec; + + Log::write(Foo::LOG, rec); } -An alternate approach to "turning off" a log is to completely disable -the stream: +When you are developing scripts that add data to the :bro:type:`connection` +record, care must be given to when and how long data is stored. +Normally data saved to the connection record will remain there for the +duration of the connection and from a practical perspective it's not +uncommon to need to delete that data before the end of the connection. -.. code:: bro - event bro_init() - { - Log::disable_stream(Conn::LOG); - } +Add Fields to a Log +------------------- -If you want to skip only some fields but keep the rest, there is a -corresponding ``exclude`` filter attribute that you can use instead of -``include`` to list only the ones you are not interested in. +You can add additional fields to a log by extending the record +type that defines its content, and setting a value for the new fields +before each log record is written. -A filter can also determine output paths *dynamically* based on the -record being logged. That allows, e.g., to record local and remote -connections into separate files. To do this, you define a function -that returns the desired path: - -.. code:: bro - - function split_log(id: Log::ID, path: string, rec: Conn::Info) : string - { - # Return "conn-local" if originator is a local IP, otherwise "conn-remote". - local lr = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; - return fmt("%s-%s", path, lr); - } - - event bro_init() - { - local filter: Log::Filter = [$name="conn-split", $path_func=split_log, $include=set("ts", "id.orig_h")]; - Log::add_filter(Conn::LOG, filter); - } - -Running this will now produce two files, ``local.log`` and -``remote.log``, with the corresponding entries. One could extend this -further for example to log information by subnets or even by IP -address. Be careful, however, as it is easy to create many files very -quickly ... - -.. sidebar:: A More Generic Path Function - - The ``split_log`` method has one draw-back: it can be used - only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded - into its argument list. However, Bro allows to do a more generic - variant: - - .. code:: bro - - function split_log(id: Log::ID, path: string, rec: record { id: conn_id; } ) : string - { - return Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; - } - - This function can be used with all log streams that have records - containing an ``id: conn_id`` field. - -While so far we have seen how to customize the columns being logged, -you can also control which records are written out by providing a -predicate that will be called for each log record: - -.. code:: bro - - function http_only(rec: Conn::Info) : bool - { - # Record only connections with successfully analyzed HTTP traffic - return rec$service == "http"; - } - - event bro_init() - { - local filter: Log::Filter = [$name="http-only", $path="conn-http", $pred=http_only]; - Log::add_filter(Conn::LOG, filter); - } - -This will result in a log file ``conn-http.log`` that contains only -traffic detected and analyzed as HTTP traffic. - -Extending ---------- - -You can add further fields to a log stream by extending the record -type that defines its content. Let's say we want to add a boolean -field ``is_private`` to :bro:type:`Conn::Info` that indicates whether the -originator IP address is part of the :rfc:`1918` space: +Let's say we want to add a boolean field ``is_private`` to +:bro:type:`Conn::Info` that indicates whether the originator IP address +is part of the :rfc:`1918` space: .. code:: bro @@ -218,9 +151,21 @@ originator IP address is part of the :rfc:`1918` space: is_private: bool &default=F &log; }; +As this example shows, when extending a log stream's "Info" record, each +new field must always be declared either with a ``&default`` value or +as ``&optional``. Furthermore, you need to add the ``&log`` attribute +or otherwise the field won't appear in the log file. -Now we need to set the field. A connection's summary is generated at -the time its state is removed from memory. We can add another handler +Now we need to set the field. Although the details vary depending on which +log is being extended, in general it is important to choose a suitable event +in which to set the additional fields because we need to make sure that +the fields are set before the log record is written. Sometimes the right +choice is the same event which writes the log record, but at a higher +priority (in order to ensure that the event handler that sets the additional +fields is executed before the event handler that writes the log record). + +In this example, since a connection's summary is generated at +the time its state is removed from memory, we can add another handler at that time that sets our field correctly: .. code:: bro @@ -232,31 +177,56 @@ at that time that sets our field correctly: } Now ``conn.log`` will show a new field ``is_private`` of type -``bool``. +``bool``. If you look at the Bro script which defines the connection +log stream :doc:`/scripts/base/protocols/conn/main.bro`, you will see +that the connection log record is written in an event handler for the +same event as used in this example to set the additional fields, but at a +lower priority than the one used in this example. -Notes: +For extending logs this way, one needs a bit of knowledge about how +the script that creates the log stream is organizing its state +keeping. Most of the standard Bro scripts attach their log state to +the :bro:type:`connection` record where it can then be accessed, just +like ``c$conn`` above. For example, the HTTP analysis adds a field +``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection` +record. -- For extending logs this way, one needs a bit of knowledge about how - the script that creates the log stream is organizing its state - keeping. Most of the standard Bro scripts attach their log state to - the :bro:type:`connection` record where it can then be accessed, just - as the ``c$conn`` above. For example, the HTTP analysis adds a field - ``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection` - record. See the script reference for more information. -- When extending records as shown above, the new fields must always be - declared either with a ``&default`` value or as ``&optional``. - Furthermore, you need to add the ``&log`` attribute or otherwise the - field won't appear in the output. - -Hooking into the Logging ------------------------- +Define a Logging Event +---------------------- Sometimes it is helpful to do additional analysis of the information being logged. For these cases, a stream can specify an event that will -be generated every time a log record is written to it. All of Bro's -default log streams define such an event. For example, the connection -log stream raises the event :bro:id:`Conn::log_conn`. You +be generated every time a log record is written to it. To do this, we +need to modify the example module shown above to look something like this: + +.. code:: bro + + module Foo; + + export { + redef enum Log::ID += { LOG }; + + type Info: record { + ts: time &log; + id: conn_id &log; + }; + + # Define a logging event. By convention, this is called + # "log_". + global log_foo: event(rec: Info); + } + + event bro_init() &priority=5 + { + # Here we specify both the "Info" record type and the "log_foo" + # event. + Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo, + $path="foo"]); + } + +All of Bro's default log streams define such an event. For example, the +connection log stream raises the event :bro:id:`Conn::log_conn`. You could use that for example for flagging when a connection to a specific destination exceeds a certain duration: @@ -281,15 +251,193 @@ externally with Perl scripts. Much of what such an external script would do later offline, one may instead do directly inside of Bro in real-time. -Rotation --------- +Disable a Stream +---------------- -By default, no log rotation occurs, but it's globally controllable for all -filters by redefining the :bro:id:`Log::default_rotation_interval` option: +One way to "turn off" a log is to completely disable the stream. For +example, the following example will prevent the conn.log from being written: .. code:: bro - redef Log::default_rotation_interval = 1 hr; + event bro_init() + { + Log::disable_stream(Conn::LOG); + } + +Note that this must run after the stream is created, so the priority +of this event handler must be lower than the priority of the event handler +where the stream was created. + + +Filters +======= + +A stream has one or more filters attached to it (a stream without any filters +will not produce any log output). When a stream is created, it automatically +gets a default filter attached to it. This default filter can be removed +or replaced, or other filters can be added to the stream. This is accomplished +by using either the :bro:id:`Log::add_filter` or :bro:id:`Log::remove_filter` +function. This section shows how to use filters to do such tasks as +rename a log file, split the output into multiple files, control which +records are written, and set a custom rotation interval. + +Rename Log File +--------------- + +Normally, the log filename for a given log stream is determined when the +stream is created, unless you explicitly specify a different one by adding +a filter. + +The easiest way to change a log filename is to simply replace the +default log filter with a new filter that specifies a value for the "path" +field. In this example, "conn.log" will be changed to "myconn.log": + +.. code:: bro + + event bro_init() + { + # Replace default filter for the Conn::LOG stream in order to + # change the log filename. + + local f = Log::get_filter(Conn::LOG, "default"); + f$path = "myconn"; + Log::add_filter(Conn::LOG, f); + } + +Keep in mind that the "path" field of a log filter never contains the +filename extension. The extension will be determined later by the log writer. + +Add a New Log File +------------------ + +Normally, a log stream writes to only one log file. However, you can +add filters so that the stream writes to multiple files. This is useful +if you want to restrict the set of fields being logged to the new file. + +In this example, a new filter is added to the Conn::LOG stream that writes +two fields to a new log file: + +.. code:: bro + + event bro_init() + { + # Add a new filter to the Conn::LOG stream that logs only + # timestamp and originator address. + + local filter: Log::Filter = [$name="orig-only", $path="origs", + $include=set("ts", "id.orig_h")]; + Log::add_filter(Conn::LOG, filter); + } + + +Notice how the "include" filter attribute specifies a set that limits the +fields to the ones given. The names correspond to those in the +:bro:type:`Conn::Info` record (however, because the "id" field is itself a +record, we can specify an individual field of "id" by the dot notation +shown in the example). + +Using the code above, in addition to the regular ``conn.log``, you will +now also get a new log file ``origs.log`` that looks like the regular +``conn.log``, but will have only the fields specified in the "include" +filter attribute. + +If you want to skip only some fields but keep the rest, there is a +corresponding ``exclude`` filter attribute that you can use instead of +``include`` to list only the ones you are not interested in. + +If you want to make this the only log file for the stream, you can +remove the default filter: + +.. code:: bro + + event bro_init() + { + # Remove the filter called "default". + Log::remove_filter(Conn::LOG, "default"); + } + +Determine Log Path Dynamically +------------------------------ + +Instead of using the "path" filter attribute, a filter can determine +output paths *dynamically* based on the record being logged. That +allows, e.g., to record local and remote connections into separate +files. To do this, you define a function that returns the desired path, +and use the "path_func" filter attribute: + +.. code:: bro + + function myfunc(id: Log::ID, path: string, rec: Conn::Info) : string + { + # Return "conn-local" if originator is a local IP, otherwise + # return "conn-remote". + + local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; + return fmt("%s-%s", path, r); + } + + event bro_init() + { + local filter: Log::Filter = [$name="conn-split", + $path_func=myfunc, $include=set("ts", "id.orig_h")]; + Log::add_filter(Conn::LOG, filter); + } + +Running this will now produce two new files, ``conn-local.log`` and +``conn-remote.log``, with the corresponding entries. One could extend this +further for example to log information by subnets or even by IP +address. Be careful, however, as it is easy to create many files very +quickly. + +The ``myfunc`` function has one drawback: it can be used +only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded +into its argument list. However, Bro allows to do a more generic +variant: + +.. code:: bro + + function myfunc(id: Log::ID, path: string, + rec: record { id: conn_id; } ) : string + { + local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; + return fmt("%s-%s", path, r); + } + +This function can be used with all log streams that have records +containing an ``id: conn_id`` field. + +Filter Log Records +------------------ + +We have seen how to customize the columns being logged, but +you can also control which records are written out by providing a +predicate that will be called for each log record: + +.. code:: bro + + function http_only(rec: Conn::Info) : bool + { + # Record only connections with successfully analyzed HTTP traffic + return rec$service == "http"; + } + + event bro_init() + { + local filter: Log::Filter = [$name="http-only", $path="conn-http", + $pred=http_only]; + Log::add_filter(Conn::LOG, filter); + } + +This will result in a new log file ``conn-http.log`` that contains only +traffic detected and analyzed as HTTP traffic. + +Rotation +-------- + +The log rotation interval is globally controllable for all +filters by redefining the :bro:id:`Log::default_rotation_interval` option +(note that when using BroControl, this option is set automatically via +the BroControl configuration). Or specifically for certain :bro:type:`Log::Filter` instances by setting their ``interv`` field. Here's an example of changing just the @@ -301,90 +449,62 @@ their ``interv`` field. Here's an example of changing just the { local f = Log::get_filter(Conn::LOG, "default"); f$interv = 1 min; - Log::remove_filter(Conn::LOG, "default"); Log::add_filter(Conn::LOG, f); } -ASCII Writer Configuration --------------------------- +Writers +======= -The ASCII writer has a number of options for customizing the format of -its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`. +Each filter has a writer. If you do not specify a writer when adding a +filter to a stream, then the ASCII writer is the default. -Adding Streams -============== +There are two ways to specify a non-default writer. To change the default +writer for all log filters, just redefine the :bro:id:`Log::default_writer` +option. Alternatively, you can specify the writer to use on a per-filter +basis by setting a value for the filter's "writer" field. Consult the +documentation of the writer to use to see if there are other options that are +needed. -It's easy to create a new log stream for custom scripts. Here's an -example for the ``Foo`` module: +ASCII Writer +------------ + +The ASCII writer has a number of options for customizing the format of its +output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`. +If you change the output format options, then be careful to check whether +your postprocessing scripts can still recognize your log files. + +Some writer options are global (i.e., they affect all log filters using +that log writer). For example, to change the output format of all ASCII +logs to JSON format: .. code:: bro - module Foo; + redef LogAscii::use_json = T; - export { - # Create an ID for our new stream. By convention, this is - # called "LOG". - redef enum Log::ID += { LOG }; +Some writer options are filter-specific (i.e., they affect only the filters +that explicitly specify the option). For example, to change the output +format of the ``conn.log`` only: - # Define the fields. By convention, the type is called "Info". - type Info: record { - ts: time &log; - id: conn_id &log; - }; +.. code:: bro - # Define a hook event. By convention, this is called - # "log_". - global log_foo: event(rec: Info); - - } - - # This event should be handled at a higher priority so that when - # users modify your stream later and they do it at priority 0, - # their code runs after this. - event bro_init() &priority=5 + event bro_init() { - # Create the stream. This also adds a default filter automatically. - Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo, $path="foo"]); + local f = Log::get_filter(Conn::LOG, "default"); + # Use tab-separated-value mode + f$config = table(["tsv"] = "T"); + Log::add_filter(Conn::LOG, f); } -You can also add the state to the :bro:type:`connection` record to make -it easily accessible across event handlers: - -.. code:: bro - - redef record connection += { - foo: Info &optional; - } - -Now you can use the :bro:id:`Log::write` method to output log records and -save the logged ``Foo::Info`` record into the connection record: - -.. code:: bro - - event connection_established(c: connection) - { - local rec: Foo::Info = [$ts=network_time(), $id=c$id]; - c$foo = rec; - Log::write(Foo::LOG, rec); - } - -See the existing scripts for how to work with such a new connection -field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`. - -When you are developing scripts that add data to the :bro:type:`connection` -record, care must be given to when and how long data is stored. -Normally data saved to the connection record will remain there for the -duration of the connection and from a practical perspective it's not -uncommon to need to delete that data before the end of the connection. Other Writers ------------- -Bro supports the following built-in output formats other than ASCII: +Bro supports the following additional built-in output formats: .. toctree:: :maxdepth: 1 logging-input-sqlite -Further formats are available as external plugins. +Additional writers are available as external plugins. + From 24701f26782aa8764edf43d82e0f906adef44363 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 29 May 2015 14:38:50 -0500 Subject: [PATCH 06/15] Fix a "make doc" warning Also fixed some indentation. --- scripts/base/frameworks/logging/main.bro | 2 +- scripts/base/frameworks/logging/writers/sqlite.bro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 1732cc5540..769136e6e6 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -152,7 +152,7 @@ export { ## easy to flood the disk by returning a new string for each ## connection. Upon adding a filter to a stream, if neither ## ``path`` nor ``path_func`` is explicitly set by them, then - ## :bro:see:`default_path_func` is used. + ## :bro:see:`Log::default_path_func` is used. ## ## id: The ID associated with the log stream. ## diff --git a/scripts/base/frameworks/logging/writers/sqlite.bro b/scripts/base/frameworks/logging/writers/sqlite.bro index d73e95ac0a..f7ebd9130c 100644 --- a/scripts/base/frameworks/logging/writers/sqlite.bro +++ b/scripts/base/frameworks/logging/writers/sqlite.bro @@ -19,7 +19,7 @@ export { const unset_field = Log::unset_field &redef; ## String to use for empty fields. This should be different from - ## *unset_field* to make the output unambiguous. + ## *unset_field* to make the output unambiguous. const empty_field = Log::empty_field &redef; } From 260b25f20a5668a7eba9074569dcb0458a226a30 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Sat, 30 May 2015 00:18:04 -0500 Subject: [PATCH 07/15] Fix typos in the "writing bro plugins" doc --- doc/devel/plugins.rst | 62 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst index 5c963a1552..091a0090d1 100644 --- a/doc/devel/plugins.rst +++ b/doc/devel/plugins.rst @@ -3,7 +3,7 @@ Writing Bro Plugins =================== -Bro internally provides plugin API that enables extending +Bro internally provides a plugin API that enables extending the system dynamically, without modifying the core code base. That way custom code remains self-contained and can be maintained, compiled, and installed independently. Currently, plugins can add the following @@ -32,7 +32,7 @@ Quick Start =========== Writing a basic plugin is quite straight-forward as long as one -follows a few conventions. In the following we walk a simple example +follows a few conventions. In the following we create a simple example plugin that adds a new built-in function (bif) to Bro: we'll add ``rot13(s: string) : string``, a function that rotates every character in a string by 13 places. @@ -81,7 +81,7 @@ The syntax of this file is just like any other ``*.bif`` file; we won't go into it here. Now we can already compile our plugin, we just need to tell the -configure script that ``init-plugin`` put in place where the Bro +configure script (that ``init-plugin`` created) where the Bro source tree is located (Bro needs to have been built there first):: # cd rot13-plugin @@ -99,7 +99,7 @@ option:: # export BRO_PLUGIN_PATH=/path/to/rot13-plugin/build # bro -N [...] - Plugin: Demo::Rot13 - (dynamic, version 1) + Demo::Rot13 - (dynamic, version 0.1) [...] That looks quite good, except for the dummy description that we should @@ -108,28 +108,30 @@ is about. We do this by editing the ``config.description`` line in ``src/Plugin.cc``, like this:: [...] - plugin::Configuration Configure() + plugin::Configuration Plugin::Configure() { plugin::Configuration config; config.name = "Demo::Rot13"; config.description = "Caesar cipher rotating a string's characters by 13 places."; - config.version.major = 1; - config.version.minor = 0; + config.version.major = 0; + config.version.minor = 1; return config; } [...] +Now rebuild and verify that the description is visible:: + # make [...] # bro -N | grep Rot13 - Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 0.1) -Better. Bro can also show us what exactly the plugin provides with the +Bro can also show us what exactly the plugin provides with the more verbose option ``-NN``:: # bro -NN [...] - Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 0.1) [Function] Demo::rot13 [...] @@ -157,10 +159,12 @@ The installed version went into ``/lib/bro/plugins/Demo_Rot13``. One can distribute the plugin independently of Bro for others to use. -To distribute in source form, just remove the ``build/`` (``make -distclean`` does that) and then tar up the whole ``rot13-plugin/`` +To distribute in source form, just remove the ``build/`` directory +(``make distclean`` does that) and then tar up the whole ``rot13-plugin/`` directory. Others then follow the same process as above after -unpacking. To distribute the plugin in binary form, the build process +unpacking. + +To distribute the plugin in binary form, the build process conveniently creates a corresponding tarball in ``build/dist/``. In this case, it's called ``Demo_Rot13-0.1.tar.gz``, with the version number coming out of the ``VERSION`` file that ``init-plugin`` put @@ -169,14 +173,14 @@ plugin, but no further source files. Optionally, one can include further files by specifying them in the plugin's ``CMakeLists.txt`` through the ``bro_plugin_dist_files`` macro; the skeleton does that for ``README``, ``VERSION``, ``CHANGES``, and ``COPYING``. To use the -plugin through the binary tarball, just unpack it and point -``BRO_PLUGIN_PATH`` there; or copy it into -``/lib/bro/plugins/`` directly. +plugin through the binary tarball, just unpack it into +``/lib/bro/plugins/``. Alternatively, if you unpack +it in another location, then you need to point ``BRO_PLUGIN_PATH`` there. Before distributing your plugin, you should edit some of the meta files that ``init-plugin`` puts in place. Edit ``README`` and ``VERSION``, and update ``CHANGES`` when you make changes. Also put a -license file in place as ``COPYING``; if BSD is fine, you find a +license file in place as ``COPYING``; if BSD is fine, you will find a template in ``COPYING.edit-me``. Plugin Directory Layout @@ -193,7 +197,7 @@ directory. With the skeleton, ```` corresponds to ``build/``. must exist, and its content must consist of a single line with the qualified name of the plugin (e.g., "Demo::Rot13"). -``/lib/--.so`` +``/lib/.-.so`` The shared library containing the plugin's compiled code. Bro will load this in dynamically at run-time if OS and architecture match the current platform. @@ -215,8 +219,8 @@ directory. With the skeleton, ```` corresponds to ``build/``. Any other files in ```` are ignored by Bro. By convention, a plugin should put its custom scripts into sub folders -of ``scripts/``, i.e., ``scripts//