diff --git a/CHANGES b/CHANGES index 2cda402384..b68469fc5d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,70 @@ +2.0-571 | 2012-05-30 19:12:43 -0700 + + * Updating submodule(s). + +2.0-570 | 2012-05-30 19:08:18 -0700 + + * A new input framework enables scripts to read in external data + dynamically on the fly as Bro is processing network traffic. + (Bernhard Amann) + + Currently, the framework supports reading ASCII input that's + structured similar as Bro's log files as well as raw blobs of + data. Other formats will come in the future. + + See doc/input.rst for more information (this will be extended + further soon). + +2.0-395 | 2012-05-30 17:03:31 -0700 + + * Remove unnecessary assert in ICMP analyzer which could lead to + aborts. Addresses #822. + + * Improve script debugger backtrace and print commands. (Jon Siwek) + + * Switching default DS compression to gzip. (Robin Sommer) + + * Improve availability of IPv6 flow label in connection records. + This adds a "flow_label" field to the "endpoint" record type, + which is used for both the "orig" and "resp" fields of + "connection" records. The new "connection_flow_label_changed" + event also allows tracking of changes in flow labels: it's raised + each time one direction of the connection starts using a different + label. (Jon Siwek) + + * Add unit tests for Broccoli SSL and Broccoli IPv6 connectivity. + (Jon Siwek) + + * Remove AI_ADDRCONFIG getaddrinfo hints flag for listening sockets. + (Jon Siwek) + + * Undo unnecessary communication protocol version bump. (Jon Siwek) + + * Add support to Bro for connecting with peers over IPv6. (Jon Siwek) + + - Communication::listen_ipv6 needs to be redef'd to true in order + for IPv6 listening sockets to be opened. + + - Added Communication::listen_retry option as an interval at which + to retry binding to socket addresses that were already in use. + + - Added some explicit baselines to check in the istate.events and + istate.events-ssl tests -- the SSL test was incorrectly passing + because it compared two empty files. (The files being empty + because "http/base" was given as an argument to Bro which it + couldn't handle because that script doesn't exist anymore). + + - Support for communication over non-global IPv6 addresses. This + usually requires specifying an additional zone identifier (see + RFC 4007). The connect() and listen() BIFs have been changed to + accept this zone identifier as an argument. + + +2.0-377 | 2012-05-24 16:46:06 -0700 + + * Documentation fixes. (Jon Siwek and Daniel Thayer) + 2.0-372 | 2012-05-17 13:59:45 -0700 * Fix compile errors. (Jon Siwek) diff --git a/VERSION b/VERSION index 9dbed189a5..b3387dc11b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-372 +2.0-571 diff --git a/aux/broccoli b/aux/broccoli index 95c93494d7..07866915a1 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 95c93494d7192f69d30f208c4caa3bd38adda6fd +Subproject commit 07866915a1450ddd25b888917f494b4824b0cc3f diff --git a/aux/broctl b/aux/broctl index ba9e1aa2f2..892b60edb9 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit ba9e1aa2f2159deac0cf96863f54405643764df0 +Subproject commit 892b60edb967bb456872638f22ba994e84530137 diff --git a/aux/btest b/aux/btest index e0da8d0e28..4697bf4c80 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit e0da8d0e284bbebbaef711c91c1b961580f225d2 +Subproject commit 4697bf4c8046a3ab7d5e00e926c5db883cb44664 diff --git a/doc/index.rst b/doc/index.rst index ec67b76fd8..ed14be1dd2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -24,6 +24,7 @@ Frameworks notice logging + input cluster signatures diff --git a/doc/input.rst b/doc/input.rst new file mode 100644 index 0000000000..d9fe8aa6b8 --- /dev/null +++ b/doc/input.rst @@ -0,0 +1,183 @@ +===================== +Loading Data into Bro +===================== + +.. rst-class:: opening + + Bro comes with a flexible input interface that allows to read + previously stored data. Data is either read into bro tables or + sent to scripts using events. + This document describes how the input framework can be used. + +.. contents:: + +Terminology +=========== + +Bro's input framework is built around three main abstracts, that are +very similar to the abstracts used in the logging framework: + + Input Streams + An input stream corresponds to a single input source + (usually a textfile). It defined the information necessary + to find the source (e.g. the filename), the reader that it used + to get data from it (see below). + It also defines exactly what data is read from the input source. + There are two different kind of streams, event streams and table + streams. + By default, event streams generate an event for each line read + from the input source. + Table streams on the other hand read the input source in a bro + table for easy later access. + + Readers + A reader defines the input format for the specific input stream. + At the moment, Bro comes with two types of reader. The default reader is READER_ASCII, + which can read the tab seperated ASCII logfiles that were generated by the + logging framework. + READER_RAW can files containing records separated by a character(like e.g. newline) and send + one event per line. + + +Event Streams +============= + +For examples, please look at the unit tests in +``testing/btest/scripts/base/frameworks/input/``. + +Event Streams are streams that generate an event for each line in of the input source. + +For example, a simple stream retrieving the fields ``i`` and ``b`` from an inputSource +could be defined as follows: + +.. code:: bro + + type Val: record { + i: int; + b: bool; + }; + + event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) { + # work with event data + } + + event bro_init { + Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]); + } + +The fields that can be set for an event stream are: + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + ``mode`` + The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``. + Default is ``MANUAL``. + ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not + be reflected in the data bro knows. + ``REREAD`` means that the whole file is read again each time a change is found. This should be used for + files that are mapped to a table where individual lines can change. + ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new + data is added to the file. + + ``name`` + A mandatory name for the stream that can later be used + to remove it. + + ``fields`` + Name of a record type containing the fields, which should be retrieved from + the input stream. + + ``ev`` + The event which is fired, after a line has been read from the input source. + The first argument that is passed to the event is an Input::Event structure, + followed by the data, either inside of a record (if ``want_record is set``) or as + individual fields. + The Input::Event structure can contain information, if the received line is ``NEW``, has + been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information + for event filters, the value is always ``NEW`` at the moment. + + + +Table Streams +============= + +Table streams are the second, more complex type of input streams. + +Table streams store the information they read from an input source in a bro table. For example, +when reading a file that contains ip addresses and connection attemt information one could use +an approach similar to this: + +.. code:: bro + + type Idx: record { + a: addr; + }; + + type Val: record { + tries: count; + }; + + global conn_attempts: table[addr] of count = table(); + + event bro_init { + Input::add_table([$source="input.txt", $name="input", $idx=Idx, $val=Val, $destination=conn_attempts]); + } + +The table conn_attempts will then contain the information about connection attemps. + +The possible fields that can be set for an table stream are: + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + ``mode`` + The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``. + Default is ``MANUAL``. + ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not + be reflected in the data bro knows. + ``REREAD`` means that the whole file is read again each time a change is found. This should be used for + files that are mapped to a table where individual lines can change. + ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new + data is added to the file. + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``idx`` + Record type that defines the index of the table + + ``val`` + Record type that defines the values of the table + + ``want_record`` + Defines if the values of the table should be stored as a record (default), + or as a simple value. Has to be set if Val contains more than one element. + + ``destination`` + The destination table + + ``ev`` + Optional event that is raised, when values are added to, changed in or deleted from the table. + Events are passed an Input::Event description as the first argument, the index record as the second argument + and the values as the third argument. + + ``pred`` + Optional predicate, that can prevent entries from being added to the table and events from being sent. diff --git a/doc/logging-dataseries.rst b/doc/logging-dataseries.rst index b41b9fb0b7..554600f055 100644 --- a/doc/logging-dataseries.rst +++ b/doc/logging-dataseries.rst @@ -6,8 +6,8 @@ Binary Output with DataSeries .. rst-class:: opening Bro's default ASCII log format is not exactly the most efficient - way for storing large volumes of data. An an alternative, Bro comes - with experimental support for `DataSeries + way for storing and searching large volumes of data. An an + alternative, Bro comes with experimental support for `DataSeries `_ output, an efficient binary format for recording structured bulk data. DataSeries is developed and maintained at HP Labs. @@ -35,9 +35,12 @@ To build and install the two into ````, do:: Please refer to the packages' documentation for more information about the installation process. In particular, there's more information on required and optional `dependencies for Lintel -`_ +`_ and `dependencies for DataSeries -`_ +`_. +For users on RedHat-style systems, you'll need the following:: + + yum install libxml2-devel boost-devel Compiling Bro with DataSeries Support ------------------------------------- @@ -166,3 +169,18 @@ with the output files. The ``man`` pages for these tool show further options, and their ``-h`` option gives some more information (either can be a bit cryptic unfortunately though). + +Deficiencies +------------ + +Due to limitations of the DataSeries format, one cannot inspect its +files before they have been fully written. In other words, when using +DataSeries, it's currently it's not possible to inspect the live log +files inside the spool directory before they are rotated to their +final location. It seems that this could be fixed with some effort, +and we will work with DataSeries development team on that if the +format gains traction among Bro users. + +Likewise, we're considering writing custom command line tools for +interacting with DataSeries files, making that a bit more convenient +than what the standard utilities provide. diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 521113ddd2..433d7edd0a 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal) rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro) @@ -31,6 +32,10 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro) rest_target(${psd} base/frameworks/communication/main.bro) rest_target(${psd} base/frameworks/control/main.bro) rest_target(${psd} base/frameworks/dpd/main.bro) +rest_target(${psd} base/frameworks/input/main.bro) +rest_target(${psd} base/frameworks/input/readers/ascii.bro) +rest_target(${psd} base/frameworks/input/readers/benchmark.bro) +rest_target(${psd} base/frameworks/input/readers/raw.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 1e89e9b2a7..766dea912f 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -77,6 +77,9 @@ export { node_type: NodeType; ## The IP address of the cluster node. ip: addr; + ## If the *ip* field is a non-global IPv6 address, this field + ## can specify a particular :rfc:`4007` ``zone_id``. + zone_id: string &default=""; ## The port to which the this local node can connect when ## establishing communication. p: port; diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index b5a0d25e1f..4576f5b913 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -19,23 +19,26 @@ event bro_init() &priority=9 # Connections from the control node for runtime control and update events. # Every node in a cluster is eligible for control from this host. if ( n$node_type == CONTROL ) - Communication::nodes["control"] = [$host=n$ip, $connect=F, - $class="control", $events=control_events]; + Communication::nodes["control"] = [$host=n$ip, $zone_id=n$zone_id, + $connect=F, $class="control", + $events=control_events]; if ( me$node_type == MANAGER ) { if ( n$node_type == WORKER && n$manager == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, + [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, $events=worker2manager_events, $request_logs=T]; if ( n$node_type == PROXY && n$manager == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, + [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, $events=proxy2manager_events, $request_logs=T]; if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i ) - Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p, + Communication::nodes["time-machine"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, + $p=nodes[i]$p, $connect=T, $retry=1min, $events=tm2manager_events]; } @@ -44,7 +47,8 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, + $sync=T, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) @@ -53,16 +57,18 @@ event bro_init() &priority=9 { if ( n?$proxy ) Communication::nodes[i] - = [$host=n$ip, $p=n$p, + = [$host=n$ip, $zone_id=n$zone_id, $p=n$p, $connect=T, $auth=F, $sync=T, $retry=1mins]; else if ( me?$proxy && me$proxy == i ) Communication::nodes[me$proxy] - = [$host=nodes[i]$ip, $connect=F, $auth=T, $sync=T]; + = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id, + $connect=F, $auth=T, $sync=T]; } # Finally the manager, to send it status updates. if ( n$node_type == MANAGER && me$manager == i ) Communication::nodes["manager"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1mins, $class=node, @@ -72,6 +78,7 @@ event bro_init() &priority=9 { if ( n$node_type == MANAGER && me$manager == i ) Communication::nodes["manager"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1mins, $class=node, @@ -79,6 +86,7 @@ event bro_init() &priority=9 if ( n$node_type == PROXY && me$proxy == i ) Communication::nodes["proxy"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1mins, $sync=T, $class=node, @@ -87,6 +95,7 @@ event bro_init() &priority=9 if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i ) Communication::nodes["time-machine"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1min, diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 04772f57aa..b9b15bfd22 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -2,6 +2,7 @@ ##! and/or transfer events. @load base/frameworks/packet-filter +@load base/utils/addrs module Communication; @@ -10,7 +11,7 @@ export { ## The communication logging stream identifier. redef enum Log::ID += { LOG }; - ## Which interface to listen on (0.0.0.0 for any interface). + ## Which interface to listen on (``0.0.0.0`` or ``[::]`` are wildcards). const listen_interface = 0.0.0.0 &redef; ## Which port to listen on. @@ -19,6 +20,19 @@ export { ## This defines if a listening socket should use SSL. const listen_ssl = F &redef; + ## Defines if a listening socket can bind to IPv6 addresses. + const listen_ipv6 = F &redef; + + ## If :bro:id:`Communication::listen_interface` is a non-global + ## IPv6 address and requires a specific :rfc:`4007` ``zone_id``, + ## it can be specified here. + const listen_ipv6_zone_id = "" &redef; + + ## Defines the interval at which to retry binding to + ## :bro:id:`Communication::listen_interface` on + ## :bro:id:`Communication::listen_port` if it's already in use. + const listen_retry = 30 secs &redef; + ## Default compression level. Compression level is 0-9, with 0 = no ## compression. global compression_level = 0 &redef; @@ -51,6 +65,10 @@ export { type Node: record { ## Remote address. host: addr; + + ## If the *host* field is a non-global IPv6 address, this field + ## can specify a particular :rfc:`4007` ``zone_id``. + zone_id: string &optional; ## Port of the remote Bro communication endpoint if we are initiating ## the connection based on the :bro:id:`connect` field. @@ -160,7 +178,7 @@ event remote_log(level: count, src: count, msg: string) # This is a core generated event. event remote_log_peer(p: event_peer, level: count, src: count, msg: string) { - local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg); + local rmsg = fmt("[#%d/%s:%d] %s", p$id, addr_to_uri(p$host), p$p, msg); do_script_log_common(level, src, rmsg); } @@ -178,7 +196,8 @@ function connect_peer(peer: string) p = node$p; local class = node?$class ? node$class : ""; - local id = connect(node$host, p, class, node$retry, node$ssl); + local zone_id = node?$zone_id ? node$zone_id : ""; + local id = connect(node$host, zone_id, p, class, node$retry, node$ssl); if ( id == PEER_ID_NONE ) Log::write(Communication::LOG, [$ts = network_time(), diff --git a/scripts/base/frameworks/control/main.bro b/scripts/base/frameworks/control/main.bro index 4fe8872801..63e5f639a0 100644 --- a/scripts/base/frameworks/control/main.bro +++ b/scripts/base/frameworks/control/main.bro @@ -11,6 +11,10 @@ export { ## The port of the host that will be controlled. const host_port = 0/tcp &redef; + ## If :bro:id:`Control::host` is a non-global IPv6 address and + ## requires a specific :rfc:`4007` ``zone_id``, it can be set here. + const zone_id = "" &redef; + ## The command that is being done. It's typically set on the ## command line. const cmd = "" &redef; diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro new file mode 100644 index 0000000000..0e7d8ffb73 --- /dev/null +++ b/scripts/base/frameworks/input/__load__.bro @@ -0,0 +1,5 @@ +@load ./main +@load ./readers/ascii +@load ./readers/raw +@load ./readers/benchmark + diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro new file mode 100644 index 0000000000..f5df72473f --- /dev/null +++ b/scripts/base/frameworks/input/main.bro @@ -0,0 +1,139 @@ +##! The input framework provides a way to read previously stored data either +##! as an event stream or into a bro table. + +module Input; + +export { + + ## The default input reader used. Defaults to `READER_ASCII`. + const default_reader = READER_ASCII &redef; + + const default_mode = MANUAL &redef; + + ## TableFilter description type used for the `table` method. + type TableDescription: record { + ## Common definitions for tables and events + + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. + source: string; + + ## Reader to use for this stream + reader: Reader &default=default_reader; + + ## Read mode to use for this stream + mode: Mode &default=default_mode; + + ## Descriptive name. Used to remove a stream at a later time + name: string; + + # Special definitions for tables + + ## Table which will receive the data read by the input framework + destination: any; + + ## Record that defines the values used as the index of the table + idx: any; + + ## Record that defines the values used as the elements of the table + ## If val is undefined, destination has to be a set. + val: any &optional; + + ## Defines if the value of the table is a record (default), or a single value. Val + ## can only contain one element when this is set to false. + want_record: bool &default=T; + + ## The event that is raised each time a value is added to, changed in or removed + ## from the table. The event will receive an Input::Event enum as the first + ## argument, the idx record as the second argument and the value (record) as the + ## third argument. + ev: any &optional; # event containing idx, val as values. + + ## Predicate function that can decide if an insertion, update or removal should + ## really be executed. Parameters are the same as for the event. If true is + ## returned, the update is performed. If false is returned, it is skipped. + pred: function(typ: Input::Event, left: any, right: any): bool &optional; + }; + + ## EventFilter description type used for the `event` method. + type EventDescription: record { + ## Common definitions for tables and events + + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. + source: string; + + ## Reader to use for this steam + reader: Reader &default=default_reader; + + ## Read mode to use for this stream + mode: Mode &default=default_mode; + + ## Descriptive name. Used to remove a stream at a later time + name: string; + + # Special definitions for events + + ## Record describing the fields to be retrieved from the source input. + fields: any; + + ## If want_record if false (default), the event receives each value in fields as a seperate argument. + ## If it is set to true, the event receives all fields in a signle record value. + want_record: bool &default=F; + + ## The event that is rised each time a new line is received from the reader. + ## The event will receive an Input::Event enum as the first element, and the fields as the following arguments. + ev: any; + + }; + + ## Create a new table input from a given source. Returns true on success. + ## + ## description: `TableDescription` record describing the source. + global add_table: function(description: Input::TableDescription) : bool; + + ## Create a new event input from a given source. Returns true on success. + ## + ## description: `TableDescription` record describing the source. + global add_event: function(description: Input::EventDescription) : bool; + + ## Remove a input stream. Returns true on success and false if the named stream was not found. + ## + ## id: string value identifying the stream to be removed + global remove: function(id: string) : bool; + + ## Forces the current input to be checked for changes. + ## Returns true on success and false if the named stream was not found + ## + ## id: string value identifying the stream + global force_update: function(id: string) : bool; + + ## Event that is called, when the update of a specific source is finished + global update_finished: event(name: string, source:string); +} + +@load base/input.bif + + +module Input; + +function add_table(description: Input::TableDescription) : bool + { + return __create_table_stream(description); + } + +function add_event(description: Input::EventDescription) : bool + { + return __create_event_stream(description); + } + +function remove(id: string) : bool + { + return __remove_stream(id); + } + +function force_update(id: string) : bool + { + return __force_update(id); + } + diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro new file mode 100644 index 0000000000..7fca1ad795 --- /dev/null +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -0,0 +1,21 @@ +##! Interface for the ascii input reader. +##! +##! The defaults are set to match Bro's ASCII output. + +module InputAscii; + +export { + ## Separator between fields. + ## Please note that the separator has to be exactly one character long + const separator = "\t" &redef; + + ## Separator between set elements. + ## Please note that the separator has to be exactly one character long + const set_separator = "," &redef; + + ## String to use for empty fields. + const empty_field = "(empty)" &redef; + + ## String to use for an unset &optional field. + const unset_field = "-" &redef; +} diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro new file mode 100644 index 0000000000..fe44914271 --- /dev/null +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -0,0 +1,23 @@ +##! Interface for the ascii input reader. + +module InputBenchmark; + +export { + ## multiplication factor for each second + const factor = 1.0 &redef; + + ## spread factor between lines + const spread = 0 &redef; + + ## spreading where usleep = 1000000 / autospread * num_lines + const autospread = 0.0 &redef; + + ## addition factor for each heartbeat + const addfactor = 0 &redef; + + ## stop spreading at x lines per heartbeat + const stopspreadat = 0 &redef; + + ## 1 -> enable timed spreading + const timedspread = 0.0 &redef; +} diff --git a/scripts/base/frameworks/input/readers/raw.bro b/scripts/base/frameworks/input/readers/raw.bro new file mode 100644 index 0000000000..45deed3eda --- /dev/null +++ b/scripts/base/frameworks/input/readers/raw.bro @@ -0,0 +1,9 @@ +##! Interface for the raw input reader. + +module InputRaw; + +export { + ## Separator between input records. + ## Please note that the separator has to be exactly one character long + const record_separator = "\n" &redef; +} diff --git a/scripts/base/frameworks/logging/writers/dataseries.bro b/scripts/base/frameworks/logging/writers/dataseries.bro index ccee500c3a..e85d9c8c49 100644 --- a/scripts/base/frameworks/logging/writers/dataseries.bro +++ b/scripts/base/frameworks/logging/writers/dataseries.bro @@ -10,7 +10,7 @@ export { ## 'lzo' -- LZO compression. Very fast decompression times. ## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output. ## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output. - const compression = "lzo" &redef; + const compression = "gz" &redef; ## The extent buffer size. ## Larger values here lead to better compression and more efficient writes, but diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 060d36aad4..920f4a47c2 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -204,9 +204,9 @@ module GLOBAL; ## directly and then remove this alias. type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn; -## Statistics about an endpoint. +## Statistics about a :bro:type:`connection` endpoint. ## -## todo::Where is this used? +## .. bro:see:: connection type endpoint: record { size: count; ##< Logical size of data sent (for TCP: derived from sequence numbers). ## Endpoint state. For TCP connection, one of the constants: @@ -220,6 +220,9 @@ type endpoint: record { ## Number of IP-level bytes sent. Only set if :bro:id:`use_conn_size_analyzer` is ## true. num_bytes_ip: count &optional; + ## The current IPv6 flow label that the connection endpoint is using. + ## Always 0 if the connection is over IPv4. + flow_label: count; }; ## A connection. This is Bro's basic connection type describing IP- and @@ -245,7 +248,7 @@ type connection: record { service: set[string]; addl: string; ##< Deprecated. hot: count; ##< Deprecated. - history: string; ##< State history of TCP connections. See *history* in :bro:see:`Conn::Info`. + history: string; ##< State history of connections. See *history* in :bro:see:`Conn::Info`. ## A globally unique connection identifier. For each connection, Bro creates an ID ## that is very likely unique across independent Bro runs. These IDs can thus be ## used to tag and locate information associated with that connection. @@ -986,7 +989,7 @@ const IPPROTO_MOBILITY = 135; ##< IPv6 mobility header. ## Values extracted from an IPv6 extension header's (e.g. hop-by-hop or ## destination option headers) option field. ## -## .. bro:see:: ip6_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts +## .. bro:see:: ip6_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts type ip6_option: record { otype: count; ##< Option type. len: count; ##< Option data length. @@ -995,7 +998,7 @@ type ip6_option: record { ## Values extracted from an IPv6 Hop-by-Hop options extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option type ip6_hopopts: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1008,7 +1011,7 @@ type ip6_hopopts: record { ## Values extracted from an IPv6 Destination options extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option type ip6_dstopts: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1021,7 +1024,7 @@ type ip6_dstopts: record { ## Values extracted from an IPv6 Routing extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_routing: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1038,7 +1041,7 @@ type ip6_routing: record { ## Values extracted from an IPv6 Fragment extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_fragment: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1057,7 +1060,7 @@ type ip6_fragment: record { ## Values extracted from an IPv6 Authentication extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_ah: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1076,7 +1079,7 @@ type ip6_ah: record { ## Values extracted from an IPv6 ESP extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_esp: record { ## Security Parameters Index. spi: count; @@ -1086,7 +1089,7 @@ type ip6_esp: record { ## Values extracted from an IPv6 Mobility Binding Refresh Request message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_brr: record { ## Reserved. rsv: count; @@ -1096,7 +1099,7 @@ type ip6_mobility_brr: record { ## Values extracted from an IPv6 Mobility Home Test Init message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_hoti: record { ## Reserved. rsv: count; @@ -1108,7 +1111,7 @@ type ip6_mobility_hoti: record { ## Values extracted from an IPv6 Mobility Care-of Test Init message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_coti: record { ## Reserved. rsv: count; @@ -1120,7 +1123,7 @@ type ip6_mobility_coti: record { ## Values extracted from an IPv6 Mobility Home Test message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_hot: record { ## Home Nonce Index. nonce_idx: count; @@ -1134,7 +1137,7 @@ type ip6_mobility_hot: record { ## Values extracted from an IPv6 Mobility Care-of Test message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_cot: record { ## Care-of Nonce Index. nonce_idx: count; @@ -1148,7 +1151,7 @@ type ip6_mobility_cot: record { ## Values extracted from an IPv6 Mobility Binding Update message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_bu: record { ## Sequence number. seq: count; @@ -1168,7 +1171,7 @@ type ip6_mobility_bu: record { ## Values extracted from an IPv6 Mobility Binding Acknowledgement message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_back: record { ## Status. status: count; @@ -1184,7 +1187,7 @@ type ip6_mobility_back: record { ## Values extracted from an IPv6 Mobility Binding Error message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_be: record { ## Status. status: count; @@ -1196,7 +1199,7 @@ type ip6_mobility_be: record { ## Values extracted from an IPv6 Mobility header's message data. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr type ip6_mobility_msg: record { ## The type of message from the header's MH Type field. id: count; @@ -1220,7 +1223,7 @@ type ip6_mobility_msg: record { ## Values extracted from an IPv6 Mobility header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_mobility_hdr: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1263,7 +1266,7 @@ type ip6_ext_hdr: record { ## Values extracted from an IPv6 header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts +## .. bro:see:: pkt_hdr ip4_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts ## ip6_routing ip6_fragment ip6_ah ip6_esp type ip6_hdr: record { class: count; ##< Traffic class. @@ -1821,6 +1824,14 @@ export { }; } # end export +module Threading; + +export { + ## The heartbeat interval used by the threading framework. + ## Changing this should usually not be neccessary and will break several tests. + const heartbeat_interval = 1.0 secs &redef; +} + module GLOBAL; ## An NTP message. @@ -2650,3 +2661,6 @@ const snaplen = 8192 &redef; # Load the logging framework here because it uses fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging + +@load base/frameworks/input + diff --git a/scripts/base/protocols/conn/main.bro b/scripts/base/protocols/conn/main.bro index 03e2cfaf71..432bb12e84 100644 --- a/scripts/base/protocols/conn/main.bro +++ b/scripts/base/protocols/conn/main.bro @@ -68,7 +68,7 @@ export { missed_bytes: count &log &default=0; ## Records the state history of connections as a string of letters. - ## For TCP connections the meaning of those letters is: + ## The meaning of those letters is: ## ## ====== ==================================================== ## Letter Meaning diff --git a/scripts/policy/frameworks/communication/listen.bro b/scripts/policy/frameworks/communication/listen.bro index e366e5b4ff..111bc64a23 100644 --- a/scripts/policy/frameworks/communication/listen.bro +++ b/scripts/policy/frameworks/communication/listen.bro @@ -8,5 +8,6 @@ module Communication; event bro_init() &priority=-10 { enable_communication(); - listen(listen_interface, listen_port, listen_ssl); + listen(listen_interface, listen_port, listen_ssl, listen_ipv6, + listen_ipv6_zone_id, listen_retry); } diff --git a/scripts/policy/frameworks/control/controller.bro b/scripts/policy/frameworks/control/controller.bro index 39647095db..22b19bf973 100644 --- a/scripts/policy/frameworks/control/controller.bro +++ b/scripts/policy/frameworks/control/controller.bro @@ -25,8 +25,8 @@ event bro_init() &priority=5 # Establish the communication configuration and only request response # messages. - Communication::nodes["control"] = [$host=host, $p=host_port, - $sync=F, $connect=T, + Communication::nodes["control"] = [$host=host, $zone_id=zone_id, + $p=host_port, $sync=F, $connect=T, $class="control", $events=Control::controllee_events]; } diff --git a/src/Attr.cc b/src/Attr.cc index 82d9c9ddc7..2e4e090c0b 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -17,7 +17,8 @@ const char* attr_name(attr_tag t) "&persistent", "&synchronized", "&postprocessor", "&encrypt", "&match", "&disable_print_hook", "&raw_output", "&mergeable", "&priority", - "&group", "&log", "&error_handler", "(&tracked)", + "&group", "&log", "&error_handler", "&type_column", + "(&tracked)", }; return attr_names[int(t)]; @@ -420,6 +421,25 @@ void Attributes::CheckAttr(Attr* a) Error("&log applied to a type that cannot be logged"); break; + case ATTR_TYPE_COLUMN: + { + if ( type->Tag() != TYPE_PORT ) + { + Error("type_column tag only applicable to ports"); + break; + } + + BroType* atype = a->AttrExpr()->Type(); + + if ( atype->Tag() != TYPE_STRING ) { + Error("type column needs to have a string argument"); + break; + } + + break; + } + + default: BadTag("Attributes::CheckAttr", attr_name(a->Tag())); } diff --git a/src/Attr.h b/src/Attr.h index 6c835dc61c..e6b09cf96b 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -35,6 +35,7 @@ typedef enum { ATTR_GROUP, ATTR_LOG, ATTR_ERROR_HANDLER, + ATTR_TYPE_COLUMN, // for input framework ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry #define NUM_ATTRS (int(ATTR_TRACKED) + 1) } attr_tag; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7d74aee1ce..0a6c2a5c76 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -142,6 +142,7 @@ endmacro(GET_BIF_OUTPUT_FILES) set(BIF_SRCS bro.bif logging.bif + input.bif event.bif const.bif types.bif @@ -429,6 +430,13 @@ set(bro_SRCS logging/writers/DataSeries.cc logging/writers/None.cc + input/Manager.cc + input/ReaderBackend.cc + input/ReaderFrontend.cc + input/readers/Ascii.cc + input/readers/Raw.cc + input/readers/Benchmark.cc + nb_dns.c digest.h ) diff --git a/src/Conn.cc b/src/Conn.cc index 0e34903bed..53abcc26eb 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -113,7 +113,7 @@ unsigned int Connection::external_connections = 0; IMPLEMENT_SERIAL(Connection, SER_CONNECTION); Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, - const Encapsulation& arg_encap) + uint32 flow, const Encapsulation& arg_encap) { sessions = s; key = k; @@ -124,6 +124,10 @@ Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, orig_port = id->src_port; resp_port = id->dst_port; proto = TRANSPORT_UNKNOWN; + orig_flow_label = flow; + resp_flow_label = 0; + saw_first_orig_packet = 1; + saw_first_resp_packet = 0; conn_val = 0; login_conn = 0; @@ -327,10 +331,12 @@ RecordVal* Connection::BuildConnVal() RecordVal *orig_endp = new RecordVal(endpoint); orig_endp->Assign(0, new Val(0, TYPE_COUNT)); orig_endp->Assign(1, new Val(0, TYPE_COUNT)); + orig_endp->Assign(4, new Val(orig_flow_label, TYPE_COUNT)); RecordVal *resp_endp = new RecordVal(endpoint); resp_endp->Assign(0, new Val(0, TYPE_COUNT)); resp_endp->Assign(1, new Val(0, TYPE_COUNT)); + resp_endp->Assign(4, new Val(resp_flow_label, TYPE_COUNT)); conn_val->Assign(0, id_val); conn_val->Assign(1, orig_endp); @@ -681,6 +687,14 @@ void Connection::FlipRoles() resp_port = orig_port; orig_port = tmp_port; + bool tmp_bool = saw_first_resp_packet; + saw_first_resp_packet = saw_first_orig_packet; + saw_first_orig_packet = tmp_bool; + + uint32 tmp_flow = resp_flow_label; + resp_flow_label = orig_flow_label; + orig_flow_label = tmp_flow; + Unref(conn_val); conn_val = 0; @@ -888,3 +902,35 @@ void Connection::SetRootAnalyzer(TransportLayerAnalyzer* analyzer, PIA* pia) root_analyzer = analyzer; primary_PIA = pia; } + +void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label) + { + uint32& my_flow_label = is_orig ? orig_flow_label : resp_flow_label; + + if ( my_flow_label != flow_label ) + { + if ( conn_val ) + { + RecordVal *endp = conn_val->Lookup(is_orig ? 1 : 2)->AsRecordVal(); + endp->Assign(4, new Val(flow_label, TYPE_COUNT)); + } + + if ( connection_flow_label_changed && + (is_orig ? saw_first_orig_packet : saw_first_resp_packet) ) + { + val_list* vl = new val_list(4); + vl->append(BuildConnVal()); + vl->append(new Val(is_orig, TYPE_BOOL)); + vl->append(new Val(my_flow_label, TYPE_COUNT)); + vl->append(new Val(flow_label, TYPE_COUNT)); + ConnectionEvent(connection_flow_label_changed, 0, vl); + } + + my_flow_label = flow_label; + } + + if ( is_orig ) + saw_first_orig_packet = 1; + else + saw_first_resp_packet = 1; + } diff --git a/src/Conn.h b/src/Conn.h index d5622efe03..b7911b84fb 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -52,7 +52,7 @@ class Analyzer; class Connection : public BroObj { public: Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, - const Encapsulation& arg_encap); + uint32 flow, const Encapsulation& arg_encap); virtual ~Connection(); void CheckEncapsulation(const Encapsulation& arg_encap) @@ -257,6 +257,8 @@ public: const Encapsulation& GetEncapsulation() const { return encapsulation; } + void CheckFlowLabel(bool is_orig, uint32 flow_label); + protected: Connection() { persistent = 0; } @@ -287,6 +289,7 @@ protected: IPAddr resp_addr; uint32 orig_port, resp_port; // in network order TransportProto proto; + uint32 orig_flow_label, resp_flow_label; // most recent IPv6 flow labels double start_time, last_time; double inactivity_timeout; RecordVal* conn_val; @@ -303,6 +306,7 @@ protected: unsigned int record_packets:1, record_contents:1; unsigned int persistent:1; unsigned int record_current_packet:1, record_current_content:1; + unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1; // Count number of connections. static unsigned int total_connections; diff --git a/src/Debug.cc b/src/Debug.cc index ea9c52f77e..535e193685 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -721,7 +721,6 @@ static char* get_prompt(bool reset_counter = false) string get_context_description(const Stmt* stmt, const Frame* frame) { - char buf[1024]; ODesc d; const BroFunc* func = frame->GetFunction(); @@ -739,10 +738,14 @@ string get_context_description(const Stmt* stmt, const Frame* frame) loc.last_line = 0; } - safe_snprintf(buf, sizeof(buf), "In %s at %s:%d", + size_t buf_size = strlen(d.Description()) + strlen(loc.filename) + 1024; + char* buf = new char[buf_size]; + safe_snprintf(buf, buf_size, "In %s at %s:%d", d.Description(), loc.filename, loc.last_line); - return string(buf); + string retval(buf); + delete [] buf; + return retval; } int dbg_handle_debug_input() @@ -924,6 +927,8 @@ bool post_execute_stmt(Stmt* stmt, Frame* f, Val* result, stmt_flow_type* flow) // Evaluates the given expression in the context of the currently selected // frame. Returns the resulting value, or nil if none (or there was an error). Expr* g_curr_debug_expr = 0; +const char* g_curr_debug_error = 0; +bool in_debug = false; // ### fix this hardwired access to external variables etc. struct yy_buffer_state; @@ -969,6 +974,11 @@ Val* dbg_eval_expr(const char* expr) Val* result = 0; if ( yyparse() ) { + if ( g_curr_debug_error ) + debug_msg("Parsing expression '%s' failed: %s\n", expr, g_curr_debug_error); + else + debug_msg("Parsing expression '%s' failed\n", expr); + if ( g_curr_debug_expr ) { delete g_curr_debug_expr; @@ -983,6 +993,9 @@ Val* dbg_eval_expr(const char* expr) delete g_curr_debug_expr; g_curr_debug_expr = 0; + delete [] g_curr_debug_error; + g_curr_debug_error = 0; + in_debug = false; return result; } diff --git a/src/DebugCmds.cc b/src/DebugCmds.cc index 1d3b9dd220..bfb4d6ecc8 100644 --- a/src/DebugCmds.cc +++ b/src/DebugCmds.cc @@ -553,7 +553,8 @@ int dbg_cmd_print(DebugCmd cmd, const vector& args) for ( int i = 0; i < int(args.size()); ++i ) { expr += args[i]; - expr += " "; + if ( i < int(args.size()) - 1 ) + expr += " "; } Val* val = dbg_eval_expr(expr.c_str()); @@ -566,8 +567,7 @@ int dbg_cmd_print(DebugCmd cmd, const vector& args) } else { - // ### Print something? - // debug_msg("\n"); + debug_msg("\n"); } return 1; diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index c41a0552c6..3394486ff2 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -15,7 +15,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false }, { "threading", 0, false } + { "logging", 0, false }, {"input", 0, false }, + { "threading", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 71e21bfa26..ca422072c5 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -24,6 +24,7 @@ enum DebugStream { DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli DBG_LOGGING, // Logging streams + DBG_INPUT, // Input streams DBG_THREADING, // Threading system NUM_DBGS // Has to be last diff --git a/src/Func.cc b/src/Func.cc index ecb341e3e0..b6fc7f0785 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -522,11 +522,13 @@ void builtin_error(const char* msg, BroObj* arg) #include "bro.bif.func_h" #include "logging.bif.func_h" +#include "input.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" #include "logging.bif.func_def" +#include "input.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -541,6 +543,7 @@ void init_builtin_funcs() #include "bro.bif.func_init" #include "logging.bif.func_init" +#include "input.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/ICMP.cc b/src/ICMP.cc index 05a6b67dff..b06c6440e1 100644 --- a/src/ICMP.cc +++ b/src/ICMP.cc @@ -49,9 +49,7 @@ void ICMP_Analyzer::DeliverPacket(int len, const u_char* data, const struct icmp* icmpp = (const struct icmp*) data; - assert(caplen >= len); // Should have been caught earlier already. - - if ( ! ignore_checksums ) + if ( ! ignore_checksums && caplen >= len ) { int chksum = 0; diff --git a/src/IP.h b/src/IP.h index 502ae857c0..c3a74b4a01 100644 --- a/src/IP.h +++ b/src/IP.h @@ -524,6 +524,12 @@ public: int DF() const { return ip4 ? ((ntohs(ip4->ip_off) & 0x4000) != 0) : 0; } + /** + * Returns value of an IPv6 header's flow label field or 0 if it's IPv4. + */ + uint32 FlowLabel() const + { return ip4 ? 0 : (ntohl(ip6->ip6_flow) & 0x000fffff); } + /** * Returns number of IP headers in packet (includes IPv6 extension headers). */ diff --git a/src/IPAddr.h b/src/IPAddr.h index 8e1921e07b..f664f649f9 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -188,11 +188,20 @@ public: * IPv4 to IPv6 address mapping to return a full 16 bytes. * * @param bytes The pointer to a memory location in which the - * raw bytes of the address are to be copied in network byte-order. + * raw bytes of the address are to be copied. + * + * @param order The byte-order in which the returned raw bytes are copied. + * The default is network order. */ - void CopyIPv6(uint32_t* bytes) const + void CopyIPv6(uint32_t* bytes, ByteOrder order = Network) const { memcpy(bytes, in6.s6_addr, sizeof(in6.s6_addr)); + + if ( order == Host ) + { + for ( unsigned int i = 0; i < 4; ++i ) + bytes[i] = ntohl(bytes[i]); + } } /** @@ -280,6 +289,19 @@ public: */ string AsString() const; + /** + * Returns a string representation of the address suitable for inclusion + * in an URI. For IPv4 addresses, this is the same as AsString(), but + * IPv6 addresses are encased in square brackets. + */ + string AsURIString() const + { + if ( GetFamily() == IPv4 ) + return AsString(); + else + return string("[") + AsString() + "]"; + } + /** * Returns a host-order, plain hex string representation of the address. */ diff --git a/src/NetVar.cc b/src/NetVar.cc index cf88794695..70aa60c886 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -239,6 +239,7 @@ StringVal* cmd_line_bpf_filter; #include "types.bif.netvar_def" #include "event.bif.netvar_def" #include "logging.bif.netvar_def" +#include "input.bif.netvar_def" #include "reporter.bif.netvar_def" void init_event_handlers() @@ -300,6 +301,7 @@ void init_net_var() #include "const.bif.netvar_init" #include "types.bif.netvar_init" #include "logging.bif.netvar_init" +#include "input.bif.netvar_init" #include "reporter.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index 7c40eca1fa..7aff9b84e6 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -249,6 +249,7 @@ extern void init_net_var(); #include "types.bif.netvar_h" #include "event.bif.netvar_h" #include "logging.bif.netvar_h" +#include "input.bif.netvar_h" #include "reporter.bif.netvar_h" #endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 61be8a9e8f..cc5e8c5ff9 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -147,6 +147,7 @@ #include #include +#include #include #include #include @@ -172,6 +173,9 @@ #include #include +#include +#include +#include #include "RemoteSerializer.h" #include "Func.h" @@ -322,6 +326,18 @@ static const char* msgToStr(int msg) } } +static vector tokenize(const string& s, char delim) + { + vector tokens; + stringstream ss(s); + string token; + + while ( std::getline(ss, token, delim) ) + tokens.push_back(token); + + return tokens; + } + // Start of every message between two processes. We do the low-level work // ourselves to make this 64-bit safe. (The actual layout is an artifact of // an earlier design that depended on how a 32-bit GCC lays out its structs ...) @@ -458,17 +474,6 @@ static inline char* fmt_uint32s(int nargs, va_list ap) } #endif - -static inline const char* ip2a(uint32 ip) - { - static char buffer[32]; - struct in_addr addr; - - addr.s_addr = htonl(ip); - - return bro_inet_ntop(AF_INET, &addr, buffer, 32); - } - static pid_t child_pid = 0; // Return true if message type is sent by a peer (rather than the child @@ -675,7 +680,8 @@ void RemoteSerializer::Fork() } RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, - uint16 port, const char* our_class, double retry, bool use_ssl) + const string& zone_id, uint16 port, const char* our_class, double retry, + bool use_ssl) { if ( ! using_communication ) return true; @@ -683,24 +689,22 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPv6 ) - Error("inter-Bro communication not supported over IPv6"); - - const uint32* bytes; - ip.GetBytes(&bytes); - uint32 ip4 = ntohl(*bytes); - if ( ! child_pid ) Fork(); - Peer* p = AddPeer(ip4, port); + Peer* p = AddPeer(ip, port); p->orig = true; if ( our_class ) p->our_class = our_class; - if ( ! SendToChild(MSG_CONNECT_TO, p, 5, p->id, - ip4, port, uint32(retry), use_ssl) ) + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%"PRIu64",%s,%s,%"PRIu16",%"PRIu32",%d", p->id, + ip.AsString().c_str(), zone_id.c_str(), port, uint32(retry), + use_ssl); + + if ( ! SendToChild(MSG_CONNECT_TO, p, data) ) { RemovePeer(p); return false; @@ -1232,7 +1236,8 @@ bool RemoteSerializer::SendCapabilities(Peer* peer) return caps ? SendToChild(MSG_CAPS, peer, 3, caps, 0, 0) : true; } -bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl) +bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, + bool ipv6, const string& zone_id, double retry) { if ( ! using_communication ) return true; @@ -1240,14 +1245,18 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl) if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPv6 ) - Error("inter-Bro communication not supported over IPv6"); + if ( ! ipv6 && ip.GetFamily() == IPv6 && + ip != IPAddr("0.0.0.0") && ip != IPAddr("::") ) + reporter->FatalError("Attempt to listen on address %s, but IPv6 " + "communication disabled", ip.AsString().c_str()); - const uint32* bytes; - ip.GetBytes(&bytes); - uint32 ip4 = ntohl(*bytes); + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%s,%"PRIu16",%d,%d,%s,%"PRIu32, + ip.AsString().c_str(), port, expect_ssl, ipv6, zone_id.c_str(), + (uint32) retry); - if ( ! SendToChild(MSG_LISTEN, 0, 3, ip4, port, expect_ssl) ) + if ( ! SendToChild(MSG_LISTEN, 0, data) ) return false; listening = true; @@ -1784,7 +1793,7 @@ RecordVal* RemoteSerializer::MakePeerVal(Peer* peer) RecordVal* v = new RecordVal(::peer); v->Assign(0, new Val(uint32(peer->id), TYPE_COUNT)); // Sic! Network order for AddrVal, host order for PortVal. - v->Assign(1, new AddrVal(htonl(peer->ip))); + v->Assign(1, new AddrVal(peer->ip)); v->Assign(2, new PortVal(peer->port, TRANSPORT_TCP)); v->Assign(3, new Val(false, TYPE_BOOL)); v->Assign(4, new StringVal("")); // set when received @@ -1793,8 +1802,8 @@ RecordVal* RemoteSerializer::MakePeerVal(Peer* peer) return v; } -RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port, - PeerID id) +RemoteSerializer::Peer* RemoteSerializer::AddPeer(const IPAddr& ip, uint16 port, + PeerID id) { Peer* peer = new Peer; peer->id = id != PEER_NONE ? id : id_counter++; @@ -1959,9 +1968,22 @@ bool RemoteSerializer::EnterPhaseRunning(Peer* peer) bool RemoteSerializer::ProcessConnected() { // IP and port follow. - uint32* args = (uint32*) current_args->data; - uint32 host = ntohl(args[0]); // ### Fix: only works for IPv4 - uint16 port = (uint16) ntohl(args[1]); + vector args = tokenize(current_args->data, ','); + + if ( args.size() != 2 ) + { + InternalCommError("ProcessConnected() bad number of arguments"); + return false; + } + + IPAddr host = IPAddr(args[0]); + uint16 port; + + if ( ! atoi_n(args[1].size(), args[1].c_str(), 0, 10, port) ) + { + InternalCommError("ProcessConnected() bad peer port string"); + return false; + } if ( ! current_peer ) { @@ -2980,7 +3002,8 @@ void RemoteSerializer::Log(LogLevel level, const char* msg, Peer* peer, if ( peer ) len += snprintf(buffer + len, sizeof(buffer) - len, "[#%d/%s:%d] ", - int(peer->id), ip2a(peer->ip), peer->port); + int(peer->id), peer->ip.AsURIString().c_str(), + peer->port); len += safe_snprintf(buffer + len, sizeof(buffer) - len, "%s", msg); @@ -3266,8 +3289,10 @@ SocketComm::SocketComm() terminating = false; killing = false; - listen_fd_clear = -1; - listen_fd_ssl = -1; + listen_port = 0; + listen_ssl = false; + enable_ipv6 = false; + bind_retry_interval = 0; listen_next_try = 0; // We don't want to use the signal handlers of our parent. @@ -3290,8 +3315,7 @@ SocketComm::~SocketComm() delete peers[i]->io; delete io; - close(listen_fd_clear); - close(listen_fd_ssl); + CloseListenFDs(); } static unsigned int first_rtime = 0; @@ -3340,20 +3364,13 @@ void SocketComm::Run() } if ( listen_next_try && time(0) > listen_next_try ) - Listen(listen_if, listen_port, listen_ssl); + Listen(); - if ( listen_fd_clear >= 0 ) + for ( size_t i = 0; i < listen_fds.size(); ++i ) { - FD_SET(listen_fd_clear, &fd_read); - if ( listen_fd_clear > max_fd ) - max_fd = listen_fd_clear; - } - - if ( listen_fd_ssl >= 0 ) - { - FD_SET(listen_fd_ssl, &fd_read); - if ( listen_fd_ssl > max_fd ) - max_fd = listen_fd_ssl; + FD_SET(listen_fds[i], &fd_read); + if ( listen_fds[i] > max_fd ) + max_fd = listen_fds[i]; } if ( io->IsFillingUp() && ! shutting_conns_down ) @@ -3442,12 +3459,9 @@ void SocketComm::Run() } } - if ( listen_fd_clear >= 0 && - FD_ISSET(listen_fd_clear, &fd_read) ) - AcceptConnection(listen_fd_clear); - - if ( listen_fd_ssl >= 0 && FD_ISSET(listen_fd_ssl, &fd_read) ) - AcceptConnection(listen_fd_ssl); + for ( size_t i = 0; i < listen_fds.size(); ++i ) + if ( FD_ISSET(listen_fds[i], &fd_read) ) + AcceptConnection(listen_fds[i]); // Hack to display CPU usage of the child, triggered via // SIGPROF. @@ -3571,13 +3585,8 @@ bool SocketComm::DoParentMessage() case MSG_LISTEN_STOP: { - if ( listen_fd_ssl >= 0 ) - close(listen_fd_ssl); + CloseListenFDs(); - if ( listen_fd_clear >= 0 ) - close(listen_fd_clear); - - listen_fd_clear = listen_fd_ssl = -1; Log("stopped listening"); return true; @@ -3717,14 +3726,43 @@ bool SocketComm::ForwardChunkToPeer() bool SocketComm::ProcessConnectTo() { assert(parent_args); - uint32* args = (uint32*) parent_args->data; + vector args = tokenize(parent_args->data, ','); + + if ( args.size() != 6 ) + { + Error(fmt("ProcessConnectTo() bad number of arguments")); + return false; + } Peer* peer = new Peer; - peer->id = ntohl(args[0]); - peer->ip = ntohl(args[1]); - peer->port = ntohl(args[2]); - peer->retry = ntohl(args[3]); - peer->ssl = ntohl(args[4]); + + if ( ! atoi_n(args[0].size(), args[0].c_str(), 0, 10, peer->id) ) + { + Error(fmt("ProccessConnectTo() bad peer id string")); + delete peer; + return false; + } + + peer->ip = IPAddr(args[1]); + peer->zone_id = args[2]; + + if ( ! atoi_n(args[3].size(), args[3].c_str(), 0, 10, peer->port) ) + { + Error(fmt("ProcessConnectTo() bad peer port string")); + delete peer; + return false; + } + + if ( ! atoi_n(args[4].size(), args[4].c_str(), 0, 10, peer->retry) ) + { + Error(fmt("ProcessConnectTo() bad peer retry string")); + delete peer; + return false; + } + + peer->ssl = false; + if ( args[5] != "0" ) + peer->ssl = true; return Connect(peer); } @@ -3732,13 +3770,39 @@ bool SocketComm::ProcessConnectTo() bool SocketComm::ProcessListen() { assert(parent_args); - uint32* args = (uint32*) parent_args->data; + vector args = tokenize(parent_args->data, ','); - uint32 addr = ntohl(args[0]); - uint16 port = uint16(ntohl(args[1])); - uint32 ssl = ntohl(args[2]); + if ( args.size() != 6 ) + { + Error(fmt("ProcessListen() bad number of arguments")); + return false; + } - return Listen(addr, port, ssl); + listen_if = args[0]; + + if ( ! atoi_n(args[1].size(), args[1].c_str(), 0, 10, listen_port) ) + { + Error(fmt("ProcessListen() bad peer port string")); + return false; + } + + listen_ssl = false; + if ( args[2] != "0" ) + listen_ssl = true; + + enable_ipv6 = false; + if ( args[3] != "0" ) + enable_ipv6 = true; + + listen_zone_id = args[4]; + + if ( ! atoi_n(args[5].size(), args[5].c_str(), 0, 10, bind_retry_interval) ) + { + Error(fmt("ProcessListen() bad peer port string")); + return false; + } + + return Listen(); } bool SocketComm::ProcessParentCompress() @@ -3900,29 +3964,54 @@ bool SocketComm::ProcessPeerCompress(Peer* peer) bool SocketComm::Connect(Peer* peer) { - struct sockaddr_in server; + int status; + addrinfo hints, *res, *res0; + bzero(&hints, sizeof(hints)); - int sockfd = socket(PF_INET, SOCK_STREAM, 0); - if ( sockfd < 0 ) + hints.ai_family = PF_UNSPEC; + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_NUMERICHOST; + + char port_str[16]; + modp_uitoa10(peer->port, port_str); + + string gaihostname(peer->ip.AsString()); + if ( peer->zone_id != "" ) + gaihostname.append("%").append(peer->zone_id); + + status = getaddrinfo(gaihostname.c_str(), port_str, &hints, &res0); + if ( status != 0 ) { - Error(fmt("can't create socket, %s", strerror(errno))); + Error(fmt("getaddrinfo error: %s", gai_strerror(status))); return false; } - bzero(&server, sizeof(server)); - server.sin_family = AF_INET; - server.sin_port = htons(peer->port); - server.sin_addr.s_addr = htonl(peer->ip); - - bool connected = true; - - if ( connect(sockfd, (sockaddr*) &server, sizeof(server)) < 0 ) + int sockfd = -1; + for ( res = res0; res; res = res->ai_next ) { - Error(fmt("connect failed: %s", strerror(errno)), peer); - close(sockfd); - connected = false; + sockfd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if ( sockfd < 0 ) + { + Error(fmt("can't create connect socket, %s", strerror(errno))); + continue; + } + + if ( connect(sockfd, res->ai_addr, res->ai_addrlen) < 0 ) + { + Error(fmt("connect failed: %s", strerror(errno)), peer); + close(sockfd); + sockfd = -1; + continue; + } + + break; } + freeaddrinfo(res0); + + bool connected = sockfd != -1; + if ( ! (connected || peer->retry) ) { CloseConnection(peer, false); @@ -3947,9 +4036,7 @@ bool SocketComm::Connect(Peer* peer) if ( connected ) { if ( peer->ssl ) - { peer->io = new ChunkedIOSSL(sockfd, false); - } else peer->io = new ChunkedIOFd(sockfd, "child->peer"); @@ -3964,7 +4051,13 @@ bool SocketComm::Connect(Peer* peer) if ( connected ) { Log("connected", peer); - if ( ! SendToParent(MSG_CONNECTED, peer, 2, peer->ip, peer->port) ) + + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%s,%"PRIu32, peer->ip.AsString().c_str(), + peer->port); + + if ( ! SendToParent(MSG_CONNECTED, peer, data) ) return false; } @@ -4001,86 +4094,148 @@ bool SocketComm::CloseConnection(Peer* peer, bool reconnect) return true; } -bool SocketComm::Listen(uint32 ip, uint16 port, bool expect_ssl) +bool SocketComm::Listen() { - int* listen_fd = expect_ssl ? &listen_fd_ssl : &listen_fd_clear; + int status, on = 1; + addrinfo hints, *res, *res0; + bzero(&hints, sizeof(hints)); - if ( *listen_fd >= 0 ) - close(*listen_fd); + IPAddr listen_ip(listen_if); - struct sockaddr_in server; - - *listen_fd = socket(PF_INET, SOCK_STREAM, 0); - if ( *listen_fd < 0 ) + if ( enable_ipv6 ) { - Error(fmt("can't create listen socket, %s", - strerror(errno))); + if ( listen_ip == IPAddr("0.0.0.0") || listen_ip == IPAddr("::") ) + hints.ai_family = PF_UNSPEC; + else + hints.ai_family = (listen_ip.GetFamily() == IPv4 ? PF_INET : PF_INET6); + } + else + hints.ai_family = PF_INET; + + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST; + + char port_str[16]; + modp_uitoa10(listen_port, port_str); + + string scoped_addr(listen_if); + if ( listen_zone_id != "" ) + scoped_addr.append("%").append(listen_zone_id); + + const char* addr_str = 0; + if ( listen_ip != IPAddr("0.0.0.0") && listen_ip != IPAddr("::") ) + addr_str = scoped_addr.c_str(); + + CloseListenFDs(); + + if ( (status = getaddrinfo(addr_str, port_str, &hints, &res0)) != 0 ) + { + Error(fmt("getaddrinfo error: %s", gai_strerror(status))); return false; } - // Set SO_REUSEADDR. - int turn_on = 1; - if ( setsockopt(*listen_fd, SOL_SOCKET, SO_REUSEADDR, - &turn_on, sizeof(turn_on)) < 0 ) + for ( res = res0; res; res = res->ai_next ) { - Error(fmt("can't set SO_REUSEADDR, %s", - strerror(errno))); - return false; - } - - bzero(&server, sizeof(server)); - server.sin_family = AF_INET; - server.sin_port = htons(port); - server.sin_addr.s_addr = htonl(ip); - - if ( bind(*listen_fd, (sockaddr*) &server, sizeof(server)) < 0 ) - { - Error(fmt("can't bind to port %d, %s", port, strerror(errno))); - close(*listen_fd); - *listen_fd = -1; - - if ( errno == EADDRINUSE ) + if ( res->ai_family != AF_INET && res->ai_family != AF_INET6 ) { - listen_if = ip; - listen_port = port; - listen_ssl = expect_ssl; - // FIXME: Make this timeout configurable. - listen_next_try = time(0) + 30; + Error(fmt("can't create listen socket: unknown address family, %d", + res->ai_family)); + continue; } - return false; + + IPAddr a = (res->ai_family == AF_INET) ? + IPAddr(((sockaddr_in*)res->ai_addr)->sin_addr) : + IPAddr(((sockaddr_in6*)res->ai_addr)->sin6_addr); + + string l_addr_str(a.AsURIString()); + if ( listen_zone_id != "") + l_addr_str.append("%").append(listen_zone_id); + + int fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if ( fd < 0 ) + { + Error(fmt("can't create listen socket, %s", strerror(errno))); + continue; + } + + if ( setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0 ) + Error(fmt("can't set SO_REUSEADDR, %s", strerror(errno))); + + // For IPv6 listening sockets, we don't want do dual binding to also + // get IPv4-mapped addresses because that's not as portable. e.g. + // many BSDs don't allow that. + if ( res->ai_family == AF_INET6 && + setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) < 0 ) + Error(fmt("can't set IPV6_V6ONLY, %s", strerror(errno))); + + if ( bind(fd, res->ai_addr, res->ai_addrlen) < 0 ) + { + Error(fmt("can't bind to %s:%s, %s", l_addr_str.c_str(), + port_str, strerror(errno))); + close(fd); + + if ( errno == EADDRINUSE ) + { + // Abandon completely this attempt to set up listening sockets, + // try again later. + CloseListenFDs(); + listen_next_try = time(0) + bind_retry_interval; + return false; + } + continue; + } + + if ( listen(fd, 50) < 0 ) + { + Error(fmt("can't listen on %s:%s, %s", l_addr_str.c_str(), + port_str, strerror(errno))); + close(fd); + continue; + } + + listen_fds.push_back(fd); + Log(fmt("listening on %s:%s (%s)", l_addr_str.c_str(), port_str, + listen_ssl ? "ssl" : "clear")); } - if ( listen(*listen_fd, 50) < 0 ) - { - Error(fmt("can't listen, %s", strerror(errno))); - return false; - } + freeaddrinfo(res0); listen_next_try = 0; - Log(fmt("listening on %s:%d (%s)", - ip2a(ip), port, expect_ssl ? "ssl" : "clear")); - return true; + return listen_fds.size() > 0; } bool SocketComm::AcceptConnection(int fd) { - sockaddr_in client; + sockaddr_storage client; socklen_t len = sizeof(client); int clientfd = accept(fd, (sockaddr*) &client, &len); if ( clientfd < 0 ) { - Error(fmt("accept failed, %s %d", - strerror(errno), errno)); + Error(fmt("accept failed, %s %d", strerror(errno), errno)); + return false; + } + + if ( client.ss_family != AF_INET && client.ss_family != AF_INET6 ) + { + Error(fmt("accept fail, unknown address family %d", client.ss_family)); + close(clientfd); return false; } Peer* peer = new Peer; peer->id = id_counter++; - peer->ip = ntohl(client.sin_addr.s_addr); - peer->port = ntohs(client.sin_port); + peer->ip = client.ss_family == AF_INET ? + IPAddr(((sockaddr_in*)&client)->sin_addr) : + IPAddr(((sockaddr_in6*)&client)->sin6_addr); + + peer->port = client.ss_family == AF_INET ? + ntohs(((sockaddr_in*)&client)->sin_port) : + ntohs(((sockaddr_in6*)&client)->sin6_port); + peer->connected = true; - peer->ssl = (fd == listen_fd_ssl); + peer->ssl = listen_ssl; peer->compressor = false; if ( peer->ssl ) @@ -4090,8 +4245,7 @@ bool SocketComm::AcceptConnection(int fd) if ( ! peer->io->Init() ) { - Error(fmt("can't init peer io: %s", - peer->io->Error()), false); + Error(fmt("can't init peer io: %s", peer->io->Error()), false); return false; } @@ -4099,7 +4253,12 @@ bool SocketComm::AcceptConnection(int fd) Log(fmt("accepted %s connection", peer->ssl ? "SSL" : "clear"), peer); - if ( ! SendToParent(MSG_CONNECTED, peer, 2, peer->ip, peer->port) ) + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%s,%"PRIu32, peer->ip.AsString().c_str(), + peer->port); + + if ( ! SendToParent(MSG_CONNECTED, peer, data) ) return false; return true; @@ -4116,13 +4275,27 @@ const char* SocketComm::MakeLogString(const char* msg, Peer* peer) int len = 0; if ( peer ) + { + string scoped_addr(peer->ip.AsURIString()); + if ( peer->zone_id != "" ) + scoped_addr.append("%").append(peer->zone_id); + len = snprintf(buffer, BUFSIZE, "[#%d/%s:%d] ", int(peer->id), - ip2a(peer->ip), peer->port); + scoped_addr.c_str(), peer->port); + } len += safe_snprintf(buffer + len, BUFSIZE - len, "%s", msg); return buffer; } +void SocketComm::CloseListenFDs() + { + for ( size_t i = 0; i < listen_fds.size(); ++i ) + close(listen_fds[i]); + + listen_fds.clear(); + } + void SocketComm::Error(const char* msg, bool kill_me) { if ( kill_me ) @@ -4165,7 +4338,7 @@ void SocketComm::Log(const char* msg, Peer* peer) void SocketComm::InternalError(const char* msg) { - fprintf(stderr, "interal error in child: %s\n", msg); + fprintf(stderr, "internal error in child: %s\n", msg); Kill(); } @@ -4180,8 +4353,7 @@ void SocketComm::Kill() LogProf(); Log("terminating"); - close(listen_fd_clear); - close(listen_fd_ssl); + CloseListenFDs(); kill(getpid(), SIGTERM); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 05d25ca525..4ebf15e68d 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -10,8 +10,8 @@ #include "Stats.h" #include "File.h" -// All IP arguments are in host byte-order. -// FIXME: Change this to network byte order +#include +#include class IncrementalSendTimer; @@ -35,7 +35,8 @@ public: static const PeerID PEER_NONE = SOURCE_LOCAL; // Connect to host (returns PEER_NONE on error). - PeerID Connect(const IPAddr& ip, uint16 port, const char* our_class, double retry, bool use_ssl); + PeerID Connect(const IPAddr& ip, const string& zone_id, uint16 port, + const char* our_class, double retry, bool use_ssl); // Close connection to host. bool CloseConnection(PeerID peer); @@ -63,7 +64,8 @@ public: bool CompleteHandshake(PeerID peer); // Start to listen. - bool Listen(const IPAddr& ip, uint16 port, bool expect_ssl); + bool Listen(const IPAddr& ip, uint16 port, bool expect_ssl, bool ipv6, + const string& zone_id, double retry); // Stop it. bool StopListening(); @@ -179,9 +181,7 @@ protected: struct Peer { PeerID id; // Unique ID (non-zero) per peer. - // ### Fix: currently, we only work for IPv4. - // addr_type ip; - uint32 ip; + IPAddr ip; uint16 port; handler_list handlers; @@ -277,7 +277,7 @@ protected: bool ProcessLogWrite(); bool ProcessRequestLogs(); - Peer* AddPeer(uint32 ip, uint16 port, PeerID id = PEER_NONE); + Peer* AddPeer(const IPAddr& ip, uint16 port, PeerID id = PEER_NONE); Peer* LookupPeer(PeerID id, bool only_if_connected); void RemovePeer(Peer* peer); bool IsConnectedPeer(PeerID id); @@ -412,7 +412,6 @@ protected: { id = 0; io = 0; - ip = 0; port = 0; state = 0; connected = false; @@ -424,7 +423,8 @@ protected: RemoteSerializer::PeerID id; ChunkedIO* io; - uint32 ip; + IPAddr ip; + string zone_id; uint16 port; char state; bool connected; @@ -437,7 +437,7 @@ protected: bool compressor; }; - bool Listen(uint32 ip, uint16 port, bool expect_ssl); + bool Listen(); bool AcceptConnection(int listen_fd); bool Connect(Peer* peer); bool CloseConnection(Peer* peer, bool reconnect); @@ -482,6 +482,9 @@ protected: bool ForwardChunkToPeer(); const char* MakeLogString(const char* msg, Peer *peer); + // Closes all file descriptors associated with listening sockets. + void CloseListenFDs(); + // Peers we are communicating with: declare(PList, Peer); typedef PList(Peer) peer_list; @@ -498,15 +501,17 @@ protected: char parent_msgtype; ChunkedIO::Chunk* parent_args; - int listen_fd_clear; - int listen_fd_ssl; + vector listen_fds; // If the port we're trying to bind to is already in use, we will retry // it regularly. - uint32 listen_if; // Fix: only supports IPv4 + string listen_if; + string listen_zone_id; // RFC 4007 IPv6 zone_id uint16 listen_port; - bool listen_ssl; - time_t listen_next_try; + bool listen_ssl; // use SSL for IO + bool enable_ipv6; // allow IPv6 listen sockets + uint32 bind_retry_interval; // retry interval for already-in-use sockets + time_t listen_next_try; // time at which to try another bind bool shutting_conns_down; bool terminating; bool killing; diff --git a/src/Sessions.cc b/src/Sessions.cc index 704bb62a25..56df65d6af 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -597,7 +597,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, conn = (Connection*) d->Lookup(h); if ( ! conn ) { - conn = NewConn(h, t, &id, data, proto, encapsulation); + conn = NewConn(h, t, &id, data, proto, ip_hdr->FlowLabel(), encapsulation); if ( conn ) d->Insert(h, conn); } @@ -618,7 +618,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, conn->Event(connection_reused, 0); Remove(conn); - conn = NewConn(h, t, &id, data, proto, encapsulation); + conn = NewConn(h, t, &id, data, proto, ip_hdr->FlowLabel(), encapsulation); if ( conn ) d->Insert(h, conn); } @@ -642,6 +642,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, int is_orig = (id.src_addr == conn->OrigAddr()) && (id.src_port == conn->OrigPort()); + conn->CheckFlowLabel(is_orig, ip_hdr->FlowLabel()); + Val* pkt_hdr_val = 0; if ( ipv6_ext_headers && ip_hdr->NumHeaders() > 1 ) @@ -1010,8 +1012,8 @@ void NetSessions::GetStats(SessionStats& s) const } Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, - const u_char* data, int proto, - const Encapsulation& encapsulation) + const u_char* data, int proto, uint32 flow_label, + const Encapsulation& encapsulation) { // FIXME: This should be cleaned up a bit, it's too protocol-specific. // But I'm not yet sure what the right abstraction for these things is. @@ -1067,7 +1069,7 @@ Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, id = &flip_id; } - Connection* conn = new Connection(this, k, t, id, encapsulation); + Connection* conn = new Connection(this, k, t, id, flow_label, encapsulation); conn->SetTransport(tproto); dpm->BuildInitialAnalyzerTree(tproto, conn, data); diff --git a/src/Sessions.h b/src/Sessions.h index 54ff74ded9..e2bc0d704e 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -149,7 +149,8 @@ protected: friend class TimerMgrExpireTimer; Connection* NewConn(HashKey* k, double t, const ConnID* id, - const u_char* data, int proto, const Encapsulation& encapsulation); + const u_char* data, int proto, uint32 flow_lable, + const Encapsulation& encapsulation); // Check whether the tag of the current packet is consistent with // the given connection. Returns: diff --git a/src/Val.cc b/src/Val.cc index f3977789f2..32a3c367bb 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -948,6 +948,11 @@ SubNetVal::SubNetVal(const IPAddr& addr, int width) : Val(TYPE_SUBNET) val.subnet_val = new IPPrefix(addr, width); } +SubNetVal::SubNetVal(const IPPrefix& prefix) : Val(TYPE_SUBNET) + { + val.subnet_val = new IPPrefix(prefix); + } + SubNetVal::~SubNetVal() { delete val.subnet_val; diff --git a/src/Val.h b/src/Val.h index ed79f04671..2ca18e6131 100644 --- a/src/Val.h +++ b/src/Val.h @@ -580,6 +580,7 @@ public: SubNetVal(uint32 addr, int width); // IPv4. SubNetVal(const uint32 addr[4], int width); // IPv6. SubNetVal(const IPAddr& addr, int width); + SubNetVal(const IPPrefix& prefix); ~SubNetVal(); Val* SizeVal() const; @@ -839,6 +840,9 @@ public: timer = 0; } + HashKey* ComputeHash(const Val* index) const + { return table_hash->ComputeHash(index, 1); } + protected: friend class Val; friend class StateAccess; @@ -849,8 +853,6 @@ protected: void CheckExpireAttr(attr_tag at); int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val); int CheckAndAssign(Val* index, Val* new_val, Opcode op = OP_ASSIGN); - HashKey* ComputeHash(const Val* index) const - { return table_hash->ComputeHash(index, 1); } bool AddProperties(Properties arg_state); bool RemoveProperties(Properties arg_state); diff --git a/src/bro.bif b/src/bro.bif index 15740a83c7..e1521adee8 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -963,7 +963,7 @@ function sha256_hash_finish%(index: any%): string ## Generates a random number. ## -## max: The maximum value the random number. +## max: The maximum value of the random number. ## ## Returns: a random positive integer in the interval *[0, max)*. ## @@ -1020,7 +1020,7 @@ extern "C" { ## data: The data to find the MIME type for. ## ## return_mime: If true, the function returns a short MIME type string (e.g., -## ``text/plain`` instead of a more elaborate textual description. +## ``text/plain`` instead of a more elaborate textual description). ## ## Returns: The MIME type of *data*. function identify_data%(data: string, return_mime: bool%): string @@ -1241,8 +1241,6 @@ function unique_id_from%(pool: int, prefix: string%) : string ## Removes all elements from a set or table. ## ## v: The set or table -## -## Returns: The cleared set/table or 0 if *v* is not a set/table type. function clear_table%(v: any%): any %{ if ( v->Type()->Tag() == TYPE_TABLE ) @@ -1290,7 +1288,7 @@ function same_object%(o1: any, o2: any%): bool return new Val(o1 == o2, TYPE_BOOL); %} -## Returns the number bytes that a value occupies in memory. +## Returns the number of bytes that a value occupies in memory. ## ## v: The value ## @@ -1306,7 +1304,7 @@ function val_size%(v: any%): count ## ## newsize: The new size of *aggr*. ## -## Returns: The old size of *aggr* and 0 if *aggr* is not a :bro:type:`vector`. +## Returns: The old size of *aggr*, or 0 if *aggr* is not a :bro:type:`vector`. function resize%(aggr: any, newsize: count%) : count %{ if ( aggr->Type()->Tag() != TYPE_VECTOR ) @@ -1423,7 +1421,7 @@ bool indirect_int_sort_function(int a, int b) %%} ## Sorts a vector in place. The second argument is a comparison function that -## takes two arguments: if the vector type is \verb|vector of T|, then the +## takes two arguments: if the vector type is ``vector of T``, then the ## comparison function must be ``function(a: T, b: T): bool``, which returns ## ``a < b`` for some type-specific notion of the less-than operator. ## @@ -1599,7 +1597,7 @@ function cat%(...%): string ## given argument. If any of the variable arguments is an empty string it is ## replaced by a given default string instead. ## -## sep: The separator to place betwen each argument. +## sep: The separator to place between each argument. ## ## def: The default string to use when an argument is the empty string. ## @@ -1657,7 +1655,7 @@ function cat_sep%(sep: string, def: string, ...%): string ## ## - ``[DT]``: ISO timestamp with microsecond precision ## -## - ``d``: Signed/Unsigned integer (using C-style ``%lld|``/``%llu`` +## - ``d``: Signed/Unsigned integer (using C-style ``%lld``/``%llu`` ## for ``int``/``count``) ## ## - ``x``: Unsigned hexadecimal (using C-style ``%llx``); @@ -1782,7 +1780,7 @@ function log10%(d: double%): double # =========================================================================== ## Determines whether *c* has been received externally. For example, -## Broccoli or the Time Machine can send packets to Bro via a mechanism that +## Broccoli or the Time Machine can send packets to Bro via a mechanism that is ## one step lower than sending events. This function checks whether the packets ## of a connection stem from one of these external *packet sources*. ## @@ -1796,7 +1794,7 @@ function is_external_connection%(c: connection%) : bool ## Returns the ID of the analyzer which raised the current event. ## -## Returns: The ID of the analyzer which raised hte current event, or 0 if +## Returns: The ID of the analyzer which raised the current event, or 0 if ## none. function current_analyzer%(%) : count %{ @@ -2053,7 +2051,7 @@ function get_gap_summary%(%): gap_info %} ## Generates a table of the size of all global variables. The table index is -## the variable name and the value the variable size in bytes. +## the variable name and the value is the variable size in bytes. ## ## Returns: A table that maps variable names to their sizes. ## @@ -2138,7 +2136,7 @@ function lookup_ID%(id: string%) : any return i->ID_Val()->Ref(); %} -## Generates meta data about a record fields. The returned information +## Generates metadata about a record's fields. The returned information ## includes the field name, whether it is logged, its value (if it has one), ## and its default value (if specified). ## @@ -2269,11 +2267,11 @@ function dump_rule_stats%(f: file%): bool return new Val(1, TYPE_BOOL); %} -## Checks wheter Bro is terminating. +## Checks if Bro is terminating. ## ## Returns: True if Bro is in the process of shutting down. ## -## .. bro:see: terminate +## .. bro:see:: terminate function bro_is_terminating%(%): bool %{ return new Val(terminating, TYPE_BOOL); @@ -2354,7 +2352,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec return rval; %} -## Converts a :bro:type:`addr` to a :bro:type:`index_vec`. +## Converts an :bro:type:`addr` to an :bro:type:`index_vec`. ## ## a: The address to convert into a vector of counts. ## @@ -2374,7 +2372,7 @@ function addr_to_counts%(a: addr%): index_vec return rval; %} -## Converts a :bro:type:`index_vec` to a :bro:type:`addr`. +## Converts an :bro:type:`index_vec` to an :bro:type:`addr`. ## ## v: The vector containing host-order IP address representation, ## one element for IPv4 addresses, four elements for IPv6 addresses. @@ -2404,7 +2402,7 @@ function counts_to_addr%(v: index_vec%): addr } %} -## Converts a :bro:type:`string` to a :bro:type:`int`. +## Converts a :bro:type:`string` to an :bro:type:`int`. ## ## str: The :bro:type:`string` to convert. ## @@ -2434,7 +2432,7 @@ function to_int%(str: string%): int ## ## n: The :bro:type:`int` to convert. ## -## Returns: The :bro:type:`int` *n* as unsigned integer or 0 if *n* < 0. +## Returns: The :bro:type:`int` *n* as unsigned integer, or 0 if *n* < 0. function int_to_count%(n: int%): count %{ if ( n < 0 ) @@ -2449,7 +2447,7 @@ function int_to_count%(n: int%): count ## ## d: The :bro:type:`double` to convert. ## -## Returns: The :bro:type:`double` *d* as unsigned integer or 0 if *d* < 0.0. +## Returns: The :bro:type:`double` *d* as unsigned integer, or 0 if *d* < 0.0. ## ## .. bro:see:: double_to_time function double_to_count%(d: double%): count @@ -2464,8 +2462,8 @@ function double_to_count%(d: double%): count ## ## str: The :bro:type:`string` to convert. ## -## Returns: The :bro:type:`string` *str* as unsigned integer or if in invalid -## format. +## Returns: The :bro:type:`string` *str* as unsigned integer, or 0 if *str* has +## an invalid format. ## ## .. bro:see:: to_addr to_int to_port to_subnet function to_count%(str: string%): count @@ -2498,7 +2496,7 @@ function interval_to_double%(i: interval%): double ## Converts a :bro:type:`time` value to a :bro:type:`double`. ## -## t: The :bro:type:`interval` to convert. +## t: The :bro:type:`time` to convert. ## ## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. ## @@ -2508,11 +2506,11 @@ function time_to_double%(t: time%): double return new Val(t, TYPE_DOUBLE); %} -## Converts a :bro:type:`time` value to a :bro:type:`double`. +## Converts a :bro:type:`double` value to a :bro:type:`time`. ## -## t: The :bro:type:`interval` to convert. +## d: The :bro:type:`double` to convert. ## -## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. +## Returns: The :bro:type:`double` value *d* as :bro:type:`time`. ## ## .. bro:see:: time_to_double double_to_count function double_to_time%(d: double%): time @@ -2550,7 +2548,7 @@ function port_to_count%(p: port%): count ## ## proto: The transport protocol. ## -## Returns: The :bro:type:`count` *c* as :bro:type:`port`. +## Returns: The :bro:type:`count` *num* as :bro:type:`port`. ## ## .. bro:see:: port_to_count function count_to_port%(num: count, proto: transport_proto%): port @@ -2562,7 +2560,7 @@ function count_to_port%(num: count, proto: transport_proto%): port ## ## ip: The :bro:type:`string` to convert. ## -## Returns: The :bro:type:`string` *ip* as :bro:type:`addr` or the unspecified +## Returns: The :bro:type:`string` *ip* as :bro:type:`addr`, or the unspecified ## address ``::`` if the input string does not parse correctly. ## ## .. bro:see:: to_count to_int to_port count_to_v4_addr raw_bytes_to_v4_addr @@ -2579,7 +2577,7 @@ function to_addr%(ip: string%): addr ## ## sn: The subnet to convert. ## -## Returns: The *sn* string as a :bro:type:`subnet` or the unspecified subnet +## Returns: The *sn* string as a :bro:type:`subnet`, or the unspecified subnet ## ``::/0`` if the input string does not parse correctly. ## ## .. bro:see:: to_count to_int to_port count_to_v4_addr raw_bytes_to_v4_addr @@ -2616,7 +2614,7 @@ function count_to_v4_addr%(ip: count%): addr ## ## b: The raw bytes (:bro:type:`string`) to convert. ## -## Returns: The byte :bro:type:`string` *ip* as :bro:type:`addr`. +## Returns: The byte :bro:type:`string` *b* as :bro:type:`addr`. ## ## .. bro:see:: raw_bytes_to_v4_addr to_addr to_subnet function raw_bytes_to_v4_addr%(b: string%): addr @@ -2635,7 +2633,7 @@ function raw_bytes_to_v4_addr%(b: string%): addr return new AddrVal(htonl(a)); %} -## Converts a :bro:type:`string` to an :bro:type:`port`. +## Converts a :bro:type:`string` to a :bro:type:`port`. ## ## s: The :bro:type:`string` to convert. ## @@ -2885,7 +2883,7 @@ function parse_ftp_port%(s: string%): ftp_port %} ## Converts a string representation of the FTP EPRT command to an ``ftp_port``. -## (see `RFC 2428 `_). +## See `RFC 2428 `_. ## The format is ``EPRT``, ## where ```` is a delimiter in the ASCII range 33-126 (usually ``|``). ## @@ -2976,7 +2974,7 @@ function fmt_ftp_port%(a: addr, p: port%): string ## Decode a NetBIOS name. See http://support.microsoft.com/kb/194203. ## -## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF:``. +## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF"``. ## ## Returns: The decoded NetBIOS name, e.g., ``"THE NETBIOS NAME"``. ## @@ -3009,7 +3007,7 @@ function decode_netbios_name%(name: string%): string return new StringVal(i, result); %} -## Converts a NetBIOS name type to its corresonding numeric value. +## Converts a NetBIOS name type to its corresponding numeric value. ## See http://support.microsoft.com/kb/163409. ## ## name: The NetBIOS name type. @@ -3029,7 +3027,7 @@ function decode_netbios_name_type%(name: string%): count ## ## bytestring: The string of bytes. ## -## Returns: The hexadecimal reprsentation of *bytestring*. +## Returns: The hexadecimal representation of *bytestring*. ## ## .. bro:see:: hexdump function bytestring_to_hexstr%(bytestring: string%): string @@ -3069,7 +3067,7 @@ function decode_base64%(s: string%): string ## s: The Base64-encoded string. ## ## a: The custom alphabet. The empty string indicates the default alphabet. The -## lengh of *a* must bt 64. For example, a custom alphabet could be +## length of *a* must be 64. For example, a custom alphabet could be ## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. ## ## Returns: The decoded version of *s*. @@ -3138,7 +3136,7 @@ function uuid_to_string%(uuid: string%): string ## ## p2: The second pattern. ## -## Returns: The compiled pattern of the concatentation of *p1* and *p2*. +## Returns: The compiled pattern of the concatenation of *p1* and *p2*. ## ## .. bro:see:: convert_for_pattern string_to_pattern ## @@ -3277,7 +3275,7 @@ function strftime%(fmt: string, d: time%) : string ## a: The address to mask. ## ## top_bits_to_keep: The number of top bits to keep in *a*; must be greater -## than 0 and less than 33. +## than 0 and less than 33 for IPv4, or 129 for IPv6. ## ## Returns: The address *a* masked down to *top_bits_to_keep* bits. ## @@ -3341,7 +3339,7 @@ function is_udp_port%(p: port%): bool ## ## p: The :bro:type:`port` to check. ## -## Returns: True iff *p* is a ICMP port. +## Returns: True iff *p* is an ICMP port. ## ## .. bro:see:: is_tcp_port is_udp_port function is_icmp_port%(p: port%): bool @@ -3383,7 +3381,7 @@ EnumVal* map_conn_type(TransportProto tp) ## ## cid: The connection identifier. ## -## Returns: The transport protocol of the connection identified by *id*. +## Returns: The transport protocol of the connection identified by *cid*. ## ## .. bro:see:: get_port_transport_proto ## get_orig_seq get_resp_seq @@ -3497,7 +3495,7 @@ const char* conn_id_string(Val* c) ## ## c: The HTTP connection. ## -## is_orig: If true, the client data is skipped and the server data otherwise. +## is_orig: If true, the client data is skipped, and the server data otherwise. ## ## .. bro:see:: skip_smtp_data function skip_http_entity_data%(c: connection, is_orig: bool%): any @@ -3572,7 +3570,7 @@ function dump_current_packet%(file_name: string%) : bool ## Returns the currently processed PCAP packet. ## -## Returns: The currently processed packet, which is as a record +## Returns: The currently processed packet, which is a record ## containing the timestamp, ``snaplen``, and packet data. ## ## .. bro:see:: dump_current_packet dump_packet send_current_packet @@ -3730,7 +3728,7 @@ function lookup_addr%(host: addr%) : string ## ## host: The hostname to lookup. ## -## Returns: A set of DNS A records associated with *host*. +## Returns: A set of DNS A and AAAA records associated with *host*. ## ## .. bro:see:: lookup_addr function lookup_hostname%(host: string%) : addr_set @@ -3897,6 +3895,7 @@ function lookup_location%(a: addr%) : geo_location %} ## Performs an AS lookup of an IP address. +## Requires Bro to be built with ``libgeoip``. ## ## a: The IP address to lookup. ## @@ -4096,7 +4095,7 @@ function x509_err2str%(err_num: count%): string ## Converts UNIX file permissions given by a mode to an ASCII string. ## -## mode: The permisssions, e.g., 644 or 755. +## mode: The permissions (an octal number like 0644 converted to decimal). ## ## Returns: A string representation of *mode* in the format ## ``rw[xsS]rw[xsS]rw[xtT]``. @@ -4273,7 +4272,7 @@ function analyzer_name%(aid: count%) : string ## ## cid: The connection ID. ## -## Returns: False if *id* does not point to an active connection and true +## Returns: False if *cid* does not point to an active connection, and true ## otherwise. ## ## .. note:: @@ -4295,10 +4294,10 @@ function skip_further_processing%(cid: conn_id%): bool ## ## cid: The connection identifier. ## -## do_record: True to enable packet contens and false to disable for the +## do_record: True to enable packet contents, and false to disable for the ## connection identified by *cid*. ## -## Returns: False if *id* does not point to an active connection and true +## Returns: False if *cid* does not point to an active connection, and true ## otherwise. ## ## .. bro:see:: skip_further_processing @@ -4309,7 +4308,7 @@ function skip_further_processing%(cid: conn_id%): bool ## connection, which is controlled separately by ## :bro:id:`skip_further_processing`. ## -## .. bro:see: get_contents_file set_contents_file +## .. bro:see:: get_contents_file set_contents_file function set_record_packets%(cid: conn_id, do_record: bool%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -4326,7 +4325,7 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool ## cid: The connection ID. ## ## direction: Controls what sides of the connection to record. The argument can -## take one the four values: +## take one of the four values: ## ## - ``CONTENTS_NONE``: Stop recording the connection's content. ## - ``CONTENTS_ORIG``: Record the data sent by the connection @@ -4340,7 +4339,7 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool ## ## f: The file handle of the file to write the contents to. ## -## Returns: Returns false if *id* does not point to an active connection and +## Returns: Returns false if *cid* does not point to an active connection, and ## true otherwise. ## ## .. note:: @@ -4351,7 +4350,7 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool ## missing data; this can happen, e.g., due to an ## :bro:id:`ack_above_hole` event. ## -## .. bro:see: get_contents_file set_record_packets +## .. bro:see:: get_contents_file set_record_packets function set_contents_file%(cid: conn_id, direction: count, f: file%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -4366,15 +4365,15 @@ function set_contents_file%(cid: conn_id, direction: count, f: file%): bool ## ## cid: The connection ID. ## -## direction: Controls what sides of the connection to record. SEe +## direction: Controls what sides of the connection to record. See ## :bro:id:`set_contents_file` for possible values. ## -## Returns: The :bro:type:`file` handle for the contentents file of the +## Returns: The :bro:type:`file` handle for the contents file of the ## connection identified by *cid*. If the connection exists -## but no contents file for *direction*, the function generates a -## error and returns a file handle to ``stderr``. +## but there is no contents file for *direction*, then the function +## generates an error and returns a file handle to ``stderr``. ## -## .. bro:see: set_contents_file set_record_packets +## .. bro:see:: set_contents_file set_record_packets function get_contents_file%(cid: conn_id, direction: count%): file %{ Connection* c = sessions->FindConnection(cid); @@ -4425,7 +4424,7 @@ function set_inactivity_timeout%(cid: conn_id, t: interval%): interval ## ## - ``LOGIN_STATE_AUTHENTICATE``: The connection is in its ## initial authentication dialog. -## - ``OGIN_STATE_LOGGED_IN``: The analyzer believes the user has +## - ``LOGIN_STATE_LOGGED_IN``: The analyzer believes the user has ## successfully authenticated. ## - ``LOGIN_STATE_SKIP``: The analyzer has skipped any further ## processing of the connection. @@ -4433,7 +4432,7 @@ function set_inactivity_timeout%(cid: conn_id, t: interval%): interval ## does not correctly know the state of the connection, and/or ## the username associated with it. ## -## .. bro:see: set_login_state +## .. bro:see:: set_login_state function get_login_state%(cid: conn_id%): count %{ Connection* c = sessions->FindConnection(cid); @@ -4456,9 +4455,9 @@ function get_login_state%(cid: conn_id%): count ## :bro:id:`get_login_state` for possible values. ## ## Returns: Returns false if *cid* is not an active connection -## or does not tagged as login analyzer, and true otherwise. +## or is not tagged as a login analyzer, and true otherwise. ## -## .. bro:see: get_login_state +## .. bro:see:: get_login_state function set_login_state%(cid: conn_id, new_state: count%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -4590,9 +4589,9 @@ function disable_event_group%(group: string%) : any ## ## f: The path to the file. ## -## Returns: A :bro:type:`file` handle for subsequent operations. +## Returns: A :bro:type:`file` handle for subsequent operations. ## -## .. bro:see;: active_file open_for_append close write_file +## .. bro:see:: active_file open_for_append close write_file ## get_file_name set_buf flush_all mkdir enable_raw_output function open%(f: string%): file %{ @@ -4609,9 +4608,9 @@ function open%(f: string%): file ## ## f: The path to the file. ## -## Returns: A :bro:type:`file` handle for subsequent operations. +## Returns: A :bro:type:`file` handle for subsequent operations. ## -## .. bro:see;: active_file open close write_file +## .. bro:see:: active_file open close write_file ## get_file_name set_buf flush_all mkdir enable_raw_output function open_for_append%(f: string%): file %{ @@ -4619,13 +4618,12 @@ function open_for_append%(f: string%): file %} ## Closes an open file and flushes any buffered content. -## exists, this function appends to it (as opposed to :bro:id:`open`). ## ## f: A :bro:type:`file` handle to an open file. ## -## Returns: True on success. +## Returns: True on success. ## -## .. bro:see;: active_file open open_for_append write_file +## .. bro:see:: active_file open open_for_append write_file ## get_file_name set_buf flush_all mkdir enable_raw_output function close%(f: file%): bool %{ @@ -4638,9 +4636,9 @@ function close%(f: file%): bool ## ## data: The data to write to *f*. ## -## Returns: True on success. +## Returns: True on success. ## -## .. bro:see;: active_file open open_for_append close +## .. bro:see:: active_file open open_for_append close ## get_file_name set_buf flush_all mkdir enable_raw_output function write_file%(f: file, data: string%): bool %{ @@ -4656,11 +4654,11 @@ function write_file%(f: file, data: string%): bool ## f: A :bro:type:`file` handle to an open file. ## ## buffered: When true, *f* is fully buffered, i.e., bytes are saved in a -## buffered until the block size has been reached. When +## buffer until the block size has been reached. When ## false, *f* is line buffered, i.e., bytes are saved up until a ## newline occurs. ## -## .. bro:see;: active_file open open_for_append close +## .. bro:see:: active_file open open_for_append close ## get_file_name write_file flush_all mkdir enable_raw_output function set_buf%(f: file, buffered: bool%): any %{ @@ -4670,9 +4668,9 @@ function set_buf%(f: file, buffered: bool%): any ## Flushes all open files to disk. ## -## Returns: True on success. +## Returns: True on success. ## -## .. bro:see;: active_file open open_for_append close +## .. bro:see:: active_file open open_for_append close ## get_file_name write_file set_buf mkdir enable_raw_output function flush_all%(%): bool %{ @@ -4683,10 +4681,10 @@ function flush_all%(%): bool ## ## f: The directory name. ## -## Returns: Returns true if the operation succeeds and false if the +## Returns: Returns true if the operation succeeds, or false if the ## creation fails or if *f* exists already. ## -## .. bro:see;: active_file open_for_append close write_file +## .. bro:see:: active_file open_for_append close write_file ## get_file_name set_buf flush_all enable_raw_output function mkdir%(f: string%): bool %{ @@ -4731,7 +4729,7 @@ function get_file_name%(f: file%): string ## ## f: An open file handle. ## -## Returns: Rotations statistics which include the original file name, the name +## Returns: Rotation statistics which include the original file name, the name ## after the rotation, and the time when *f* was opened/closed. ## ## .. bro:see:: rotate_file_by_name calc_next_rotate @@ -4755,7 +4753,7 @@ function rotate_file%(f: file%): rotate_info ## ## f: The name of the file to rotate ## -## Returns: Rotations statistics which include the original file name, the name +## Returns: Rotation statistics which include the original file name, the name ## after the rotation, and the time when *f* was opened/closed. ## ## .. bro:see:: rotate_file calc_next_rotate @@ -4851,7 +4849,7 @@ function disable_print_hook%(f: file%): any return 0; %} -## Prevents escaping of non-ASCII character when writing to a file. +## Prevents escaping of non-ASCII characters when writing to a file. ## This function is equivalent to :bro:attr:`&disable_print_hook`. ## ## f: The file to disable raw output for. @@ -5213,9 +5211,9 @@ function checkpoint_state%(%) : bool return new Val(persistence_serializer->WriteState(true), TYPE_BOOL); %} -## Reads persistent state from the \texttt{.state} directory and populates the -## in-memory data structures accordingly. This function is the dual to -## :bro:id:`checkpoint_state`. +## Reads persistent state and populates the in-memory data structures +## accordingly. Persistent state is read from the ``.state`` directory. +## This function is the dual to :bro:id:`checkpoint_state`. ## ## Returns: True on success. ## @@ -5267,16 +5265,20 @@ function capture_state_updates%(filename: string%) : bool ## ## ip: The IP address of the remote peer. ## -## port: The port of the remote peer. +## zone_id: If *ip* is a non-global IPv6 address, a particular :rfc:`4007` +## ``zone_id`` can given here. An empty string, ``""``, means +## not to add any ``zone_id``. ## -## our_class: If an non-empty string, the remote (listening) peer checks it +## p: The port of the remote peer. +## +## our_class: If a non-empty string, then the remote (listening) peer checks it ## against its class name in its peer table and terminates the ## connection if they don't match. ## ## retry: If the connection fails, try to reconnect with the peer after this ## time interval. ## -## ssl: If true, uses SSL to encrypt the session. +## ssl: If true, use SSL to encrypt the session. ## ## Returns: A locally unique ID of the new peer. ## @@ -5290,16 +5292,17 @@ function capture_state_updates%(filename: string%) : bool ## set_compression_level ## send_state ## send_id -function connect%(ip: addr, p: port, our_class: string, retry: interval, ssl: bool%) : count +function connect%(ip: addr, zone_id: string, p: port, our_class: string, retry: interval, ssl: bool%) : count %{ - return new Val(uint32(remote_serializer->Connect(ip->AsAddr(), p->Port(), - our_class->CheckString(), retry, ssl)), + return new Val(uint32(remote_serializer->Connect(ip->AsAddr(), + zone_id->CheckString(), p->Port(), our_class->CheckString(), + retry, ssl)), TYPE_COUNT); %} ## Terminate the connection with a peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. ## @@ -5313,7 +5316,7 @@ function disconnect%(p: event_peer%) : bool ## Subscribes to all events from a remote peer whose names match a given ## pattern. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## handlers: The pattern describing the events to request from peer *p*. ## @@ -5331,7 +5334,7 @@ function request_remote_events%(p: event_peer, handlers: pattern%) : bool ## Requests synchronization of IDs with a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## auth: If true, the local instance considers its current state authoritative ## and sends it to *p* right after the handshake. @@ -5349,7 +5352,7 @@ function request_remote_sync%(p: event_peer, auth: bool%) : bool ## Requests logs from a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. ## @@ -5361,9 +5364,11 @@ function request_remote_logs%(p: event_peer%) : bool return new Val(remote_serializer->RequestLogs(id), TYPE_BOOL); %} -## Sets a boolean flag whether Bro accepts state from a remote peer. +## Sets a boolean flag indicating whether Bro accepts state from a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. +## +## accept: True if Bro accepts state from peer *p*, or false otherwise. ## ## Returns: True on success. ## @@ -5379,7 +5384,7 @@ function set_accept_state%(p: event_peer, accept: bool%) : bool ## Sets the compression level of the session with a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## level: Allowed values are in the range *[0, 9]*, where 0 is the default and ## means no compression. @@ -5394,20 +5399,29 @@ function set_compression_level%(p: event_peer, level: count%) : bool TYPE_BOOL); %} -## Listens on address a given IP address and port for remote connections. +## Listens on a given IP address and port for remote connections. ## ## ip: The IP address to bind to. ## -## p: The TCP port to listen to. +## p: The TCP port to listen on. ## ## ssl: If true, Bro uses SSL to encrypt the session. ## +## ipv6: If true, enable listening on IPv6 addresses. +## +## zone_id: If *ip* is a non-global IPv6 address, a particular :rfc:`4007` +## ``zone_id`` can given here. An empty string, ``""``, means +## not to add any ``zone_id``. +## +## retry_interval: If address *ip* is found to be already in use, this is +## the interval at which to automatically retry binding. +## ## Returns: True on success. ## ## .. bro:see:: connect disconnect -function listen%(ip: addr, p: port, ssl: bool %) : bool +function listen%(ip: addr, p: port, ssl: bool, ipv6: bool, zone_id: string, retry_interval: interval%) : bool %{ - return new Val(remote_serializer->Listen(ip->AsAddr(), p->Port(), ssl), TYPE_BOOL); + return new Val(remote_serializer->Listen(ip->AsAddr(), p->Port(), ssl, ipv6, zone_id->CheckString(), retry_interval), TYPE_BOOL); %} ## Checks whether the last raised event came from a remote peer. @@ -5420,7 +5434,7 @@ function is_remote_event%(%) : bool ## Sends all persistent state to a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. ## @@ -5431,10 +5445,10 @@ function send_state%(p: event_peer%) : bool return new Val(persistence_serializer->SendState(id, true), TYPE_BOOL); %} -## Sends a global identifier to a remote peer, which them might install it +## Sends a global identifier to a remote peer, which then might install it ## locally. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## id: The identifier to send. ## @@ -5468,7 +5482,7 @@ function terminate_communication%(%) : bool ## Signals a remote peer that the local Bro instance finished the initial ## handshake. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. function complete_handshake%(p: event_peer%) : bool @@ -5481,7 +5495,7 @@ function complete_handshake%(p: event_peer%) : bool ## for :bro:id:`remote_pong`, this function can be used to measure latency ## between two peers. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## seq: A sequence number (also included by :bro:id:`remote_pong`). ## @@ -5496,7 +5510,7 @@ function send_ping%(p: event_peer, seq: count%) : bool ## Sends the currently processed packet to a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True if sending the packet succeeds. ## @@ -5522,7 +5536,7 @@ function send_current_packet%(p: event_peer%) : bool ## Returns the peer who generated the last event. ## -## Returns: The ID of the peer who genereated the last event. +## Returns: The ID of the peer who generated the last event. ## ## .. bro:see:: get_local_event_peer function get_event_peer%(%) : event_peer @@ -5565,7 +5579,7 @@ function get_local_event_peer%(%) : event_peer ## Sends a capture filter to a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## s: The capture filter. ## @@ -5582,7 +5596,7 @@ function send_capture_filter%(p: event_peer, s: string%) : bool ## distributed trace processing with communication enabled ## (*pseudo-realtime* mode). ## -## .. bro:see: continue_processing suspend_state_updates resume_state_updates +## .. bro:see:: continue_processing suspend_state_updates resume_state_updates function suspend_processing%(%) : any %{ net_suspend_processing(); @@ -5591,7 +5605,7 @@ function suspend_processing%(%) : any ## Resumes Bro's packet processing. ## -## .. bro:see: suspend_processing suspend_state_updates resume_state_updates +## .. bro:see:: suspend_processing suspend_state_updates resume_state_updates function continue_processing%(%) : any %{ net_continue_processing(); @@ -5600,7 +5614,7 @@ function continue_processing%(%) : any ## Stops propagating :bro:attr:`&synchronized` accesses. ## -## .. bro:see: suspend_processing continue_processing resume_state_updates +## .. bro:see:: suspend_processing continue_processing resume_state_updates function suspend_state_updates%(%) : any %{ if ( remote_serializer ) @@ -5610,7 +5624,7 @@ function suspend_state_updates%(%) : any ## Resumes propagating :bro:attr:`&synchronized` accesses. ## -## .. bro:see: suspend_processing continue_processing suspend_state_updates +## .. bro:see:: suspend_processing continue_processing suspend_state_updates function resume_state_updates%(%) : any %{ if ( remote_serializer ) diff --git a/src/const.bif b/src/const.bif index 09ce769261..553e8b6d58 100644 --- a/src/const.bif +++ b/src/const.bif @@ -12,3 +12,5 @@ const NFS3::return_data_max: count; const NFS3::return_data_first_only: bool; const Tunnel::max_depth: count; + +const Threading::heartbeat_interval: interval; diff --git a/src/event.bif b/src/event.bif index 7e428aabdd..c4ed03e013 100644 --- a/src/event.bif +++ b/src/event.bif @@ -182,8 +182,11 @@ event new_connection_contents%(c: connection%); ## new_connection new_connection_contents partial_connection event connection_attempt%(c: connection%); -## Generated for an established TCP connection. The event is raised when the -## initial 3-way TCP handshake has successfully finished for a connection. +## Generated when a SYN-ACK packet is seen in response to SYN a packet during +## a TCP handshake. The final ACK of the handshake in response to SYN-ACK may +## or may not occur later, one way to tell is to check the *history* field of +## :bro:type:`connection` to see if the originator sent an ACK, indicated by +## 'A' in the history string. ## ## c: The connection. ## @@ -346,8 +349,6 @@ event connection_SYN_packet%(c: connection, pkt: SYN_packet%); ## ## c: The connection. ## -## pkt: Information extracted from the SYN packet. -## ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt ## connection_established connection_external connection_finished ## connection_half_finished connection_partial_close connection_pending @@ -412,6 +413,20 @@ event connection_reused%(c: connection%); ## new_connection new_connection_contents partial_connection event connection_status_update%(c: connection%); +## Generated for a connection over IPv6 when one direction has changed +## the flow label that it's using. +## +## c: The connection. +## +## is_orig: True if the event is raised for the originator side. +## +## old_label: The old flow label that the endpoint was using. +## +## new_label: The new flow label that the endpoint is using. +## +## .. bro:see:: connection_established new_connection +event connection_flow_label_changed%(c: connection, is_orig: bool, old_label: count, new_label: count%); + ## Generated at the end of reassembled TCP connections. The TCP reassembler ## raised the event once for each endpoint of a connection when it finished ## reassembling the corresponding side of the communication. diff --git a/src/input.bif b/src/input.bif new file mode 100644 index 0000000000..f494ef3b2f --- /dev/null +++ b/src/input.bif @@ -0,0 +1,55 @@ +# functions and types for the input framework + +module Input; + +%%{ +#include "input/Manager.h" +#include "NetVar.h" +%%} + +type TableDescription: record; +type EventDescription: record; + +function Input::__create_table_stream%(description: Input::TableDescription%) : bool + %{ + bool res = input_mgr->CreateTableStream(description->AsRecordVal()); + return new Val(res, TYPE_BOOL); + %} + +function Input::__create_event_stream%(description: Input::EventDescription%) : bool + %{ + bool res = input_mgr->CreateEventStream(description->AsRecordVal()); + return new Val(res, TYPE_BOOL); + %} + +function Input::__remove_stream%(id: string%) : bool + %{ + bool res = input_mgr->RemoveStream(id->AsString()->CheckString()); + return new Val(res, TYPE_BOOL); + %} + +function Input::__force_update%(id: string%) : bool + %{ + bool res = input_mgr->ForceUpdate(id->AsString()->CheckString()); + return new Val(res, TYPE_BOOL); + %} + +# Options for Ascii Reader + +module InputAscii; + +const separator: string; +const set_separator: string; +const empty_field: string; +const unset_field: string; + +module InputRaw; +const record_separator: string; + +module InputBenchmark; +const factor: double; +const spread: count; +const autospread: double; +const addfactor: count; +const stopspreadat: count; +const timedspread: double; diff --git a/src/input/Manager.cc b/src/input/Manager.cc new file mode 100644 index 0000000000..bc79a2390b --- /dev/null +++ b/src/input/Manager.cc @@ -0,0 +1,2052 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "Manager.h" +#include "ReaderFrontend.h" +#include "ReaderBackend.h" +#include "readers/Ascii.h" +#include "readers/Raw.h" +#include "readers/Benchmark.h" + +#include "Event.h" +#include "EventHandler.h" +#include "NetVar.h" +#include "Net.h" + + +#include "CompHash.h" + +#include "../threading/SerialTypes.h" + +using namespace input; +using threading::Value; +using threading::Field; + +struct ReaderDefinition { + bro_int_t type; // The reader type. + const char *name; // Descriptive name for error messages. + bool (*init)(); // Optional one-time initializing function. + ReaderBackend* (*factory)(ReaderFrontend* frontend); // Factory function for creating instances. +}; + +ReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, + { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, + { BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate }, + + // End marker + { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } +}; + +/** + * InputHashes are used as Dictionaries to store the value and index hashes + * for all lines currently stored in a table. Index hash is stored as + * HashKey*, because it is thrown into other Bro functions that need the + * complex structure of it. For everything we do (with values), we just take + * the hash_t value and compare it directly with "==" + */ +struct InputHash { + hash_t valhash; + HashKey* idxkey; + ~InputHash(); +}; + +InputHash::~InputHash() + { + delete idxkey; + } + +static void input_hash_delete_func(void* val) + { + InputHash* h = (InputHash*) val; + delete h; + } + +declare(PDict, InputHash); + +/** + * Base stuff that every stream can do. + */ +class Manager::Stream { +public: + string name; + string source; + bool removed; + + ReaderMode mode; + + StreamType stream_type; // to distinguish between event and table streams + + EnumVal* type; + ReaderFrontend* reader; + + RecordVal* description; + + Stream(); + virtual ~Stream(); +}; + +Manager::Stream::Stream() + { + type = 0; + reader = 0; + description = 0; + removed = false; + } + +Manager::Stream::~Stream() + { + if ( type ) + Unref(type); + + if ( description ) + Unref(description); + + if ( reader ) + delete(reader); + } + +class Manager::TableStream: public Manager::Stream { +public: + + unsigned int num_idx_fields; + unsigned int num_val_fields; + bool want_record; + EventHandlerPtr table_event; + + TableVal* tab; + RecordType* rtype; + RecordType* itype; + + PDict(InputHash)* currDict; + PDict(InputHash)* lastDict; + + Func* pred; + + EventHandlerPtr event; + + TableStream(); + ~TableStream(); +}; + +class Manager::EventStream: public Manager::Stream { +public: + EventHandlerPtr event; + + RecordType* fields; + unsigned int num_fields; + + bool want_record; + EventStream(); + ~EventStream(); +}; + +Manager::TableStream::TableStream() : Manager::Stream::Stream() + { + stream_type = TABLE_STREAM; + + tab = 0; + itype = 0; + rtype = 0; + + currDict = 0; + lastDict = 0; + + pred = 0; + } + +Manager::EventStream::EventStream() : Manager::Stream::Stream() + { + fields = 0; + stream_type = EVENT_STREAM; + } + +Manager::EventStream::~EventStream() + { + if ( fields ) + Unref(fields); + } + +Manager::TableStream::~TableStream() + { + if ( tab ) + Unref(tab); + + if ( itype ) + Unref(itype); + + if ( rtype ) // can be 0 for sets + Unref(rtype); + + if ( currDict != 0 ) + { + currDict->Clear(); + delete currDict; + } + + if ( lastDict != 0 ) + { + lastDict->Clear();; + delete lastDict; + } + } + +Manager::Manager() + { + } + +Manager::~Manager() + { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) + { + delete s->second; + delete s->first; + } + + } + +ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) + { + ReaderDefinition* ir = input_readers; + + while ( true ) + { + if ( ir->type == BifEnum::Input::READER_DEFAULT ) + { + reporter->Error("The reader that was requested was not found and could not be initialized."); + return 0; + } + + if ( ir->type != type ) + { + // no, didn't find the right one... + ++ir; + continue; + } + + + // call init function of writer if presnt + if ( ir->init ) + { + if ( (*ir->init)() ) + { + //clear it to be not called again + ir->init = 0; + } + + else { + // ohok. init failed, kill factory for all eternity + ir->factory = 0; + DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name); + return 0; + } + + } + + if ( ! ir->factory ) + // no factory? + return 0; + + // all done. break. + break; + } + + assert(ir->factory); + + ReaderBackend* backend = (*ir->factory)(frontend); + assert(backend); + + frontend->ty_name = ir->name; + return backend; + } + +// Create a new input reader object to be used at whomevers leisure lateron. +bool Manager::CreateStream(Stream* info, RecordVal* description) + { + ReaderDefinition* ir = input_readers; + + RecordType* rtype = description->Type()->AsRecordType(); + if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) + || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) + { + reporter->Error("Streamdescription argument not of right type for new input stream"); + return false; + } + + Val* name_val = description->LookupWithDefault(rtype->FieldOffset("name")); + string name = name_val->AsString()->CheckString(); + Unref(name_val); + + Stream *i = FindStream(name); + if ( i != 0 ) + { + reporter->Error("Trying create already existing input stream %s", + name.c_str()); + return false; + } + + EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); + + ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); + assert(reader_obj); + + // get the source ... + Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source")); + assert ( sourceval != 0 ); + const BroString* bsource = sourceval->AsString(); + string source((const char*) bsource->Bytes(), bsource->Len()); + Unref(sourceval); + + EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); + + switch ( mode->InternalInt() ) + { + case 0: + info->mode = MODE_MANUAL; + break; + + case 1: + info->mode = MODE_REREAD; + break; + + case 2: + info->mode = MODE_STREAM; + break; + + default: + reporter->InternalError("unknown reader mode"); + } + + Unref(mode); + + info->reader = reader_obj; + info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault + info->name = name; + info->source = source; + Ref(description); + info->description = description; + + DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", + name.c_str()); + + return true; + } + +bool Manager::CreateEventStream(RecordVal* fval) + { + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) + { + reporter->Error("EventDescription argument not of right type"); + return false; + } + + EventStream* stream = new EventStream(); + { + bool res = CreateStream(stream, fval); + if ( res == false ) + { + delete stream; + return false; + } + } + + + RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); + Func* event = event_val->AsFunc(); + Unref(event_val); + + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) + { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() < 2 ) + { + reporter->Error("event takes not enough arguments"); + return false; + } + + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("events second attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) + { + reporter->Error("events first attribute must be of type Input::EventDescription"); + return false; + } + + if ( want_record->InternalInt() == 0 ) + { + if ( args->length() != fields->NumFields() + 2 ) + { + reporter->Error("event has wrong number of arguments"); + return false; + } + + for ( int i = 0; i < fields->NumFields(); i++ ) + { + if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) + { + reporter->Error("Incompatible type for event"); + return false; + } + } + + } + + else if ( want_record->InternalInt() == 1 ) + { + if ( args->length() != 3 ) + { + reporter->Error("event has wrong number of arguments"); + return false; + } + + if ( !same_type((*args)[2], fields ) ) + { + reporter->Error("Incompatible type for event"); + return false; + } + + } + + else + assert(false); + + + vector fieldsV; // vector, because UnrollRecordType needs it + + bool status = !UnrollRecordType(&fieldsV, fields, ""); + + if ( status ) + { + reporter->Error("Problem unrolling"); + return false; + } + + Field** logf = new Field*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) + logf[i] = fieldsV[i]; + + Unref(fields); // ref'd by lookupwithdefault + stream->num_fields = fieldsV.size(); + stream->fields = fields->Ref()->AsRecordType(); + stream->event = event_registry->Lookup(event->GetID()->Name()); + stream->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + assert(stream->reader); + stream->reader->Init(stream->source, stream->mode, stream->num_fields, logf ); + + readers[stream->reader] = stream; + + DBG_LOG(DBG_INPUT, "Successfully created event stream %s", + stream->name.c_str()); + + return true; +} + +bool Manager::CreateTableStream(RecordVal* fval) + { + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) + { + reporter->Error("TableDescription argument not of right type"); + return false; + } + + TableStream* stream = new TableStream(); + { + bool res = CreateStream(stream, fval); + if ( res == false ) + { + delete stream; + return false; + } + } + + Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); + + RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = 0; + + if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) + { + val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + Unref(val); // The lookupwithdefault in the if-clause ref'ed val. + } + + TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); + + // check if index fields match table description + int num = idx->NumFields(); + const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); + + loop_over_list(*tl, j) + { + if ( j >= num ) + { + reporter->Error("Table type has more indexes than index definition"); + return false; + } + + if ( ! same_type(idx->FieldType(j), (*tl)[j]) ) + { + reporter->Error("Table type does not match index type"); + return false; + } + } + + if ( num != j ) + { + reporter->Error("Table has less elements than index definition"); + return false; + } + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); + Func* event = event_val ? event_val->AsFunc() : 0; + Unref(event_val); + + if ( event ) + { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) + { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() != 4 ) + { + reporter->Error("Table event must take 4 arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) + { + reporter->Error("table events first attribute must be of type Input::TableDescription"); + return false; + } + + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events second attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[2], idx) ) + { + reporter->Error("table events index attributes do not match"); + return false; + } + + if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) + { + reporter->Error("table events value attributes do not match"); + return false; + } + else if ( want_record->InternalInt() == 0 + && !same_type((*args)[3], val->FieldType(0) ) ) + { + reporter->Error("table events value attribute does not match"); + return false; + } + + assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); + + } + + vector fieldsV; // vector, because we don't know the length beforehands + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + if ( val ) // if we are not a set + status = status || !UnrollRecordType(&fieldsV, val, ""); + + int valfields = fieldsV.size() - idxfields; + + if ( ! val ) + assert(valfields == 0); + + if ( status ) + { + reporter->Error("Problem unrolling"); + return false; + } + + Field** fields = new Field*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) + fields[i] = fieldsV[i]; + + stream->pred = pred ? pred->AsFunc() : 0; + stream->num_idx_fields = idxfields; + stream->num_val_fields = valfields; + stream->tab = dst->AsTableVal(); + stream->rtype = val ? val->AsRecordType() : 0; + stream->itype = idx->AsRecordType(); + stream->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + stream->currDict = new PDict(InputHash); + stream->currDict->SetDeleteFunc(input_hash_delete_func); + stream->lastDict = new PDict(InputHash); + stream->lastDict->SetDeleteFunc(input_hash_delete_func); + stream->want_record = ( want_record->InternalInt() == 1 ); + + Unref(want_record); // ref'd by lookupwithdefault + Unref(pred); + + if ( valfields > 1 ) + { + if ( ! stream->want_record ) + { + reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", stream->name.c_str()); + delete stream; + return false; + } + } + + + assert(stream->reader); + stream->reader->Init(stream->source, stream->mode, fieldsV.size(), fields ); + + readers[stream->reader] = stream; + + DBG_LOG(DBG_INPUT, "Successfully created table stream %s", + stream->name.c_str()); + + return true; + } + + +bool Manager::IsCompatibleType(BroType* t, bool atomic_only) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + return true; + + case TYPE_RECORD: + return ! atomic_only; + + case TYPE_TABLE: + { + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + } + + case TYPE_VECTOR: + { + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); + } + + default: + return false; + } + + return false; + } + + +bool Manager::RemoveStream(const string &name) + { + Stream *i = FindStream(name); + + if ( i == 0 ) + return false; // not found + + if ( i->removed ) + { + reporter->Error("Stream %s is already queued for removal. Ignoring remove.", name.c_str()); + return false; + } + + i->removed = true; + + i->reader->Close(); + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s", + name.c_str()); +#endif + + return true; + } + +bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) + { + Stream *i = FindStream(reader); + + if ( i == 0 ) + { + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; + } + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", + i->name.c_str()); +#endif + + readers.erase(reader); + delete(i); + + return true; + } + +bool Manager::UnrollRecordType(vector *fields, + const RecordType *rec, const string& nameprepend) + { + + for ( int i = 0; i < rec->NumFields(); i++ ) + { + + if ( ! IsCompatibleType(rec->FieldType(i)) ) + { + reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); + return false; + } + + if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) + { + string prep = nameprepend + rec->FieldName(i) + "."; + + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + { + return false; + } + + } + + else + { + Field* field = new Field(); + field->name = nameprepend + rec->FieldName(i); + field->type = rec->FieldType(i)->Tag(); + + if ( field->type == TYPE_TABLE ) + field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + + else if ( field->type == TYPE_VECTOR ) + field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); + + else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) + { + // we have an annotation for the second column + + Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); + + assert(c); + assert(c->Type()->Tag() == TYPE_STRING); + + field->secondary_name = c->AsStringVal()->AsString()->CheckString(); + } + + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) + field->optional = true; + + fields->push_back(field); + } + } + + return true; + } + +bool Manager::ForceUpdate(const string &name) + { + Stream *i = FindStream(name); + if ( i == 0 ) + { + reporter->Error("Stream %s not found", name.c_str()); + return false; + } + + if ( i->removed ) + { + reporter->Error("Stream %s is already queued for removal. Ignoring force update.", name.c_str()); + return false; + } + + i->reader->Update(); + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Forcing update of stream %s", name.c_str()); +#endif + + return true; // update is async :( +} + + +Val* Manager::RecordValToIndexVal(RecordVal *r) + { + Val* idxval; + + RecordType *type = r->Type()->AsRecordType(); + + int num_fields = type->NumFields(); + + if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) + idxval = r->LookupWithDefault(0); + + else + { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < num_fields; j++ ) + l->Append(r->LookupWithDefault(j)); + + idxval = l; + } + + + return idxval; + } + + +Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) + { + Val* idxval; + int position = 0; + + + if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) + { + idxval = ValueToVal(vals[0], type->FieldType(0)); + position = 1; + } + else + { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < type->NumFields(); j++ ) + { + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) + l->Append(ValueToRecordVal(vals, + type->FieldType(j)->AsRecordType(), &position)); + else + { + l->Append(ValueToVal(vals[position], type->FieldType(j))); + position++; + } + } + idxval = l; + } + + assert ( position == num_fields ); + + return idxval; + } + + +void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) + { + Stream *i = FindStream(reader); + if ( i == 0 ) + { + reporter->InternalError("Unknown reader in SendEntry"); + return; + } + + int readFields = 0; + + if ( i->stream_type == TABLE_STREAM ) + readFields = SendEntryTable(i, vals); + + else if ( i->stream_type == EVENT_STREAM ) + { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + readFields = SendEventStreamEvent(i, type, vals); + } + + else + assert(false); + + for ( int i = 0; i < readFields; i++ ) + delete vals[i]; + + delete [] vals; + } + +int Manager::SendEntryTable(Stream* i, const Value* const *vals) + { + bool updated = false; + + assert(i); + + assert(i->stream_type == TABLE_STREAM); + TableStream* stream = (TableStream*) i; + + HashKey* idxhash = HashValues(stream->num_idx_fields, vals); + + if ( idxhash == 0 ) + { + reporter->Error("Could not hash line. Ignoring"); + return stream->num_val_fields + stream->num_idx_fields; + } + + hash_t valhash = 0; + if ( stream->num_val_fields > 0 ) + { + HashKey* valhashkey = HashValues(stream->num_val_fields, vals+stream->num_idx_fields); + if ( valhashkey == 0 ) + { + // empty line. index, but no values. + // hence we also have no hash value... + } + else + { + valhash = valhashkey->Hash(); + delete(valhashkey); + } + } + + InputHash *h = stream->lastDict->Lookup(idxhash); + if ( h != 0 ) + { + // seen before + if ( stream->num_val_fields == 0 || h->valhash == valhash ) + { + // ok, exact duplicate, move entry to new dicrionary and do nothing else. + stream->lastDict->Remove(idxhash); + stream->currDict->Insert(idxhash, h); + delete idxhash; + return stream->num_val_fields + stream->num_idx_fields; + } + + else + { + assert( stream->num_val_fields > 0 ); + // entry was updated in some way + stream->lastDict->Remove(idxhash); + // keep h for predicates + updated = true; + + } + } + + Val* valval; + RecordVal* predidx = 0; + + int position = stream->num_idx_fields; + + if ( stream->num_val_fields == 0 ) + valval = 0; + + else if ( stream->num_val_fields == 1 && !stream->want_record ) + valval = ValueToVal(vals[position], stream->rtype->FieldType(0)); + + else + valval = ValueToRecordVal(vals, stream->rtype, &position); + + + // call stream first to determine if we really add / change the entry + if ( stream->pred ) + { + EnumVal* ev; + int startpos = 0; + predidx = ValueToRecordVal(vals, stream->itype, &startpos); + + if ( updated ) + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + else + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + + bool result; + if ( stream->num_val_fields > 0 ) // we have values + result = CallPred(stream->pred, 3, ev, predidx->Ref(), valval->Ref()); + else // no values + result = CallPred(stream->pred, 2, ev, predidx->Ref()); + + if ( result == false ) + { + Unref(predidx); + Unref(valval); + + if ( ! updated ) + { + // throw away. Hence - we quit. And remove the entry from the current dictionary... + // (but why should it be in there? assert this). + assert ( stream->currDict->RemoveEntry(idxhash) == 0 ); + delete idxhash; + delete h; + return stream->num_val_fields + stream->num_idx_fields; + } + + else + { + // keep old one + stream->currDict->Insert(idxhash, h); + delete idxhash; + return stream->num_val_fields + stream->num_idx_fields; + } + } + } + + // now we don't need h anymore - if we are here, the entry is updated and a new h is created. + if ( h ) + { + delete h; + h = 0; + } + + + Val* idxval; + if ( predidx != 0 ) + { + idxval = RecordValToIndexVal(predidx); + // I think there is an unref missing here. But if I insert is, it crashes :) + } + else + idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); + + Val* oldval = 0; + if ( updated == true ) + { + assert(stream->num_val_fields > 0); + // in that case, we need the old value to send the event (if we send an event). + oldval = stream->tab->Lookup(idxval, false); + } + + assert(idxval); + HashKey* k = stream->tab->ComputeHash(idxval); + if ( ! k ) + reporter->InternalError("could not hash"); + + InputHash* ih = new InputHash(); + ih->idxkey = new HashKey(k->Key(), k->Size(), k->Hash()); + ih->valhash = valhash; + + if ( stream->event && updated ) + Ref(oldval); // otherwise it is no longer accessible after the assignment + + stream->tab->Assign(idxval, k, valval); + Unref(idxval); // asssign does not consume idxval. + + if ( predidx != 0 ) + Unref(predidx); + + stream->currDict->Insert(idxhash, ih); + delete idxhash; + + if ( stream->event ) + { + EnumVal* ev; + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); + + if ( updated ) + { // in case of update send back the old value. + assert ( stream->num_val_fields > 0 ); + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + assert ( oldval != 0 ); + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, oldval); + } + + else + { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + if ( stream->num_val_fields == 0 ) + { + Ref(stream->description); + SendEvent(stream->event, 3, stream->description->Ref(), ev, predidx); + } + else + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, valval->Ref()); + + } + } + + return stream->num_val_fields + stream->num_idx_fields; + } + +void Manager::EndCurrentSend(ReaderFrontend* reader) + { + Stream *i = FindStream(reader); + + if ( i == 0 ) + { + reporter->InternalError("Unknown reader in EndCurrentSend"); + return; + } + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", i->name.c_str()); +#endif + + if ( i->stream_type == EVENT_STREAM ) // nothing to do.. + return; + + assert(i->stream_type == TABLE_STREAM); + TableStream* stream = (TableStream*) i; + + // lastdict contains all deleted entries and should be empty apart from that + IterCookie *c = stream->lastDict->InitForIteration(); + stream->lastDict->MakeRobustCookie(c); + InputHash* ih; + HashKey *lastDictIdxKey; + + while ( ( ih = stream->lastDict->NextEntry(lastDictIdxKey, c) ) ) + { + ListVal * idx = 0; + Val *val = 0; + + Val* predidx = 0; + EnumVal* ev = 0; + int startpos = 0; + + if ( stream->pred || stream->event ) + { + idx = stream->tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + val = stream->tab->Lookup(idx); + assert(val != 0); + predidx = ListValToRecordVal(idx, stream->itype, &startpos); + Unref(idx); + ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + } + + if ( stream->pred ) + { + // ask predicate, if we want to expire this element... + + Ref(ev); + Ref(predidx); + Ref(val); + + bool result = CallPred(stream->pred, 3, ev, predidx, val); + + if ( result == false ) + { + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + Unref(predidx); + Unref(ev); + stream->currDict->Insert(lastDictIdxKey, stream->lastDict->RemoveEntry(lastDictIdxKey)); + delete lastDictIdxKey; + continue; + } + } + + if ( stream->event ) + { + Ref(predidx); + Ref(val); + Ref(ev); + SendEvent(stream->event, 3, ev, predidx, val); + } + + if ( predidx ) // if we have a stream or an event... + Unref(predidx); + + if ( ev ) + Unref(ev); + + Unref(stream->tab->Delete(ih->idxkey)); + stream->lastDict->Remove(lastDictIdxKey); // delete in next line + delete lastDictIdxKey; + delete(ih); + } + + stream->lastDict->Clear(); // should be empt. buti- well... who knows... + delete(stream->lastDict); + + stream->lastDict = stream->currDict; + stream->currDict = new PDict(InputHash); + stream->currDict->SetDeleteFunc(input_hash_delete_func); + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", + i->name.c_str()); +#endif + + // Send event that the current update is indeed finished. + EventHandler* handler = event_registry->Lookup("Input::update_finished"); + if ( handler == 0 ) + reporter->InternalError("Input::update_finished not found!"); + + SendEvent(handler, 2, new StringVal(i->name.c_str()), new StringVal(i->source.c_str())); + } + +void Manager::Put(ReaderFrontend* reader, Value* *vals) + { + Stream *i = FindStream(reader); + if ( i == 0 ) + { + reporter->InternalError("Unknown reader in Put"); + return; + } + + int readFields = 0; + + if ( i->stream_type == TABLE_STREAM ) + readFields = PutTable(i, vals); + + else if ( i->stream_type == EVENT_STREAM ) + { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + readFields = SendEventStreamEvent(i, type, vals); + } + + else + assert(false); + + for ( int i = 0; i < readFields; i++ ) + delete vals[i]; + + delete [] vals; + } + +int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) + { + assert(i); + + assert(i->stream_type == EVENT_STREAM); + EventStream* stream = (EventStream*) i; + + Val *val; + list out_vals; + Ref(stream->description); + out_vals.push_back(stream->description); + // no tracking, send everything with a new event... + out_vals.push_back(type); + + int position = 0; + + if ( stream->want_record ) + { + RecordVal * r = ValueToRecordVal(vals, stream->fields, &position); + out_vals.push_back(r); + } + + else + { + for ( int j = 0; j < stream->fields->NumFields(); j++) + { + Val* val = 0; + + if ( stream->fields->FieldType(j)->Tag() == TYPE_RECORD ) + val = ValueToRecordVal(vals, + stream->fields->FieldType(j)->AsRecordType(), + &position); + + else + { + val = ValueToVal(vals[position], stream->fields->FieldType(j)); + position++; + } + + out_vals.push_back(val); + } + } + + SendEvent(stream->event, out_vals); + + return stream->fields->NumFields(); + } + +int Manager::PutTable(Stream* i, const Value* const *vals) + { + assert(i); + + assert(i->stream_type == TABLE_STREAM); + TableStream* stream = (TableStream*) i; + + Val* idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); + Val* valval; + + int position = stream->num_idx_fields; + + if ( stream->num_val_fields == 0 ) + valval = 0; + + else if ( stream->num_val_fields == 1 && stream->want_record == 0 ) + valval = ValueToVal(vals[position], stream->rtype->FieldType(0)); + else + valval = ValueToRecordVal(vals, stream->rtype, &position); + + // if we have a subscribed event, we need to figure out, if this is an update or not + // same for predicates + if ( stream->pred || stream->event ) + { + bool updated = false; + Val* oldval = 0; + + if ( stream->num_val_fields > 0 ) + { + // in that case, we need the old value to send the event (if we send an event). + oldval = stream->tab->Lookup(idxval, false); + } + + if ( oldval != 0 ) + { + // it is an update + updated = true; + Ref(oldval); // have to do that, otherwise it may disappear in assign + } + + + // predicate if we want the update or not + if ( stream->pred ) + { + EnumVal* ev; + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); + Ref(valval); + + if ( updated ) + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + BifType::Enum::Input::Event); + else + ev = new EnumVal(BifEnum::Input::EVENT_NEW, + BifType::Enum::Input::Event); + + bool result; + if ( stream->num_val_fields > 0 ) // we have values + result = CallPred(stream->pred, 3, ev, predidx, valval); + else // no values + result = CallPred(stream->pred, 2, ev, predidx); + + if ( result == false ) + { + // do nothing + Unref(idxval); + Unref(valval); + Unref(oldval); + return stream->num_val_fields + stream->num_idx_fields; + } + + } + + stream->tab->Assign(idxval, valval); + + if ( stream->event ) + { + EnumVal* ev; + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); + + if ( updated ) + { + // in case of update send back the old value. + assert ( stream->num_val_fields > 0 ); + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + BifType::Enum::Input::Event); + assert ( oldval != 0 ); + SendEvent(stream->event, 4, stream->description->Ref(), + ev, predidx, oldval); + } + else + { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, + BifType::Enum::Input::Event); + if ( stream->num_val_fields == 0 ) + SendEvent(stream->event, 4, stream->description->Ref(), + ev, predidx); + else + SendEvent(stream->event, 4, stream->description->Ref(), + ev, predidx, valval->Ref()); + } + + } + + } + + else // no predicates or other stuff + stream->tab->Assign(idxval, valval); + + Unref(idxval); // not consumed by assign + + return stream->num_idx_fields + stream->num_val_fields; + } + +// Todo:: perhaps throw some kind of clear-event? +void Manager::Clear(ReaderFrontend* reader) + { + Stream *i = FindStream(reader); + if ( i == 0 ) + { + reporter->InternalError("Unknown reader in Clear"); + return; + } + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Got Clear for stream %s", + i->name.c_str()); +#endif + + assert(i->stream_type == TABLE_STREAM); + TableStream* stream = (TableStream*) i; + + stream->tab->RemoveAll(); + } + +// put interface: delete old entry from table. +bool Manager::Delete(ReaderFrontend* reader, Value* *vals) + { + Stream *i = FindStream(reader); + if ( i == 0 ) + { + reporter->InternalError("Unknown reader in Delete"); + return false; + } + + bool success = false; + int readVals = 0; + + if ( i->stream_type == TABLE_STREAM ) + { + TableStream* stream = (TableStream*) i; + Val* idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); + assert(idxval != 0); + readVals = stream->num_idx_fields + stream->num_val_fields; + bool streamresult = true; + + if ( stream->pred || stream->event ) + { + Val *val = stream->tab->Lookup(idxval); + + if ( stream->pred ) + { + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); + + streamresult = CallPred(stream->pred, 3, ev, predidx, val); + + if ( streamresult == false ) + { + // keep it. + Unref(idxval); + success = true; + } + + } + + // only if stream = true -> no streaming + if ( streamresult && stream->event ) + { + Ref(idxval); + assert(val != 0); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(stream->event, 4, stream->description->Ref(), ev, idxval, val); + } + } + + // only if stream = true -> no streaming + if ( streamresult ) + { + Val* retptr = stream->tab->Delete(idxval); + success = ( retptr != 0 ); + if ( ! success ) + reporter->Error("Internal error while deleting values from input table"); + else + Unref(retptr); + } + + } + + else if ( i->stream_type == EVENT_STREAM ) + { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + readVals = SendEventStreamEvent(i, type, vals); + success = true; + } + + else + { + assert(false); + return false; + } + + for ( int i = 0; i < readVals; i++ ) + delete vals[i]; + + delete [] vals; + + return success; + } + +bool Manager::CallPred(Func* pred_func, const int numvals, ...) + { + bool result; + val_list vl(numvals); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + vl.append( va_arg(lP, Val*) ); + + va_end(lP); + + Val* v = pred_func->Call(&vl); + result = v->AsBool(); + Unref(v); + + return(result); + } + +bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) + { + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) + { + reporter->Error("Event %s not found", name.c_str()); + return false; + } + + RecordType *type = handler->FType()->Args(); + int num_event_vals = type->NumFields(); + if ( num_vals != num_event_vals ) + { + reporter->Error("Wrong number of values for event %s", name.c_str()); + return false; + } + + val_list* vl = new val_list; + for ( int i = 0; i < num_vals; i++) + vl->append(ValueToVal(vals[i], type->FieldType(i))); + + mgr.Dispatch(new Event(handler, vl)); + + for ( int i = 0; i < num_vals; i++ ) + delete vals[i]; + + delete [] vals; + + return true; +} + +void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) + { + val_list* vl = new val_list; + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + vl->append( va_arg(lP, Val*) ); + + va_end(lP); + + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); + } + +void Manager::SendEvent(EventHandlerPtr ev, list events) + { + val_list* vl = new val_list; + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) + { + vl->append( *i ); + } + + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); + } + +// Convert a bro list value to a bro record value. +// I / we could think about moving this functionality to val.cc +RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) + { + assert(position != 0 ); // we need the pointer to point to data; + + if ( request_type->Tag() != TYPE_RECORD ) + { + reporter->InternalError("ListValToRecordVal called on non-record-value."); + return 0; + } + + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + + assert(list != 0); + int maxpos = list->Length(); + + for ( int i = 0; i < request_type->NumFields(); i++ ) + { + assert ( (*position) <= maxpos ); + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) + fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); + else + { + fieldVal = list->Index(*position); + (*position)++; + } + + rec->Assign(i, fieldVal->Ref()); + } + + return rec; + } + +// Convert a threading value to a record value +RecordVal* Manager::ValueToRecordVal(const Value* const *vals, + RecordType *request_type, int* position) + { + assert(position != 0); // we need the pointer to point to data. + + if ( request_type->Tag() != TYPE_RECORD ) + { + reporter->InternalError("ValueToRecordVal called on non-record-value."); + return 0; + } + + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + for ( int i = 0; i < request_type->NumFields(); i++ ) + { + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) + fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + else + { + fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; + } + +// Count the length of the values used to create a correct length buffer for +// hashing later +int Manager::GetValueLength(const Value* val) { + assert( val->present ); // presence has to be checked elsewhere + int length = 0; + + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + length += sizeof(val->val.uint_val); + break; + + case TYPE_PORT: + length += sizeof(val->val.port_val.port); + length += sizeof(val->val.port_val.proto); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + { + switch ( val->val.addr_val.family ) { + case IPv4: + length += sizeof(val->val.addr_val.in.in4); + break; + case IPv6: + length += sizeof(val->val.addr_val.in.in6); + break; + default: + assert(false); + } + } + break; + + case TYPE_SUBNET: + { + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + length += sizeof(val->val.subnet_val.prefix.in.in4)+ + sizeof(val->val.subnet_val.length); + break; + case IPv6: + length += sizeof(val->val.subnet_val.prefix.in.in6)+ + sizeof(val->val.subnet_val.length); + break; + default: + assert(false); + } + } + break; + + case TYPE_TABLE: + { + for ( int i = 0; i < val->val.set_val.size; i++ ) + length += GetValueLength(val->val.set_val.vals[i]); + break; + } + + case TYPE_VECTOR: + { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) + length += GetValueLength(val->val.vector_val.vals[i]); + break; + } + + default: + reporter->InternalError("unsupported type %d for GetValueLength", val->type); + } + + return length; + +} + +// Given a threading::value, copy the raw data bytes into *data and return how many bytes were copied. +// Used for hashing the values for lookup in the bro table +int Manager::CopyValue(char *data, const int startpos, const Value* val) + { + assert( val->present ); // presence has to be checked elsewhere + + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + return sizeof(val->val.int_val); + + case TYPE_COUNT: + case TYPE_COUNTER: + memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); + return sizeof(val->val.uint_val); + + case TYPE_PORT: + { + int length = 0; + memcpy(data+startpos, (const void*) &(val->val.port_val.port), + sizeof(val->val.port_val.port)); + length += sizeof(val->val.port_val.port); + memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), + sizeof(val->val.port_val.proto)); + length += sizeof(val->val.port_val.proto); + return length; + } + + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + memcpy(data+startpos, (const void*) &(val->val.double_val), + sizeof(val->val.double_val)); + return sizeof(val->val.double_val); + + case TYPE_STRING: + case TYPE_ENUM: + { + memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length()); + return val->val.string_val->size(); + } + + case TYPE_ADDR: + { + int length; + switch ( val->val.addr_val.family ) { + case IPv4: + length = sizeof(val->val.addr_val.in.in4); + memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in4), length); + break; + + case IPv6: + length = sizeof(val->val.addr_val.in.in6); + memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in6), length); + break; + + default: + assert(false); + } + + return length; + } + + case TYPE_SUBNET: + { + int length; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + length = sizeof(val->val.addr_val.in.in4); + memcpy(data + startpos, + (const char*) &(val->val.subnet_val.prefix.in.in4), length); + break; + + case IPv6: + length = sizeof(val->val.addr_val.in.in6); + memcpy(data + startpos, + (const char*) &(val->val.subnet_val.prefix.in.in4), length); + break; + + default: + assert(false); + } + + int lengthlength = sizeof(val->val.subnet_val.length); + memcpy(data + startpos + length , + (const char*) &(val->val.subnet_val.length), lengthlength); + length += lengthlength; + + return length; + } + + case TYPE_TABLE: + { + int length = 0; + int j = val->val.set_val.size; + for ( int i = 0; i < j; i++ ) + length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); + + return length; + } + + case TYPE_VECTOR: + { + int length = 0; + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) + length += CopyValue(data, startpos+length, val->val.vector_val.vals[i]); + + return length; + } + + default: + reporter->InternalError("unsupported type %d for CopyValue", val->type); + return 0; + } + + assert(false); + return 0; + } + +// Hash num_elements threading values and return the HashKey for them. At least one of the vals has to be ->present. +HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) + { + int length = 0; + + for ( int i = 0; i < num_elements; i++ ) + { + const Value* val = vals[i]; + if ( val->present ) + length += GetValueLength(val); + } + + if ( length == 0 ) + { + reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); + return NULL; + } + + int position = 0; + char *data = (char*) malloc(length); + if ( data == 0 ) + reporter->InternalError("Could not malloc?"); + + for ( int i = 0; i < num_elements; i++ ) + { + const Value* val = vals[i]; + if ( val->present ) + position += CopyValue(data, position, val); + } + + HashKey *key = new HashKey(data, length); + delete data; + + assert(position == length); + return key; + } + +// convert threading value to Bro value +Val* Manager::ValueToVal(const Value* val, BroType* request_type) + { + + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) + { + reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); + return 0; + } + + if ( !val->present ) + return 0; // unset field + + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + return new Val(val->val.int_val, val->type); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + return new Val(val->val.uint_val, val->type); + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return new Val(val->val.double_val, val->type); + + case TYPE_STRING: + { + BroString *s = new BroString(*(val->val.string_val)); + return new StringVal(s); + } + + case TYPE_PORT: + return new PortVal(val->val.port_val.port, val->val.port_val.proto); + + case TYPE_ADDR: + { + IPAddr* addr; + switch ( val->val.addr_val.family ) { + case IPv4: + addr = new IPAddr(val->val.addr_val.in.in4); + break; + + case IPv6: + addr = new IPAddr(val->val.addr_val.in.in6); + break; + + default: + assert(false); + } + + AddrVal* addrval = new AddrVal(*addr); + delete addr; + return addrval; + } + + case TYPE_SUBNET: + { + IPAddr* addr; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + addr = new IPAddr(val->val.subnet_val.prefix.in.in4); + break; + + case IPv6: + addr = new IPAddr(val->val.subnet_val.prefix.in.in6); + break; + + default: + assert(false); + } + + SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); + delete addr; + return subnetval; + } + + case TYPE_TABLE: + { + // all entries have to have the same type... + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) + { + Val* assignval = ValueToVal( val->val.set_val.vals[i], type ); + t->Assign(assignval, 0); + Unref(assignval); // idex is not consumed by assign. + } + + Unref(s); + return t; + } + + case TYPE_VECTOR: + { + // all entries have to have the same type... + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) + v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); + + Unref(vt); + return v; + } + + case TYPE_ENUM: { + // well, this is kind of stupid, because EnumType just mangles the module name and the var name together again... + // but well + string module = extract_module_name(val->val.string_val->c_str()); + string var = extract_var_name(val->val.string_val->c_str()); + bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); + if ( index == -1 ) + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", + module.c_str(), var.c_str()); + + return new EnumVal(index, request_type->Ref()->AsEnumType() ); + } + + + default: + reporter->InternalError("unsupported type for input_read"); + } + + assert(false); + return NULL; + } + +Manager::Stream* Manager::FindStream(const string &name) + { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s).second->name == name ) + return (*s).second; + } + + return 0; + } + +Manager::Stream* Manager::FindStream(ReaderFrontend* reader) + { + map::iterator s = readers.find(reader); + if ( s != readers.end() ) + return s->second; + + return 0; + } diff --git a/src/input/Manager.h b/src/input/Manager.h new file mode 100644 index 0000000000..269e562e23 --- /dev/null +++ b/src/input/Manager.h @@ -0,0 +1,195 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Class for managing input streams. + +#ifndef INPUT_MANAGER_H +#define INPUT_MANAGER_H + +#include "BroString.h" +#include "EventHandler.h" +#include "RemoteSerializer.h" +#include "Val.h" + +#include + +namespace input { + +class ReaderFrontend; +class ReaderBackend; + +/** + * Singleton class for managing input streams. + */ +class Manager { +public: + /** + * Constructor. + */ + Manager(); + + /** + * Destructor. + */ + ~Manager(); + + /** + * Creates a new input stream which will write the data from the data + * source into a table. + * + * @param description A record of script type \c + * Input:StreamDescription. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ + bool CreateTableStream(RecordVal* description); + + /** + * Creates a new input stream which sends events for read input data. + * + * @param description A record of script type \c + * Input:StreamDescription. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ + bool CreateEventStream(RecordVal* description); + + /** + * Force update on a input stream. Forces a re-read of the whole + * input source. Usually used when an input stream is opened in + * managed mode. Otherwise, this can be used to trigger a input + * source check before a heartbeat message arrives. May be ignored by + * the reader. + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ + bool ForceUpdate(const string &id); + + /** + * Deletes an existing input stream. + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ + bool RemoveStream(const string &id); + +protected: + friend class ReaderFrontend; + friend class PutMessage; + friend class DeleteMessage; + friend class ClearMessage; + friend class SendEventMessage; + friend class SendEntryMessage; + friend class EndCurrentSendMessage; + friend class ReaderClosedMessage; + + // For readers to write to input stream in direct mode (reporting + // new/deleted values directly). Functions take ownership of + // threading::Value fields. + void Put(ReaderFrontend* reader, threading::Value* *vals); + void Clear(ReaderFrontend* reader); + bool Delete(ReaderFrontend* reader, threading::Value* *vals); + + // For readers to write to input stream in indirect mode (manager is + // monitoring new/deleted values) Functions take ownership of + // threading::Value fields. + void SendEntry(ReaderFrontend* reader, threading::Value* *vals); + void EndCurrentSend(ReaderFrontend* reader); + + // Allows readers to directly send Bro events. The num_vals and vals + // must be the same the named event expects. Takes ownership of + // threading::Value fields. + bool SendEvent(const string& name, const int num_vals, threading::Value* *vals); + + // Instantiates a new ReaderBackend of the given type (note that + // doing so creates a new thread!). + ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + + // Function called from the ReaderBackend to notify the manager that + // a stream has been removed or a stream has been closed. Used to + // prevent race conditions where data for a specific stream is still + // in the queue when the RemoveStream directive is executed by the + // main thread. This makes sure all data that has ben queued for a + // stream is still received. + bool RemoveStreamContinuation(ReaderFrontend* reader); + +private: + class Stream; + class TableStream; + class EventStream; + + bool CreateStream(Stream*, RecordVal* description); + + // SendEntry implementation for Table stream. + int SendEntryTable(Stream* i, const threading::Value* const *vals); + + // Put implementation for Table stream. + int PutTable(Stream* i, const threading::Value* const *vals); + + // SendEntry and Put implementation for Event stream. + int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const *vals); + + // Checks that a Bro type can be used for data reading. The + // equivalend in threading cannot be used, because we have support + // different types from the log framework + bool IsCompatibleType(BroType* t, bool atomic_only=false); + + // Check if a record is made up of compatible types and return a list + // of all fields that are in the record in order. Recursively unrolls + // records + bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + + // Send events + void SendEvent(EventHandlerPtr ev, const int numvals, ...); + void SendEvent(EventHandlerPtr ev, list events); + + // Call predicate function and return result. + bool CallPred(Func* pred_func, const int numvals, ...); + + // Get a hashkey for a set of threading::Values. + HashKey* HashValues(const int num_elements, const threading::Value* const *vals); + + // Get the memory used by a specific value. + int GetValueLength(const threading::Value* val); + + // Copies the raw data in a specific threading::Value to position + // startpos. + int CopyValue(char *data, const int startpos, const threading::Value* val); + + // Convert Threading::Value to an internal Bro Type (works also with + // Records). + Val* ValueToVal(const threading::Value* val, BroType* request_type); + + // Convert Threading::Value to an internal Bro List type. + Val* ValueToIndexVal(int num_fields, const RecordType* type, const threading::Value* const *vals); + + // Converts a threading::value to a record type. Mostly used by + // ValueToVal. + RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position); + + Val* RecordValToIndexVal(RecordVal *r); + + // Converts a Bro ListVal to a RecordVal given the record type. + RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); + + Stream* FindStream(const string &name); + Stream* FindStream(ReaderFrontend* reader); + + enum StreamType { TABLE_STREAM, EVENT_STREAM }; + + map readers; +}; + + +} + +extern input::Manager* input_mgr; + + +#endif /* INPUT_MANAGER_H */ diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc new file mode 100644 index 0000000000..b5d898fedd --- /dev/null +++ b/src/input/ReaderBackend.cc @@ -0,0 +1,292 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "ReaderBackend.h" +#include "ReaderFrontend.h" +#include "Manager.h" + +using threading::Value; +using threading::Field; + +namespace input { + +class PutMessage : public threading::OutputMessage { +public: + PutMessage(ReaderFrontend* reader, Value* *val) + : threading::OutputMessage("Put", reader), + val(val) {} + + virtual bool Process() + { + input_mgr->Put(Object(), val); + return true; + } + +private: + Value* *val; +}; + +class DeleteMessage : public threading::OutputMessage { +public: + DeleteMessage(ReaderFrontend* reader, Value* *val) + : threading::OutputMessage("Delete", reader), + val(val) {} + + virtual bool Process() + { + return input_mgr->Delete(Object(), val); + } + +private: + Value* *val; +}; + +class ClearMessage : public threading::OutputMessage { +public: + ClearMessage(ReaderFrontend* reader) + : threading::OutputMessage("Clear", reader) {} + + virtual bool Process() + { + input_mgr->Clear(Object()); + return true; + } + +private: +}; + +class SendEventMessage : public threading::OutputMessage { +public: + SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, Value* *val) + : threading::OutputMessage("SendEvent", reader), + name(name), num_vals(num_vals), val(val) {} + + virtual bool Process() + { + bool success = input_mgr->SendEvent(name, num_vals, val); + + if ( ! success ) + reporter->Error("SendEvent for event %s failed", name.c_str()); + + return true; // We do not want to die if sendEvent fails because the event did not return. + } + +private: + const string name; + const int num_vals; + Value* *val; +}; + +class SendEntryMessage : public threading::OutputMessage { +public: + SendEntryMessage(ReaderFrontend* reader, Value* *val) + : threading::OutputMessage("SendEntry", reader), + val(val) { } + + virtual bool Process() + { + input_mgr->SendEntry(Object(), val); + return true; + } + +private: + Value* *val; +}; + +class EndCurrentSendMessage : public threading::OutputMessage { +public: + EndCurrentSendMessage(ReaderFrontend* reader) + : threading::OutputMessage("EndCurrentSend", reader) {} + + virtual bool Process() + { + input_mgr->EndCurrentSend(Object()); + return true; + } + +private: +}; + +class ReaderClosedMessage : public threading::OutputMessage { +public: + ReaderClosedMessage(ReaderFrontend* reader) + : threading::OutputMessage("ReaderClosed", reader) {} + + virtual bool Process() + { + return input_mgr->RemoveStreamContinuation(Object()); + } + +private: +}; + + +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(ReaderFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() + { + Object()->SetDisable(); + return true; + } +}; + + +ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() + { + disabled = true; // disabled will be set correcty in init. + frontend = arg_frontend; + + SetName(frontend->Name()); + } + +ReaderBackend::~ReaderBackend() + { + } + +void ReaderBackend::Put(Value* *val) + { + SendOut(new PutMessage(frontend, val)); + } + +void ReaderBackend::Delete(Value* *val) + { + SendOut(new DeleteMessage(frontend, val)); + } + +void ReaderBackend::Clear() + { + SendOut(new ClearMessage(frontend)); + } + +void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) + { + SendOut(new SendEventMessage(frontend, name, num_vals, vals)); + } + +void ReaderBackend::EndCurrentSend() + { + SendOut(new EndCurrentSendMessage(frontend)); + } + +void ReaderBackend::SendEntry(Value* *vals) + { + SendOut(new SendEntryMessage(frontend, vals)); + } + +bool ReaderBackend::Init(string arg_source, ReaderMode arg_mode, const int arg_num_fields, + const threading::Field* const* arg_fields) + { + source = arg_source; + mode = arg_mode; + num_fields = arg_num_fields; + fields = arg_fields; + + SetName("InputReader/"+source); + + // disable if DoInit returns error. + int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); + + if ( ! success ) + { + Error("Init failed"); + DisableFrontend(); + } + + disabled = !success; + + return success; + } + +void ReaderBackend::Close() + { + DoClose(); + disabled = true; + DisableFrontend(); + SendOut(new ReaderClosedMessage(frontend)); + + if ( fields != 0 ) + { + for ( unsigned int i = 0; i < num_fields; i++ ) + delete(fields[i]); + + delete [] (fields); + fields = 0; + } + } + +bool ReaderBackend::Update() + { + if ( disabled ) + return false; + + bool success = DoUpdate(); + if ( ! success ) + DisableFrontend(); + + return success; + } + +void ReaderBackend::DisableFrontend() + { + // We also set disabled here, because there still may be other + // messages queued and we will dutifully ignore these from now. + disabled = true; + SendOut(new DisableMessage(frontend)); + } + +bool ReaderBackend::DoHeartbeat(double network_time, double current_time) + { + MsgThread::DoHeartbeat(network_time, current_time); + return true; + } + +TransportProto ReaderBackend::StringToProto(const string &proto) + { + if ( proto == "unknown" ) + return TRANSPORT_UNKNOWN; + else if ( proto == "tcp" ) + return TRANSPORT_TCP; + else if ( proto == "udp" ) + return TRANSPORT_UDP; + else if ( proto == "icmp" ) + return TRANSPORT_ICMP; + + Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); + + return TRANSPORT_UNKNOWN; + } + + +// More or less verbose copy from IPAddr.cc -- which uses reporter. +Value::addr_t ReaderBackend::StringToAddr(const string &s) + { + Value::addr_t val; + + if ( s.find(':') == std::string::npos ) // IPv4. + { + val.family = IPv4; + + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + { + Error(Fmt("Bad address: %s", s.c_str())); + memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); + } + } + + else + { + val.family = IPv6; + if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) + { + Error(Fmt("Bad address: %s", s.c_str())); + memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); + } + } + + return val; + } + +} diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h new file mode 100644 index 0000000000..8b5e7d674b --- /dev/null +++ b/src/input/ReaderBackend.h @@ -0,0 +1,285 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERBACKEND_H +#define INPUT_READERBACKEND_H + +#include "BroString.h" + +#include "threading/SerialTypes.h" +#include "threading/MsgThread.h" + +namespace input { + +/** + * The modes a reader can be in. + */ +enum ReaderMode { + /** + * TODO Bernhard. + */ + MODE_MANUAL, + + /** + * TODO Bernhard. + */ + MODE_REREAD, + + /** + * TODO Bernhard. + */ + MODE_STREAM +}; + +class ReaderFrontend; + +/** + * Base class for reader implementation. When the input:Manager creates a new + * input stream, it instantiates a ReaderFrontend. That then in turn creates + * a ReaderBackend of the right type. The frontend then forwards messages + * over the backend as its methods are called. + * + * All methods must be called only from the corresponding child thread (the + * constructor is the one exception.) + */ +class ReaderBackend : public threading::MsgThread { +public: + /** + * Constructor. + * + * @param frontend The frontend reader that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * an argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + */ + ReaderBackend(ReaderFrontend* frontend); + + /** + * Destructor. + */ + virtual ~ReaderBackend(); + + /** + * One-time initialization of the reader to define the input source. + * + * @param source A string left to the interpretation of the + * reader implementation; it corresponds to the value configured on + * the script-level for the input stream. + * + * @param mode The opening mode for the input source. + * + * @param num_fields Number of fields contained in \a fields. + * + * @param fields The types and names of the fields to be retrieved + * from the input source. + * + * @return False if an error occured. + */ + bool Init(string source, ReaderMode mode, int num_fields, const threading::Field* const* fields); + + /** + * Finishes reading from this input stream in a regular fashion. Must + * not be called if an error has been indicated earlier. After + * calling this, no further reading from the stream can be performed. + * + * @return False if an error occured. + */ + void Close(); + + /** + * Force trigger an update of the input stream. The action that will + * be taken depends on the current read mode and the individual input + * backend. + * + * An backend can choose to ignore this. + * + * @return False if an error occured. + */ + bool Update(); + + /** + * Disables the frontend that has instantiated this backend. Once + * disabled, the frontend will not send any further message over. + */ + void DisableFrontend(); + +protected: + // Methods that have to be overwritten by the individual readers + + /** + * Reader-specific intialization method. Note that data may only be + * read from the input source after the Init() function has been + * called. + * + * A reader implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the reader from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + * + * Arguments are the same as Init(). + * + * Note that derived classes don't need to store the values passed in + * here if other methods need them to; the \a ReaderBackend class + * provides accessor methods to get them later, and they are passed + * in here only for convinience. + */ + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields) = 0; + + /** + * Reader-specific method implementing input finalization at + * termination. + * + * A reader implementation must override this method but it can just + * ignore calls if an input source can't actually be closed. + * + * After the method is called, the writer will be deleted. If an + * error occurs during shutdown, an implementation should also call + * Error() to indicate what happened. + */ + virtual void DoClose() = 0; + + /** + * Reader-specific method implementing the forced update trigger. + * + * A reader implementation must override this method but it can just + * ignore calls if a forced update does not fit the input source or + * the current input reading mode. + * + * If it returns false, it will be assumed that a fatal error has + * occured that prevents the reader from further operation; it will + * then be disabled and eventually deleted. When returning false, an + * implementation should also call Error to indicate what happened. + */ + virtual bool DoUpdate() = 0; + + /** + * Returns the input source as passed into Init()/. + */ + const string Source() const { return source; } + + /** + * Returns the reader mode as passed into Init(). + */ + const ReaderMode Mode() const { return mode; } + + /** + * Returns the number of log fields as passed into Init(). + */ + unsigned int NumFields() const { return num_fields; } + + /** + * Returns the log fields as passed into Init(). + */ + const threading::Field* const * Fields() const { return fields; } + + /** + * Method allowing a reader to send a specified Bro event. Vals must + * match the values expected by the bro event. + * + * @param name name of the bro event to send + * + * @param num_vals number of entries in \a vals + * + * @param vals the values to be given to the event + */ + void SendEvent(const string& name, const int num_vals, threading::Value* *vals); + + // Content-sending-functions (simple mode). Include table-specific + // functionality that simply is not used if we have no table. + + /** + * Method allowing a reader to send a list of values read from a + * specific stream back to the manager in simple mode. + * + * If the stream is a table stream, the values are inserted into the + * table; if it is an event stream, the event is raised. + * + * @param val Array of threading::Values expected by the stream. The + * array must have exactly NumEntries() elements. + */ + void Put(threading::Value** val); + + /** + * Method allowing a reader to delete a specific value from a Bro + * table. + * + * If the receiving stream is an event stream, only a removed event + * is raised. + * + * @param val Array of threading::Values expected by the stream. The + * array must have exactly NumEntries() elements. + */ + void Delete(threading::Value** val); + + /** + * Method allowing a reader to clear a Bro table. + * + * If the receiving stream is an event stream, this is ignored. + * + */ + void Clear(); + + // Content-sending-functions (tracking mode): Only changed lines are propagated. + + /** + * Method allowing a reader to send a list of values read from + * specific stream back to the manager in tracking mode. + * + * If the stream is a table stream, the values are inserted into the + * table; if it is an event stream, the event is raised. + * + * @param val Array of threading::Values expected by the stream. The + * array must have exactly NumEntries() elements. + */ + void SendEntry(threading::Value** vals); + + /** + * Method telling the manager, that the current list of entries sent + * by SendEntry is finished. + * + * For table streams, all entries that were not updated since the + * last EndCurrentSend will be deleted, because they are no longer + * present in the input source + */ + void EndCurrentSend(); + + /** + * Triggered by regular heartbeat messages from the main thread. + * + * This method can be overridden but once must call + * ReaderBackend::DoHeartbeat(). + */ + virtual bool DoHeartbeat(double network_time, double current_time); + + /** + * Convert a string into a TransportProto. This is just a utility + * function for Readers. + * + * @param proto the transport protocol + */ + TransportProto StringToProto(const string &proto); + + /** + * Convert a string into a Value::addr_t. This is just a utility + * function for Readers. + * + * @param addr containing an ipv4 or ipv6 address + */ + threading::Value::addr_t StringToAddr(const string &addr); + +private: + // Frontend that instantiated us. This object must not be accessed + // from this class, it's running in a different thread! + ReaderFrontend* frontend; + + string source; + ReaderMode mode; + unsigned int num_fields; + const threading::Field* const * fields; // raw mapping + + bool disabled; +}; + +} + +#endif /* INPUT_READERBACKEND_H */ diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc new file mode 100644 index 0000000000..0236ac83be --- /dev/null +++ b/src/input/ReaderFrontend.cc @@ -0,0 +1,121 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Manager.h" +#include "ReaderFrontend.h" +#include "ReaderBackend.h" + +#include "threading/MsgThread.h" + +// FIXME: cleanup of disabled inputreaders is missing. we need this, because +// stuff can e.g. fail in init and might never be removed afterwards. + +namespace input { + +class InitMessage : public threading::InputMessage +{ +public: + InitMessage(ReaderBackend* backend, const string source, ReaderMode mode, + const int num_fields, const threading::Field* const* fields) + : threading::InputMessage("Init", backend), + source(source), mode(mode), num_fields(num_fields), fields(fields) { } + + virtual bool Process() + { + return Object()->Init(source, mode, num_fields, fields); + } + +private: + const string source; + const ReaderMode mode; + const int num_fields; + const threading::Field* const* fields; +}; + +class UpdateMessage : public threading::InputMessage +{ +public: + UpdateMessage(ReaderBackend* backend) + : threading::InputMessage("Update", backend) + { } + + virtual bool Process() { return Object()->Update(); } +}; + +class CloseMessage : public threading::InputMessage +{ +public: + CloseMessage(ReaderBackend* backend) + : threading::InputMessage("Close", backend) + { } + + virtual bool Process() { Object()->Close(); return true; } +}; + + +ReaderFrontend::ReaderFrontend(bro_int_t type) + { + disabled = initialized = false; + ty_name = ""; + backend = input_mgr->CreateBackend(this, type); + + assert(backend); + backend->Start(); + } + +ReaderFrontend::~ReaderFrontend() + { + } + +void ReaderFrontend::Init(string arg_source, ReaderMode mode, const int num_fields, + const threading::Field* const* fields) + { + if ( disabled ) + return; + + if ( initialized ) + reporter->InternalError("reader initialize twice"); + + source = arg_source; + initialized = true; + + backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields)); + } + +void ReaderFrontend::Update() + { + if ( disabled ) + return; + + if ( ! initialized ) + { + reporter->Error("Tried to call update on uninitialized reader"); + return; + } + + backend->SendIn(new UpdateMessage(backend)); + } + +void ReaderFrontend::Close() + { + if ( disabled ) + return; + + if ( ! initialized ) + { + reporter->Error("Tried to call finish on uninitialized reader"); + return; + } + + backend->SendIn(new CloseMessage(backend)); + } + +string ReaderFrontend::Name() const + { + if ( source.size() ) + return ty_name; + + return ty_name + "/" + source; + } + +} + diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h new file mode 100644 index 0000000000..c61b194e24 --- /dev/null +++ b/src/input/ReaderFrontend.h @@ -0,0 +1,131 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERFRONTEND_H +#define INPUT_READERFRONTEND_H + +#include "ReaderBackend.h" + +#include "threading/MsgThread.h" +#include "threading/SerialTypes.h" + +namespace input { + +class Manager; + +/** + * Bridge class between the input::Manager and backend input threads. The + * Manager instantiates one \a ReaderFrontend for each open input stream. + * Each frontend in turns instantiates a ReaderBackend-derived class + * internally that's specific to the particular input format. That backend + * spawns a new thread, and it receives messages from the frontend that + * correspond to method called by the manager. + */ +class ReaderFrontend { +public: + /** + * Constructor. + * + * type: The backend writer type, with the value corresponding to the + * script-level \c Input::Reader enum (e.g., \a READER_ASCII). The + * frontend will internally instantiate a ReaderBackend of the + * corresponding type. + * + * Frontends must only be instantiated by the main thread. + */ + ReaderFrontend(bro_int_t type); + + /** + * Destructor. + * + * Frontends must only be destroyed by the main thread. + */ + virtual ~ReaderFrontend(); + + /** + * Initializes the reader. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See ReaderBackend::Init() for arguments. + * + * This method must only be called from the main thread. + */ + void Init(string arg_source, ReaderMode mode, const int arg_num_fields, const threading::Field* const* fields); + + /** + * Force an update of the current input source. Actual action depends + * on the opening mode and on the input source. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. + * + * This method must only be called from the main thread. + */ + void Update(); + + /** + * Finalizes reading from this stream. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. This method must only be called + * from the main thread. + */ + void Close(); + + /** + * Disables the reader frontend. From now on, all method calls that + * would normally send message over to the backend, turn into no-ops. + * Note though that it does not stop the backend itself, use Finish() + * to do that as well (this method is primarily for use as callback + * when the backend wants to disable the frontend). + * + * Disabled frontends will eventually be discarded by the + * input::Manager. + * + * This method must only be called from the main thread. + */ + void SetDisable() { disabled = true; } + + /** + * Returns true if the reader frontend has been disabled with + * SetDisable(). + */ + bool Disabled() { return disabled; } + + /** + * Returns a descriptive name for the reader, including the type of + * the backend and the source used. + * + * This method is safe to call from any thread. + */ + string Name() const; + +protected: + friend class Manager; + + /** + * Returns the source as passed into the constructor. + */ + const string& Source() const { return source; }; + + /** + * Returns the name of the backend's type. + */ + const string& TypeName() const { return ty_name; } + +private: + ReaderBackend* backend; // The backend we have instanatiated. + string source; + string ty_name; // Backend type, set by manager. + bool disabled; // True if disabled. + bool initialized; // True if initialized. +}; + +} + + +#endif /* INPUT_READERFRONTEND_H */ + + diff --git a/src/input/fdstream.h b/src/input/fdstream.h new file mode 100644 index 0000000000..cda767dd52 --- /dev/null +++ b/src/input/fdstream.h @@ -0,0 +1,189 @@ +/* The following code declares classes to read from and write to + * file descriptore or file handles. + * + * See + * http://www.josuttis.com/cppcode + * for details and the latest version. + * + * - open: + * - integrating BUFSIZ on some systems? + * - optimized reading of multiple characters + * - stream for reading AND writing + * - i18n + * + * (C) Copyright Nicolai M. Josuttis 2001. + * Permission to copy, use, modify, sell and distribute this software + * is granted provided this copyright notice appears in all copies. + * This software is provided "as is" without express or implied + * warranty, and with no claim as to its suitability for any purpose. + * + * Version: Jul 28, 2002 + * History: + * Jul 28, 2002: bugfix memcpy() => memmove() + * fdinbuf::underflow(): cast for return statements + * Aug 05, 2001: first public version + */ +#ifndef BOOST_FDSTREAM_HPP +#define BOOST_FDSTREAM_HPP + +#include +#include +#include +// for EOF: +#include +// for memmove(): +#include + + + +// low-level read and write functions +#ifdef _MSC_VER +# include +#else +# include +# include +//extern "C" { +// int write (int fd, const char* buf, int num); +// int read (int fd, char* buf, int num); +//} +#endif + + +// BEGIN namespace BOOST +namespace boost { + + +/************************************************************ + * fdostream + * - a stream that writes on a file descriptor + ************************************************************/ + + +class fdoutbuf : public std::streambuf { + protected: + int fd; // file descriptor + public: + // constructor + fdoutbuf (int _fd) : fd(_fd) { + } + protected: + // write one character + virtual int_type overflow (int_type c) { + if (c != EOF) { + char z = c; + if (write (fd, &z, 1) != 1) { + return EOF; + } + } + return c; + } + // write multiple characters + virtual + std::streamsize xsputn (const char* s, + std::streamsize num) { + return write(fd,s,num); + } +}; + +class fdostream : public std::ostream { + protected: + fdoutbuf buf; + public: + fdostream (int fd) : std::ostream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +/************************************************************ + * fdistream + * - a stream that reads on a file descriptor + ************************************************************/ + +class fdinbuf : public std::streambuf { + protected: + int fd; // file descriptor + protected: + /* data buffer: + * - at most, pbSize characters in putback area plus + * - at most, bufSize characters in ordinary read buffer + */ + static const int pbSize = 4; // size of putback area + static const int bufSize = 1024; // size of the data buffer + char buffer[bufSize+pbSize]; // data buffer + + public: + /* constructor + * - initialize file descriptor + * - initialize empty data buffer + * - no putback area + * => force underflow() + */ + fdinbuf (int _fd) : fd(_fd) { + setg (buffer+pbSize, // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize); // end position + } + + protected: + // insert new characters into the buffer + virtual int_type underflow () { +#ifndef _MSC_VER + using std::memmove; +#endif + + // is read position before end of buffer? + if (gptr() < egptr()) { + return traits_type::to_int_type(*gptr()); + } + + /* process size of putback area + * - use number of characters read + * - but at most size of putback area + */ + int numPutback; + numPutback = gptr() - eback(); + if (numPutback > pbSize) { + numPutback = pbSize; + } + + /* copy up to pbSize characters previously read into + * the putback area + */ + memmove (buffer+(pbSize-numPutback), gptr()-numPutback, + numPutback); + + // read at most bufSize new characters + int num; + num = read (fd, buffer+pbSize, bufSize); + if ( num == EAGAIN ) { + return 0; + } + if (num <= 0) { + // ERROR or EOF + return EOF; + } + + // reset buffer pointers + setg (buffer+(pbSize-numPutback), // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize+num); // end of buffer + + // return next character + return traits_type::to_int_type(*gptr()); + } +}; + +class fdistream : public std::istream { + protected: + fdinbuf buf; + public: + fdistream (int fd) : std::istream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +} // END namespace boost + +#endif /*BOOST_FDSTREAM_HPP*/ diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc new file mode 100644 index 0000000000..186d765d21 --- /dev/null +++ b/src/input/readers/Ascii.cc @@ -0,0 +1,528 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Ascii.h" +#include "NetVar.h" + +#include +#include + +#include "../../threading/SerialTypes.h" + +#include +#include +#include + +using namespace input::reader; +using threading::Value; +using threading::Field; + +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) + : name(arg_name), type(arg_type) + { + position = arg_position; + secondary_position = -1; + present = true; + } + +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, + const TypeTag& arg_subtype, int arg_position) + : name(arg_name), type(arg_type), subtype(arg_subtype) + { + position = arg_position; + secondary_position = -1; + present = true; + } + +FieldMapping::FieldMapping(const FieldMapping& arg) + : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) + { + position = arg.position; + secondary_position = arg.secondary_position; + } + +FieldMapping FieldMapping::subType() + { + return FieldMapping(name, subtype, position); + } + +Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) + { + file = 0; + + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), + BifConst::InputAscii::separator->Len()); + + if ( separator.size() != 1 ) + Error("separator length has to be 1. Separator will be truncated."); + + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), + BifConst::InputAscii::set_separator->Len()); + + if ( set_separator.size() != 1 ) + Error("set_separator length has to be 1. Separator will be truncated."); + + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), + BifConst::InputAscii::empty_field->Len()); + + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), + BifConst::InputAscii::unset_field->Len()); +} + +Ascii::~Ascii() + { + DoClose(); + } + +void Ascii::DoClose() + { + if ( file != 0 ) + { + file->close(); + delete(file); + file = 0; + } + } + +bool Ascii::DoInit(string path, ReaderMode mode, int num_fields, const Field* const* fields) + { + mtime = 0; + + file = new ifstream(path.c_str()); + if ( ! file->is_open() ) + { + Error(Fmt("Init: cannot open %s", path.c_str())); + delete(file); + file = 0; + return false; + } + + if ( ReadHeader(false) == false ) + { + Error(Fmt("Init: cannot open %s; headers are incorrect", path.c_str())); + file->close(); + delete(file); + file = 0; + return false; + } + + DoUpdate(); + + return true; + } + + +bool Ascii::ReadHeader(bool useCached) + { + // try to read the header line... + string line; + map ifields; + + if ( ! useCached ) + { + if ( ! GetLine(line) ) + { + Error("could not read first line"); + return false; + } + + headerline = line; + } + + else + line = headerline; + + // construct list of field names. + istringstream splitstream(line); + int pos=0; + while ( splitstream ) + { + string s; + if ( ! getline(splitstream, s, separator[0])) + break; + + ifields[s] = pos; + pos++; + } + + //printf("Updating fields from description %s\n", line.c_str()); + columnMap.clear(); + + for ( unsigned int i = 0; i < NumFields(); i++ ) + { + const Field* field = Fields()[i]; + + map::iterator fit = ifields.find(field->name); + if ( fit == ifields.end() ) + { + if ( field->optional ) + { + // we do not really need this field. mark it as not present and always send an undef back. + FieldMapping f(field->name, field->type, field->subtype, -1); + f.present = false; + columnMap.push_back(f); + continue; + } + + Error(Fmt("Did not find requested field %s in input data file %s.", + field->name.c_str(), Source().c_str())); + return false; + } + + + FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); + + if ( field->secondary_name != "" ) + { + map::iterator fit2 = ifields.find(field->secondary_name); + if ( fit2 == ifields.end() ) + { + Error(Fmt("Could not find requested port type field %s in input data file.", + field->secondary_name.c_str())); + return false; + } + + f.secondary_position = ifields[field->secondary_name]; + } + + columnMap.push_back(f); + } + + + // well, that seems to have worked... + return true; + } + +bool Ascii::GetLine(string& str) + { + while ( getline(*file, str) ) + { + if ( str[0] != '#' ) + return true; + + if ( str.compare(0,8, "#fields\t") == 0 ) + { + str = str.substr(8); + return true; + } + } + + return false; + } + + +Value* Ascii::EntryToVal(string s, FieldMapping field) + { + if ( s.compare(unset_field) == 0 ) // field is not set... + return new Value(field.type, false); + + Value* val = new Value(field.type, true); + + switch ( field.type ) { + case TYPE_ENUM: + case TYPE_STRING: + val->val.string_val = new string(s); + break; + + case TYPE_BOOL: + if ( s == "T" ) + val->val.int_val = 1; + else if ( s == "F" ) + val->val.int_val = 0; + else + { + Error(Fmt("Field: %s Invalid value for boolean: %s", + field.name.c_str(), s.c_str())); + return false; + } + break; + + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + val->val.uint_val = atoi(s.c_str()); + break; + + case TYPE_PORT: + val->val.port_val.port = atoi(s.c_str()); + val->val.port_val.proto = TRANSPORT_UNKNOWN; + break; + + case TYPE_SUBNET: + { + size_t pos = s.find("/"); + if ( pos == s.npos ) + { + Error(Fmt("Invalid value for subnet: %s", s.c_str())); + return false; + } + + int width = atoi(s.substr(pos+1).c_str()); + string addr = s.substr(0, pos); + + val->val.subnet_val.prefix = StringToAddr(addr); + val->val.subnet_val.length = width; + break; + } + + case TYPE_ADDR: + val->val.addr_val = StringToAddr(s); + break; + + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + if ( s[i] == ',' ) length++; + + unsigned int pos = 0; + + if ( s.compare(empty_field) == 0 ) + length = 0; + + Value** lvals = new Value* [length]; + + if ( field.type == TYPE_TABLE ) + { + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } + + else if ( field.type == TYPE_VECTOR ) + { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } + + else + assert(false); + + if ( length == 0 ) + break; //empty + + istringstream splitstream(s); + while ( splitstream ) + { + string element; + + if ( ! getline(splitstream, element, set_separator[0]) ) + break; + + if ( pos >= length ) + { + Error(Fmt("Internal error while parsing set. pos %d >= length %d." + " Element: %s", pos, length, element.c_str())); + break; + } + + Value* newval = EntryToVal(element, field.subType()); + if ( newval == 0 ) + { + Error("Error while reading set"); + return 0; + } + + lvals[pos] = newval; + + pos++; + } + + if ( pos != length ) + { + Error("Internal error while parsing set: did not find all elements"); + return 0; + } + + break; + } + + default: + Error(Fmt("unsupported field format %d for %s", field.type, + field.name.c_str())); + return 0; + } + + return val; + } + +// read the entire file and send appropriate thingies back to InputMgr +bool Ascii::DoUpdate() + { + switch ( Mode() ) { + case MODE_REREAD: + { + // check if the file has changed + struct stat sb; + if ( stat(Source().c_str(), &sb) == -1 ) + { + Error(Fmt("Could not get stat for %s", Source().c_str())); + return false; + } + + if ( sb.st_mtime <= mtime ) // no change + return true; + + mtime = sb.st_mtime; + // file changed. reread. + + // fallthrough + } + + case MODE_MANUAL: + case MODE_STREAM: + { + // dirty, fix me. (well, apparently after trying seeking, etc + // - this is not that bad) + if ( file && file->is_open() ) + { + if ( Mode() == MODE_STREAM ) + { + file->clear(); // remove end of file evil bits + if ( !ReadHeader(true) ) + return false; // header reading failed + + break; + } + + file->close(); + delete file; + file = 0; + } + + file = new ifstream(Source().c_str()); + if ( ! file->is_open() ) + { + Error(Fmt("cannot open %s", Source().c_str())); + return false; + } + + if ( ReadHeader(false) == false ) + { + return false; + } + + break; + } + + default: + assert(false); + + } + + string line; + while ( GetLine(line ) ) + { + // split on tabs + istringstream splitstream(line); + + map stringfields; + int pos = 0; + while ( splitstream ) + { + string s; + if ( ! getline(splitstream, s, separator[0]) ) + break; + + stringfields[pos] = s; + pos++; + } + + pos--; // for easy comparisons of max element. + + Value** fields = new Value*[NumFields()]; + + int fpos = 0; + for ( vector::iterator fit = columnMap.begin(); + fit != columnMap.end(); + fit++ ) + { + + if ( ! fit->present ) + { + // add non-present field + fields[fpos] = new Value((*fit).type, false); + fpos++; + continue; + } + + assert(fit->position >= 0 ); + + if ( (*fit).position > pos || (*fit).secondary_position > pos ) + { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", + line.c_str(), pos, (*fit).position, (*fit).secondary_position)); + return false; + } + + Value* val = EntryToVal(stringfields[(*fit).position], *fit); + if ( val == 0 ) + { + Error("Could not convert String value to Val"); + return false; + } + + if ( (*fit).secondary_position != -1 ) + { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); + + val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); + } + + fields[fpos] = val; + + fpos++; + } + + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); + assert ( (unsigned int) fpos == NumFields() ); + + if ( Mode() == MODE_STREAM ) + Put(fields); + else + SendEntry(fields); + } + + if ( Mode () != MODE_STREAM ) + EndCurrentSend(); + + return true; + } + +bool Ascii::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( Mode() ) { + case MODE_MANUAL: + // yay, we do nothing :) + break; + + case MODE_REREAD: + case MODE_STREAM: + Update(); // call update and not DoUpdate, because update + // checks disabled. + break; + + default: + assert(false); + } + + return true; + } + diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h new file mode 100644 index 0000000000..335616abfb --- /dev/null +++ b/src/input/readers/Ascii.h @@ -0,0 +1,72 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_ASCII_H +#define INPUT_READERS_ASCII_H + +#include +#include + +#include "../ReaderBackend.h" + +namespace input { namespace reader { + +// Description for input field mapping. +struct FieldMapping { + string name; + TypeTag type; + TypeTag subtype; // internal type for sets and vectors + int position; + int secondary_position; // for ports: pos of the second field + bool present; + + FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); + FieldMapping(const FieldMapping& arg); + FieldMapping() { position = -1; secondary_position = -1; } + + FieldMapping subType(); +}; + +/** + * Reader for structured ASCII files. + */ +class Ascii : public ReaderBackend { +public: + Ascii(ReaderFrontend* frontend); + ~Ascii(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); } + +protected: + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields); + virtual void DoClose(); + virtual bool DoUpdate(); + virtual bool DoHeartbeat(double network_time, double current_time); + +private: + + bool ReadHeader(bool useCached); + bool GetLine(string& str); + threading::Value* EntryToVal(string s, FieldMapping type); + + ifstream* file; + time_t mtime; + + // map columns in the file to columns to send back to the manager + vector columnMap; + + // keep a copy of the headerline to determine field locations when stream descriptions change + string headerline; + + // options set from the script-level. + string separator; + string set_separator; + string empty_field; + string unset_field; +}; + + +} +} + +#endif /* INPUT_READERS_ASCII_H */ diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc new file mode 100644 index 0000000000..5e4ef090f7 --- /dev/null +++ b/src/input/readers/Benchmark.cc @@ -0,0 +1,263 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Benchmark.h" +#include "NetVar.h" + +#include "../../threading/SerialTypes.h" + +#include +#include +#include + +#include "../../threading/Manager.h" + +using namespace input::reader; +using threading::Value; +using threading::Field; + +Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) + { + multiplication_factor = double(BifConst::InputBenchmark::factor); + autospread = double(BifConst::InputBenchmark::autospread); + spread = int(BifConst::InputBenchmark::spread); + add = int(BifConst::InputBenchmark::addfactor); + autospread_time = 0; + stopspreadat = int(BifConst::InputBenchmark::stopspreadat); + timedspread = double(BifConst::InputBenchmark::timedspread); + heartbeat_interval = double(BifConst::Threading::heartbeat_interval); + } + +Benchmark::~Benchmark() + { + DoClose(); + } + +void Benchmark::DoClose() + { + } + +bool Benchmark::DoInit(string path, ReaderMode mode, int num_fields, const Field* const* fields) + { + num_lines = atoi(path.c_str()); + + if ( autospread != 0.0 ) + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); + + heartbeatstarttime = CurrTime(); + DoUpdate(); + + return true; + } + +string Benchmark::RandomString(const int len) + { + string s(len, ' '); + + static const char values[] = + "0123456789!@#$%^&*()-_=+{}[]\\|" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + for (int i = 0; i < len; ++i) + s[i] = values[rand() / (RAND_MAX / sizeof(values))]; + + return s; + } + +double Benchmark::CurrTime() + { + struct timeval tv; + assert ( gettimeofday(&tv, 0) >= 0 ); + + return double(tv.tv_sec) + double(tv.tv_usec) / 1e6; + } + + +// read the entire file and send appropriate thingies back to InputMgr +bool Benchmark::DoUpdate() + { + int linestosend = num_lines * heartbeat_interval; + for ( int i = 0; i < linestosend; i++ ) + { + Value** field = new Value*[NumFields()]; + for (unsigned int j = 0; j < NumFields(); j++ ) + field[j] = EntryToVal(Fields()[j]->type, Fields()[j]->subtype); + + if ( Mode() == MODE_STREAM ) + // do not do tracking, spread out elements over the second that we have... + Put(field); + else + SendEntry(field); + + if ( stopspreadat == 0 || num_lines < stopspreadat ) + { + if ( spread != 0 ) + usleep(spread); + + if ( autospread_time != 0 ) + usleep( autospread_time ); + } + + if ( timedspread != 0.0 ) + { + double diff; + do + diff = CurrTime() - heartbeatstarttime; + while ( diff/heartbeat_interval < i/(linestosend + + (linestosend * timedspread) ) ); + } + + } + + if ( Mode() != MODE_STREAM ) + EndCurrentSend(); + + return true; +} + +threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) + { + Value* val = new Value(type, true); + + // basically construct something random from the fields that we want. + + switch ( type ) { + case TYPE_ENUM: + assert(false); // no enums, please. + + case TYPE_STRING: + val->val.string_val = new string(RandomString(10)); + break; + + case TYPE_BOOL: + val->val.int_val = 1; // we never lie. + break; + + case TYPE_INT: + val->val.int_val = rand(); + break; + + case TYPE_TIME: + val->val.double_val = CurrTime(); + break; + + case TYPE_DOUBLE: + case TYPE_INTERVAL: + val->val.double_val = random(); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + val->val.uint_val = rand(); + break; + + case TYPE_PORT: + val->val.port_val.port = rand() / (RAND_MAX / 60000); + val->val.port_val.proto = TRANSPORT_UNKNOWN; + break; + + case TYPE_SUBNET: + { + val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); + val->val.subnet_val.length = 16; + } + break; + + case TYPE_ADDR: + val->val.addr_val = StringToAddr("192.168.17.1"); + break; + + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { + // how many entries do we have... + unsigned int length = rand() / (RAND_MAX / 15); + + Value** lvals = new Value* [length]; + + if ( type == TYPE_TABLE ) + { + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } + else if ( type == TYPE_VECTOR ) + { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } + else + assert(false); + + if ( length == 0 ) + break; //empty + + for ( unsigned int pos = 0; pos < length; pos++ ) + { + Value* newval = EntryToVal(subtype, TYPE_ENUM); + if ( newval == 0 ) + { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d", type)); + return 0; + } + + return val; + + } + + +bool Benchmark::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + num_lines = (int) ( (double) num_lines*multiplication_factor); + num_lines += add; + heartbeatstarttime = CurrTime(); + + switch ( Mode() ) { + case MODE_MANUAL: + // yay, we do nothing :) + break; + + case MODE_REREAD: + case MODE_STREAM: + if ( multiplication_factor != 1 || add != 0 ) + { + // we have to document at what time we changed the factor to what value. + Value** v = new Value*[2]; + v[0] = new Value(TYPE_COUNT, true); + v[0]->val.uint_val = num_lines; + v[1] = new Value(TYPE_TIME, true); + v[1]->val.double_val = CurrTime(); + + SendEvent("lines_changed", 2, v); + } + + if ( autospread != 0.0 ) + // because executing this in every loop is apparently too expensive. + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); + + Update(); // call update and not DoUpdate, because update actually checks disabled. + + SendEvent("HeartbeatDone", 0, 0); + break; + + default: + assert(false); + } + + return true; +} diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h new file mode 100644 index 0000000000..6bb70781fd --- /dev/null +++ b/src/input/readers/Benchmark.h @@ -0,0 +1,47 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_BENCHMARK_H +#define INPUT_READERS_BENCHMARK_H + +#include "../ReaderBackend.h" + +namespace input { namespace reader { + +/** + * A benchmark reader to measure performance of the input framework. + */ +class Benchmark : public ReaderBackend { +public: + Benchmark(ReaderFrontend* frontend); + ~Benchmark(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); } + +protected: + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields); + virtual void DoClose(); + virtual bool DoUpdate(); + virtual bool DoHeartbeat(double network_time, double current_time); + +private: + double CurrTime(); + string RandomString(const int len); + threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); + + int num_lines; + double multiplication_factor; + int spread; + double autospread; + int autospread_time; + int add; + int stopspreadat; + double heartbeatstarttime; + double timedspread; + double heartbeat_interval; +}; + + +} +} + +#endif /* INPUT_READERS_BENCHMARK_H */ diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc new file mode 100644 index 0000000000..fa1b09da7c --- /dev/null +++ b/src/input/readers/Raw.cc @@ -0,0 +1,265 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Raw.h" +#include "NetVar.h" + +#include +#include + +#include "../../threading/SerialTypes.h" +#include "../fdstream.h" + +#include +#include +#include +#include +#include + +using namespace input::reader; +using threading::Value; +using threading::Field; + +Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) + { + file = 0; + in = 0; + + separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), + BifConst::InputRaw::record_separator->Len()); + + if ( separator.size() != 1 ) + Error("separator length has to be 1. Separator will be truncated."); + } + +Raw::~Raw() + { + DoClose(); + } + +void Raw::DoClose() + { + if ( file != 0 ) + CloseInput(); + } + +bool Raw::OpenInput() + { + if ( execute ) + { + file = popen(fname.c_str(), "r"); + if ( file == NULL ) + { + Error(Fmt("Could not execute command %s", fname.c_str())); + return false; + } + } + else + { + file = fopen(fname.c_str(), "r"); + if ( file == NULL ) + { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + } + + // This is defined in input/fdstream.h + in = new boost::fdistream(fileno(file)); + + if ( execute && Mode() == MODE_STREAM ) + fcntl(fileno(file), F_SETFL, O_NONBLOCK); + + return true; + } + +bool Raw::CloseInput() + { + if ( file == NULL ) + { + InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); + return false; + } + + delete in; + + if ( execute ) + pclose(file); + else + fclose(file); + + in = NULL; + file = NULL; + + return true; + } + +bool Raw::DoInit(string path, ReaderMode mode, int num_fields, const Field* const* fields) + { + fname = path; + mtime = 0; + execute = false; + firstrun = true; + bool result; + + if ( path.length() == 0 ) + { + Error("No source path provided"); + return false; + } + + if ( num_fields != 1 ) + { + Error("Filter for raw reader contains more than one field. " + "Filters for the raw reader may only contain exactly one string field. " + "Filter ignored."); + return false; + } + + if ( fields[0]->type != TYPE_STRING ) + { + Error("Filter for raw reader contains a field that is not of type string."); + return false; + } + + // do Initialization + char last = path[path.length()-1]; + if ( last == '|' ) + { + execute = true; + fname = path.substr(0, fname.length() - 1); + + if ( (mode != MODE_MANUAL) && (mode != MODE_STREAM) ) + { + Error(Fmt("Unsupported read mode %d for source %s in execution mode", + mode, fname.c_str())); + return false; + } + + result = OpenInput(); + + } + else + { + execute = false; + result = OpenInput(); + } + + if ( result == false ) + return result; + +#ifdef DEBUG + Debug(DBG_INPUT, "Raw reader created, will perform first update"); +#endif + + // after initialization - do update + DoUpdate(); + +#ifdef DEBUG + Debug(DBG_INPUT, "First update went through"); +#endif + return true; + } + + +bool Raw::GetLine(string& str) + { + if ( in->peek() == std::iostream::traits_type::eof() ) + return false; + + if ( in->eofbit == true || in->failbit == true ) + return false; + + return getline(*in, str, separator[0]); + } + +// read the entire file and send appropriate thingies back to InputMgr +bool Raw::DoUpdate() + { + if ( firstrun ) + firstrun = false; + + else + { + switch ( Mode() ) { + case MODE_REREAD: + { + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) + { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } + + if ( sb.st_mtime <= mtime ) + // no change + return true; + + mtime = sb.st_mtime; + // file changed. reread. + // + // fallthrough + } + + case MODE_MANUAL: + case MODE_STREAM: + if ( Mode() == MODE_STREAM && file != NULL && in != NULL ) + { + //fpurge(file); + in->clear(); // remove end of file evil bits + break; + } + + CloseInput(); + if ( ! OpenInput() ) + return false; + + break; + + default: + assert(false); + } + } + + string line; + while ( GetLine(line) ) + { + assert (NumFields() == 1); + + Value** fields = new Value*[1]; + + // filter has exactly one text field. convert to it. + Value* val = new Value(TYPE_STRING, true); + val->val.string_val = new string(line); + fields[0] = val; + + Put(fields); + } + +#ifdef DEBUG + Debug(DBG_INPUT, "DoUpdate finished successfully"); +#endif + + return true; + } + +bool Raw::DoHeartbeat(double network_time, double current_time) + { + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( Mode() ) { + case MODE_MANUAL: + // yay, we do nothing :) + break; + + case MODE_REREAD: + case MODE_STREAM: + Update(); // call update and not DoUpdate, because update + // checks disabled. + break; + default: + assert(false); + } + + return true; + } diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h new file mode 100644 index 0000000000..b9b45f0b20 --- /dev/null +++ b/src/input/readers/Raw.h @@ -0,0 +1,49 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_RAW_H +#define INPUT_READERS_RAW_H + +#include +#include + +#include "../ReaderBackend.h" + +namespace input { namespace reader { + +/** + * A reader that returns a file (or the output of a command) as a single + * blob. + */ +class Raw : public ReaderBackend { +public: + Raw(ReaderFrontend* frontend); + ~Raw(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); } + +protected: + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields); + virtual void DoClose(); + virtual bool DoUpdate(); + virtual bool DoHeartbeat(double network_time, double current_time); + +private: + bool OpenInput(); + bool CloseInput(); + bool GetLine(string& str); + + string fname; // Source with a potential "|" removed. + istream* in; + FILE* file; + bool execute; + bool firstrun; + time_t mtime; + + // options set from the script-level. + string separator; +}; + +} +} + +#endif /* INPUT_READERS_RAW_H */ diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 34d10a1abf..baf832e6a9 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -477,6 +477,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, threading::Field* field = new threading::Field(); field->name = new_path; field->type = t->Tag(); + field->optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL); if ( field->type == TYPE_TABLE ) field->subtype = t->AsSetType()->Indices()->PureType()->Tag(); diff --git a/src/main.cc b/src/main.cc index 19910aebc5..9e9c867714 100644 --- a/src/main.cc +++ b/src/main.cc @@ -50,6 +50,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "Brofiler.h" #include "threading/Manager.h" +#include "input/Manager.h" #include "logging/Manager.h" #include "logging/writers/Ascii.h" @@ -80,6 +81,7 @@ DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; logging::Manager* log_mgr = 0; threading::Manager* thread_mgr = 0; +input::Manager* input_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -314,6 +316,8 @@ void terminate_bro() log_mgr->Terminate(); thread_mgr->Terminate(); + mgr.Drain(); + delete timer_mgr; delete dns_mgr; delete persistence_serializer; @@ -763,6 +767,7 @@ int main(int argc, char** argv) remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); log_mgr = new logging::Manager(); + input_mgr = new input::Manager(); if ( events_file ) event_player = new EventPlayer(events_file); diff --git a/src/parse.y b/src/parse.y index f78003f08b..75e09dc60f 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 87 +%expect 90 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -24,6 +24,7 @@ %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER +%token TOK_ATTR_TYPE_COLUMN %token TOK_DEBUG @@ -112,13 +113,14 @@ bool is_export = false; // true if in an export {} block * (obviously not reentrant). */ extern Expr* g_curr_debug_expr; +extern bool in_debug; +extern const char* g_curr_debug_error; #define YYLTYPE yyltype Expr* bro_this = 0; int in_init = 0; int in_record = 0; -bool in_debug = false; bool resolving_global_ID = false; bool defining_global_ID = false; @@ -249,7 +251,6 @@ bro: TOK_DEBUG { in_debug = true; } expr { g_curr_debug_expr = $3; - in_debug = false; } ; @@ -1306,6 +1307,8 @@ attr: { $$ = new Attr(ATTR_PRIORITY, $3); } | TOK_ATTR_GROUP '=' expr { $$ = new Attr(ATTR_GROUP, $3); } + | TOK_ATTR_TYPE_COLUMN '=' expr + { $$ = new Attr(ATTR_TYPE_COLUMN, $3); } | TOK_ATTR_LOG { $$ = new Attr(ATTR_LOG); } | TOK_ATTR_ERROR_HANDLER @@ -1685,6 +1688,9 @@ int yyerror(const char msg[]) strcat(msgbuf, "\nDocumentation mode is enabled: " "remember to check syntax of ## style comments\n"); + if ( in_debug ) + g_curr_debug_error = copy_string(msg); + reporter->Error("%s", msgbuf); return 0; diff --git a/src/scan.l b/src/scan.l index ca7a01574c..30d521c6bd 100644 --- a/src/scan.l +++ b/src/scan.l @@ -331,6 +331,7 @@ when return TOK_WHEN; &optional return TOK_ATTR_OPTIONAL; &persistent return TOK_ATTR_PERSISTENT; &priority return TOK_ATTR_PRIORITY; +&type_column return TOK_ATTR_TYPE_COLUMN; &read_expire return TOK_ATTR_EXPIRE_READ; &redef return TOK_ATTR_REDEF; &rotate_interval return TOK_ATTR_ROTATE_INTERVAL; diff --git a/src/strings.bif b/src/strings.bif index ebe16529ea..27c11b4013 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -11,8 +11,8 @@ using namespace std; %%} -## Concates all arguments into a single string. The function takes a variable -## number of arguments of type string and stiches them together. +## Concatenates all arguments into a single string. The function takes a +## variable number of arguments of type string and stitches them together. ## ## Returns: The concatenation of all (string) arguments. ## @@ -157,9 +157,9 @@ function join_string_array%(sep: string, a: string_array%): string ## ## sep: The separator to place between each element. ## -## a: The :bro:type:`string_vec` (``vector of string``). +## vec: The :bro:type:`string_vec` (``vector of string``). ## -## Returns: The concatenation of all elements in *a*, with *sep* placed +## Returns: The concatenation of all elements in *vec*, with *sep* placed ## between each element. ## ## .. bro:see:: cat cat_sep string_cat cat_string_array cat_string_array_n @@ -219,7 +219,7 @@ function sort_string_array%(a: string_array%): string_array ## Returns an edited version of a string that applies a special ## "backspace character" (usually ``\x08`` for backspace or ``\x7f`` for DEL). -## For ## example, ``edit("hello there", "e")`` returns ``"llo t"``. +## For example, ``edit("hello there", "e")`` returns ``"llo t"``. ## ## arg_s: The string to edit. ## @@ -229,7 +229,7 @@ function sort_string_array%(a: string_array%): string_array ## the string. ## ## Returns: An edited version of *arg_s* where *arg_edit_char* triggers the -## deletetion of the last character. +## deletion of the last character. ## ## .. bro:see:: clean ## to_string_literal @@ -278,7 +278,7 @@ function byte_len%(s: string%): count return new Val(s->Len(), TYPE_COUNT); %} -## Get a substring of from a string, given a starting position length. +## Get a substring from a string, given a starting position and length. ## ## s: The string to obtain a substring from. ## @@ -486,10 +486,10 @@ function split%(str: string, re: pattern%): string_array return do_split(str, re, 0, 0, 0); %} -## Splits a string *once* into a a two-element array of strings according to a -## pattern. This function is the same as :bro:id:`split`, but * is only split -## once (if possible) at the earliest position and an array of two strings is -## returned. +## Splits a string *once* into a two-element array of strings according to a +## pattern. This function is the same as :bro:id:`split`, but *str* is only +## split once (if possible) at the earliest position and an array of two strings +## is returned. ## ## str: The string to split. ## @@ -518,7 +518,7 @@ function split1%(str: string, re: pattern%): string_array ## ## Returns: An array of strings where each two successive elements correspond ## to a substring in *str* of the part not matching *re* (odd-indexed) -## and thei part that matches *re* (even-indexed). +## and the part that matches *re* (even-indexed). ## ## .. bro:see:: split split1 split_n str_split function split_all%(str: string, re: pattern%): string_array @@ -568,7 +568,7 @@ function split_complete%(str: string, ## ## re: The pattern being replaced with *repl*. ## -## repl: The string that replacs *re*. +## repl: The string that replaces *re*. ## ## Returns: A copy of *str* with the first occurence of *re* replaced with ## *repl*. @@ -579,16 +579,16 @@ function sub%(str: string, re: pattern, repl: string%): string return do_sub(str, re, repl, 0); %} -## Substitutes a given replacement string for the all occurrences of a pattern +## Substitutes a given replacement string for all occurrences of a pattern ## in a given string. ## ## str: The string to perform the substitution in. ## ## re: The pattern being replaced with *repl*. ## -## repl: The string that replacs *re*. +## repl: The string that replaces *re*. ## -## Returns: A copy of *str* with all occurences of *re* replaced with *repl*. +## Returns: A copy of *str* with all occurrences of *re* replaced with *repl*. ## ## .. bro:see:: sub subst_string function gsub%(str: string, re: pattern, repl: string%): string @@ -597,7 +597,7 @@ function gsub%(str: string, re: pattern, repl: string%): string %} -## Lexicographically compares two string. +## Lexicographically compares two strings. ## ## s1: The first string. ## @@ -616,7 +616,7 @@ function strcmp%(s1: string, s2: string%): int ## ## little: The (smaller) string to find inside *big*. ## -## Returns: The location of *little* in *big* or 0 if *little* is not found in +## Returns: The location of *little* in *big*, or 0 if *little* is not found in ## *big*. ## ## .. bro:see:: find_all find_last @@ -685,7 +685,7 @@ function subst_string%(s: string, from: string, to: string%): string ## str: The string to convert to lowercase letters. ## ## Returns: A copy of the given string with the uppercase letters (as indicated -## by ``isascii`` and \verb|isupper|``) folded to lowercase +## by ``isascii`` and ``isupper``) folded to lowercase ## (via ``tolower``). ## ## .. bro:see:: to_upper is_ascii @@ -714,7 +714,7 @@ function to_lower%(str: string%): string ## str: The string to convert to uppercase letters. ## ## Returns: A copy of the given string with the lowercase letters (as indicated -## by ``isascii`` and \verb|islower|``) folded to uppercase +## by ``isascii`` and ``islower``) folded to uppercase ## (via ``toupper``). ## ## .. bro:see:: to_lower is_ascii @@ -744,7 +744,7 @@ function to_upper%(str: string%): string ## - ``NUL`` to ``\0`` ## - ``DEL`` to ``^?`` ## - values <= 26 to ``^[A-Z]`` -## - values not in *[32, 126]** to ``%XX`` +## - values not in *[32, 126]* to ``%XX`` ## ## If the string does not yet have a trailing NUL, one is added. ## @@ -765,7 +765,7 @@ function clean%(str: string%): string ## - ``NUL`` to ``\0`` ## - ``DEL`` to ``^?`` ## - values <= 26 to ``^[A-Z]`` -## - values not in *[32, 126]** to ``%XX`` +## - values not in *[32, 126]* to ``%XX`` ## ## str: The string to escape. ## @@ -831,14 +831,16 @@ function string_to_ascii_hex%(s: string%): string return new StringVal(new BroString(1, (u_char*) x, s->Len() * 2)); %} -## Uses the Smith Waterman algorithm to find similar/overlapping substrings. +## Uses the Smith-Waterman algorithm to find similar/overlapping substrings. ## See `Wikipedia `_. ## ## s1: The first string. ## ## s2: The second string. ## -## Returns: The result of the Smit Waterman algorithm calculation. +## params: Parameters for the Smith-Waterman algorithm. +## +## Returns: The result of the Smith-Waterman algorithm calculation. function str_smith_waterman%(s1: string, s2: string, params: sw_params%) : sw_substring_vec %{ SWParams sw_params(params->AsRecordVal()->Lookup(0)->AsCount(), diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index abdbf849b5..d8f3936037 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -1,5 +1,6 @@ #include "Manager.h" +#include "NetVar.h" using namespace threading; @@ -81,6 +82,12 @@ double Manager::NextTimestamp(double* network_time) // is due or not set yet), we want to check for more asap. return timer_mgr->Time(); + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + if ( (*i)->MightHaveOut() ) + return timer_mgr->Time(); + } + return -1.0; } @@ -91,7 +98,7 @@ void Manager::Process() if ( network_time && (network_time > next_beat || ! next_beat) ) { do_beat = true; - next_beat = ::network_time + HEART_BEAT_INTERVAL; + next_beat = ::network_time + BifConst::Threading::heartbeat_interval; } did_process = false; diff --git a/src/threading/Manager.h b/src/threading/Manager.h index ab8189f39d..1afd115da0 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -44,7 +44,7 @@ public: void Terminate(); /** - * Returns True if we are currently in Terminate() waiting for + * Returns True if we are currently in Terminate() waiting for * threads to exit. */ bool Terminating() const { return terminating; } @@ -99,7 +99,7 @@ protected: * Registers a new message thread with the manager. This is * automatically called by the thread's constructor. This must be * called \a in \a addition to AddThread(BasicThread* thread). The - * MsgThread constructor makes sure to do so. + * MsgThread constructor makes sure to do so. * * @param thread The thread. */ @@ -126,8 +126,6 @@ protected: virtual const char* Tag() { return "threading::Manager"; } private: - static const int HEART_BEAT_INTERVAL = 1; - typedef std::list all_thread_list; all_thread_list all_threads; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index dd73fae154..c3f694cdc1 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -301,7 +301,7 @@ void MsgThread::Run() if ( ! result ) { - string s = msg->Name() + " failed, terminating thread"; + string s = msg->Name() + " failed, terminating thread (MsgThread)"; Error(s.c_str()); Stop(); break; diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index cd29fe2a44..a917f54396 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -273,6 +273,13 @@ private: */ bool HasOut() { return queue_out.Ready(); } + /** + * Returns true if there might be at least one message pending for + * the main thread. This function may occasionally return a value not + * indicating the actual state, but won't do so very often. + */ + bool MightHaveOut() { return queue_out.MaybeReady(); } + Queue queue_in; Queue queue_out; diff --git a/src/threading/Queue.h b/src/threading/Queue.h index 985ba31714..b2ccd2a0ce 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -53,6 +53,13 @@ public: */ bool Ready(); + /** + * Returns true if the next Get() operation might succeed. + * This function may occasionally return a value not + * indicating the actual state, but won't do so very often. + */ + bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); } + /** * Returns the number of queued items not yet retrieved. */ diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 5ab61b0d41..4494e1b245 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -12,7 +12,12 @@ bool Field::Read(SerializationFormat* fmt) int t; int st; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + bool success = (fmt->Read(&name, "name") + && fmt->Read(&secondary_name, "secondary_name") + && fmt->Read(&t, "type") + && fmt->Read(&st, "subtype") + && fmt->Read(&optional, "optional")); + type = (TypeTag) t; subtype = (TypeTag) st; @@ -21,7 +26,11 @@ bool Field::Read(SerializationFormat* fmt) bool Field::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + return (fmt->Write(name, "name") + && fmt->Write(secondary_name, "secondary_name") + && fmt->Write((int)type, "type") + && fmt->Write((int)subtype, "subtype"), + fmt->Write(optional, "optional")); } string Field::TypeName() const @@ -186,7 +195,7 @@ bool Value::Read(SerializationFormat* fmt) char length; char family; - if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) ) + if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) ) return false; switch ( family ) { diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index eee3b750fe..9ce53c7cb1 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -20,19 +20,23 @@ namespace threading { */ struct Field { string name; //! Name of the field. + //! Needed by input framework. Port fields have two names (one for the + //! port, one for the type), and this specifies the secondary name. + string secondary_name; TypeTag type; //! Type of the field. TypeTag subtype; //! Inner type for sets. + bool optional; //! True if field is optional. /** * Constructor. */ - Field() { subtype = TYPE_VOID; } + Field() { subtype = TYPE_VOID; optional = false; } /** * Copy constructor. */ Field(const Field& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) { } /** * Unserializes a field. diff --git a/src/types.bif b/src/types.bif index c795e73b99..033ee975a0 100644 --- a/src/types.bif +++ b/src/types.bif @@ -169,7 +169,6 @@ enum ID %{ Unknown, %} - module Tunnel; enum Type %{ NONE, @@ -181,4 +180,25 @@ enum Type %{ type EncapsulatingConn: record; +module Input; + +enum Reader %{ + READER_DEFAULT, + READER_ASCII, + READER_RAW, + READER_BENCHMARK, +%} + +enum Event %{ + EVENT_NEW, + EVENT_CHANGED, + EVENT_REMOVED, +%} + +enum Mode %{ + MANUAL = 0, + REREAD = 1, + STREAM = 2, +%} + module GLOBAL; diff --git a/src/util.cc b/src/util.cc index 90143923f1..798be400d1 100644 --- a/src/util.cc +++ b/src/util.cc @@ -376,6 +376,8 @@ template int atoi_n(int len, const char* s, const char** end, int base, // Instantiate the ones we need. template int atoi_n(int len, const char* s, const char** end, int base, int& result); +template int atoi_n(int len, const char* s, const char** end, int base, uint16_t& result); +template int atoi_n(int len, const char* s, const char** end, int base, uint32_t& result); template int atoi_n(int len, const char* s, const char** end, int base, int64_t& result); template int atoi_n(int len, const char* s, const char** end, int base, uint64_t& result); diff --git a/testing/Makefile b/testing/Makefile index 1c82580ec4..d56ee4e0e1 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -6,13 +6,13 @@ all: make-verbose coverage brief: make-brief coverage make-verbose: - @for repo in $(DIRS); do (cd $$repo && make ); done + @for repo in $(DIRS); do (cd $$repo && make -s ); done make-brief: - @for repo in $(DIRS); do (cd $$repo && make brief ); done + @for repo in $(DIRS); do (cd $$repo && make -s brief ); done coverage: - @for repo in $(DIRS); do (cd $$repo && echo "Coverage for '$$repo' dir:" && make coverage); done + @for repo in $(DIRS); do (cd $$repo && echo "Coverage for '$$repo' dir:" && make -s coverage); done @test -f btest/coverage.log && cp btest/coverage.log `mktemp brocov.tmp.XXX` || true @for f in external/*/coverage.log; do test -f $$f && cp $$f `mktemp brocov.tmp.XXX` || true; done @echo "Complete test suite code coverage:" diff --git a/testing/btest/Baseline/core.ipv6-flow-labels/output b/testing/btest/Baseline/core.ipv6-flow-labels/output new file mode 100644 index 0000000000..9f7292d485 --- /dev/null +++ b/testing/btest/Baseline/core.ipv6-flow-labels/output @@ -0,0 +1,74 @@ +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 7407 + old_label 0 + new_label 7407 +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 176012 + old_label 0 + new_label 176012 +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 390927 + old_label 0 + new_label 390927 +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 364705 + old_label 0 + new_label 364705 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 176012 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 390927 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 364705 +new_connection: [orig_h=2001:470:4867:99::21, orig_p=55785/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49189/tcp] + orig_flow 267377 + resp_flow 0 +connection_established: [orig_h=2001:470:4867:99::21, orig_p=55785/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49189/tcp] + orig_flow 267377 + resp_flow 126027 +new_connection: [orig_h=2001:470:4867:99::21, orig_p=55647/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49190/tcp] + orig_flow 355265 + resp_flow 0 +connection_established: [orig_h=2001:470:4867:99::21, orig_p=55647/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49190/tcp] + orig_flow 355265 + resp_flow 126028 +connection_state_remove: [orig_h=2001:470:4867:99::21, orig_p=55785/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49189/tcp] + orig_flow 267377 + resp_flow 126027 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 7407 +connection_state_remove: [orig_h=2001:470:4867:99::21, orig_p=55647/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49190/tcp] + orig_flow 355265 + resp_flow 126028 diff --git a/testing/btest/Baseline/core.truncation/output b/testing/btest/Baseline/core.truncation/output index f3d64b8b28..95d9073648 100644 --- a/testing/btest/Baseline/core.truncation/output +++ b/testing/btest/Baseline/core.truncation/output @@ -22,3 +22,11 @@ #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer #types time string addr port addr port string string bool string 1334094648.590126 - - - - - truncated_IP - F bro +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer +#types time string addr port addr port string string bool string +1338328954.078361 - - - - - internally_truncated_header - F bro diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 2936e3b698..a7b369e337 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -20,4 +20,10 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/./sftp.bro scripts/base/frameworks/logging/./writers/ascii.bro scripts/base/frameworks/logging/./writers/dataseries.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro + scripts/base/frameworks/input/./readers/raw.bro + scripts/base/frameworks/input/./readers/benchmark.bro scripts/policy/misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 27846e6e82..ff2e2fc701 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -20,6 +20,12 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/./sftp.bro scripts/base/frameworks/logging/./writers/ascii.bro scripts/base/frameworks/logging/./writers/dataseries.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro + scripts/base/frameworks/input/./readers/raw.bro + scripts/base/frameworks/input/./readers/benchmark.bro scripts/base/init-default.bro scripts/base/utils/site.bro scripts/base/utils/./patterns.bro diff --git a/testing/btest/Baseline/istate.bro-ipv6-socket/recv..stdout b/testing/btest/Baseline/istate.bro-ipv6-socket/recv..stdout new file mode 100644 index 0000000000..673af68234 --- /dev/null +++ b/testing/btest/Baseline/istate.bro-ipv6-socket/recv..stdout @@ -0,0 +1 @@ +handshake done with peer: ::1 diff --git a/testing/btest/Baseline/istate.bro-ipv6-socket/send..stdout b/testing/btest/Baseline/istate.bro-ipv6-socket/send..stdout new file mode 100644 index 0000000000..fbc855464d --- /dev/null +++ b/testing/btest/Baseline/istate.bro-ipv6-socket/send..stdout @@ -0,0 +1,2 @@ +handshake done with peer: ::1 +my_event: hello world diff --git a/testing/btest/Baseline/istate.broccoli-ipv6-socket/bro..stdout b/testing/btest/Baseline/istate.broccoli-ipv6-socket/bro..stdout new file mode 100644 index 0000000000..0a7bac52c5 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ipv6-socket/bro..stdout @@ -0,0 +1,9 @@ +handshake done with peer +bro_addr(1.2.3.4) +bro_subnet(10.0.0.0/16) +bro_addr(2607:f8b0:4009:802::1014) +bro_subnet(2607:f8b0::/32) +broccoli_addr(1.2.3.4) +broccoli_subnet(10.0.0.0/16) +broccoli_addr(2607:f8b0:4009:802::1014) +broccoli_subnet(2607:f8b0::/32) diff --git a/testing/btest/Baseline/istate.broccoli-ipv6-socket/broccoli..stdout b/testing/btest/Baseline/istate.broccoli-ipv6-socket/broccoli..stdout new file mode 100644 index 0000000000..dba9318891 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ipv6-socket/broccoli..stdout @@ -0,0 +1,6 @@ +Connected to Bro instance at: ::1:47757 +Received bro_addr(1.2.3.4) +Received bro_subnet(10.0.0.0/16) +Received bro_addr(2607:f8b0:4009:802::1014) +Received bro_subnet(2607:f8b0::/32) +Terminating diff --git a/testing/btest/Baseline/istate.broccoli-ssl/bro..stdout b/testing/btest/Baseline/istate.broccoli-ssl/bro..stdout new file mode 100644 index 0000000000..0a7bac52c5 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ssl/bro..stdout @@ -0,0 +1,9 @@ +handshake done with peer +bro_addr(1.2.3.4) +bro_subnet(10.0.0.0/16) +bro_addr(2607:f8b0:4009:802::1014) +bro_subnet(2607:f8b0::/32) +broccoli_addr(1.2.3.4) +broccoli_subnet(10.0.0.0/16) +broccoli_addr(2607:f8b0:4009:802::1014) +broccoli_subnet(2607:f8b0::/32) diff --git a/testing/btest/Baseline/istate.broccoli-ssl/broccoli..stdout b/testing/btest/Baseline/istate.broccoli-ssl/broccoli..stdout new file mode 100644 index 0000000000..481778c98a --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ssl/broccoli..stdout @@ -0,0 +1,6 @@ +Connected to Bro instance at: localhost:47757 +Received bro_addr(1.2.3.4) +Received bro_subnet(10.0.0.0/16) +Received bro_addr(2607:f8b0:4009:802::1014) +Received bro_subnet(2607:f8b0::/32) +Terminating diff --git a/testing/btest/Baseline/istate.events-ssl/events.rec.log b/testing/btest/Baseline/istate.events-ssl/events.rec.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events-ssl/events.rec.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events-ssl/events.snd.log b/testing/btest/Baseline/istate.events-ssl/events.snd.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events-ssl/events.snd.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 1601f8ad3c..5a7912d23d 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336588614.060989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 1601f8ad3c..5a7912d23d 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336588614.060989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/events.rec.log b/testing/btest/Baseline/istate.events/events.rec.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events/events.rec.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events/events.snd.log b/testing/btest/Baseline/istate.events/events.snd.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events/events.snd.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index 25a7f289c0..55a0189cec 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336587178.164598 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index 25a7f289c0..55a0189cec 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336587178.164598 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out new file mode 100644 index 0000000000..ebac1866b6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out @@ -0,0 +1,14 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out new file mode 100644 index 0000000000..f75248cf97 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out @@ -0,0 +1,4 @@ +{ +[2] = [b=], +[1] = [b=T] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out new file mode 100644 index 0000000000..bb3b6d0a9e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -0,0 +1,70 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] +Input::EVENT_NEW +1 +T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] +Input::EVENT_NEW +2 +T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] +Input::EVENT_NEW +3 +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] +Input::EVENT_NEW +4 +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] +Input::EVENT_NEW +5 +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] +Input::EVENT_NEW +6 +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] +Input::EVENT_NEW +7 +T diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out new file mode 100644 index 0000000000..a38f3fce84 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out @@ -0,0 +1,9 @@ +[source=wc -l ../input.log |, reader=Input::READER_RAW, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print outfile, description; +print outfile, tpe; +print outfile, s; +close(outfile); +}] +Input::EVENT_NEW +8 ../input.log diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out new file mode 100644 index 0000000000..bb69da3267 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out @@ -0,0 +1,145 @@ +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out new file mode 100644 index 0000000000..bbce48f4f6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out @@ -0,0 +1,3 @@ +{ +[-42] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out new file mode 100644 index 0000000000..3f9af35c59 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out @@ -0,0 +1,3 @@ +{ +[-42] = [b=T] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.optional/out b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out new file mode 100644 index 0000000000..7a304fc918 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out @@ -0,0 +1,9 @@ +{ +[2] = [b=T, notb=F], +[4] = [b=F, notb=T], +[6] = [b=F, notb=T], +[7] = [b=T, notb=F], +[1] = [b=T, notb=F], +[5] = [b=F, notb=T], +[3] = [b=F, notb=T] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out new file mode 100644 index 0000000000..858551aa2f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -0,0 +1,3 @@ +[p=80/tcp] +[p=52/udp] +[p=30/unknown] diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out new file mode 100644 index 0000000000..c648e63710 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out @@ -0,0 +1,4 @@ +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=T, s=testmodified] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out new file mode 100644 index 0000000000..0adccc1856 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out @@ -0,0 +1,23 @@ +Update_finished for input, try 1 +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=T, s=testmodified] +} +Update_finished for input, try 2 +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=F, s=testmodified] +} +Update_finished for input, try 3 +{ +[2, idxmodified] = [b=F, s=test2], +[1, idx1] = [b=F, s=testmodified] +} +Update_finished for input, try 4 +{ +[2, idxmodified] = [b=F, s=test2] +} +Update_finished for input, try 5 +{ +[1, idx1] = [b=T, s=testmodified] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out new file mode 100644 index 0000000000..55e7610e1e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -0,0 +1,64 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out new file mode 100644 index 0000000000..71de0d2570 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out @@ -0,0 +1,160 @@ +input0 +input.log +{ +[1] = T +} +input1 +input.log +{ +[1] = T +} +input2 +input.log +{ +[1] = T +} +input3 +input.log +{ +[1] = T +} +input4 +input.log +{ +[1] = T +} +input5 +input.log +{ +[1] = T +} +input6 +input.log +{ +[1] = T +} +input7 +input.log +{ +[1] = T +} +input8 +input.log +{ +[1] = T +} +input9 +input.log +{ +[1] = T +} +input10 +input.log +{ +[1] = T +} +input11 +input.log +{ +[1] = T +} +input12 +input.log +{ +[1] = T +} +input13 +input.log +{ +[1] = T +} +input14 +input.log +{ +[1] = T +} +input15 +input.log +{ +[1] = T +} +input16 +input.log +{ +[1] = T +} +input17 +input.log +{ +[1] = T +} +input18 +input.log +{ +[1] = T +} +input19 +input.log +{ +[1] = T +} +input20 +input.log +{ +[1] = T +} +input21 +input.log +{ +[1] = T +} +input22 +input.log +{ +[1] = T +} +input23 +input.log +{ +[1] = T +} +input24 +input.log +{ +[1] = T +} +input25 +input.log +{ +[1] = T +} +input26 +input.log +{ +[1] = T +} +input27 +input.log +{ +[1] = T +} +input28 +input.log +{ +[1] = T +} +input29 +input.log +{ +[1] = T +} +input30 +input.log +{ +[1] = T +} +input31 +input.log +{ +[1] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out new file mode 100644 index 0000000000..5cce15f6c7 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -0,0 +1,1288 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-42] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-43] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_CHANGED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_CHANGED +Left +[i=-43] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-45] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-46] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-47] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-48] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-43] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-46] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-44] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-47] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-45] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-42] +Left +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +==========SERVERS============ +{ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out new file mode 100644 index 0000000000..9d62fdbef4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out @@ -0,0 +1,128 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.stream/out b/testing/btest/Baseline/scripts.base.frameworks.input.stream/out new file mode 100644 index 0000000000..39b06c9092 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.stream/out @@ -0,0 +1,115 @@ +============EVENT============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_CHANGED +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out new file mode 100644 index 0000000000..07a3ffdba5 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out @@ -0,0 +1,120 @@ +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out new file mode 100644 index 0000000000..a1bbb9bbe4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -0,0 +1,126 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] +Input::EVENT_NEW +[i=1] +T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] +Input::EVENT_NEW +[i=2] +T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] +Input::EVENT_NEW +[i=3] +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] +Input::EVENT_NEW +[i=4] +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] +Input::EVENT_NEW +[i=5] +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] +Input::EVENT_NEW +[i=6] +F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] +Input::EVENT_NEW +[i=7] +T diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out new file mode 100644 index 0000000000..e9e03add3a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out @@ -0,0 +1,172 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE 2============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +============EVENT============ +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/Traces/trunc/icmp-header-trunc.pcap b/testing/btest/Traces/trunc/icmp-header-trunc.pcap new file mode 100644 index 0000000000..5765cf2886 Binary files /dev/null and b/testing/btest/Traces/trunc/icmp-header-trunc.pcap differ diff --git a/testing/btest/Traces/trunc/icmp-payload-trunc.pcap b/testing/btest/Traces/trunc/icmp-payload-trunc.pcap new file mode 100644 index 0000000000..13607dd50c Binary files /dev/null and b/testing/btest/Traces/trunc/icmp-payload-trunc.pcap differ diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 6afbde1ddb..4c4074ee24 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -3,7 +3,7 @@ TestDirs = doc bifs language core scripts istate coverage TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* *.trace +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/core/ipv6-flow-labels.test b/testing/btest/core/ipv6-flow-labels.test new file mode 100644 index 0000000000..b4e60cb0a4 --- /dev/null +++ b/testing/btest/core/ipv6-flow-labels.test @@ -0,0 +1,32 @@ +# @TEST-EXEC: bro -b -r $TRACES/ipv6-ftp.trace %INPUT >output +# @TEST-EXEC: btest-diff output + +function print_connection(c: connection, event_name: string) + { + print fmt("%s: %s", event_name, c$id); + print fmt(" orig_flow %d", c$orig$flow_label); + print fmt(" resp_flow %d", c$resp$flow_label); + } + +event new_connection(c: connection) + { + print_connection(c, "new_connection"); + } + +event connection_established(c: connection) + { + print_connection(c, "connection_established"); + } + +event connection_state_remove(c: connection) + { + print_connection(c, "connection_state_remove"); + } + +event connection_flow_label_changed(c: connection, is_orig: bool, + old_label: count, new_label: count) + { + print_connection(c, fmt("connection_flow_label_changed(%s)", is_orig ? "orig" : "resp")); + print fmt(" old_label %d", old_label); + print fmt(" new_label %d", new_label); + } diff --git a/testing/btest/core/truncation.test b/testing/btest/core/truncation.test index ee8bdd5bf9..3406879183 100644 --- a/testing/btest/core/truncation.test +++ b/testing/btest/core/truncation.test @@ -6,4 +6,17 @@ # @TEST-EXEC: cat weird.log >> output # @TEST-EXEC: bro -r $TRACES/trunc/ip6-ext-trunc.pcap # @TEST-EXEC: cat weird.log >> output + +# If an ICMP packet's payload is truncated due to too small snaplen, +# the checksum calculation is bypassed (and Bro doesn't crash, of course). + +# @TEST-EXEC: rm -f weird.log +# @TEST-EXEC: bro -r $TRACES/trunc/icmp-payload-trunc.pcap +# @TEST-EXEC: test ! -e weird.log + +# If an ICMP packet has the ICMP header truncated due to too small snaplen, +# an internally_truncated_header weird gets generated. + +# @TEST-EXEC: bro -r $TRACES/trunc/icmp-header-trunc.pcap +# @TEST-EXEC: cat weird.log >> output # @TEST-EXEC: btest-diff output diff --git a/testing/btest/istate/bro-ipv6-socket.bro b/testing/btest/istate/bro-ipv6-socket.bro new file mode 100644 index 0000000000..b339bf4487 --- /dev/null +++ b/testing/btest/istate/bro-ipv6-socket.bro @@ -0,0 +1,56 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1" +# +# @TEST-EXEC: btest-bg-run recv bro -b ../recv.bro +# @TEST-EXEC: btest-bg-run send bro -b ../send.bro +# @TEST-EXEC: btest-bg-wait -k 20 +# +# @TEST-EXEC: btest-diff recv/.stdout +# @TEST-EXEC: btest-diff send/.stdout + +@TEST-START-FILE send.bro + +@load base/frameworks/communication + +redef Communication::nodes += { + ["foo"] = [$host=[::1], $connect=T, $events=/my_event/] +}; + +global my_event: event(s: string); + +event remote_connection_handshake_done(p: event_peer) + { + print fmt("handshake done with peer: %s", p$host); + } + +event my_event(s: string) + { + print fmt("my_event: %s", s); + terminate(); + } + +@TEST-END-FILE + +############# + +@TEST-START-FILE recv.bro + +@load frameworks/communication/listen + +redef Communication::listen_ipv6=T; + +global my_event: event(s: string); + +event remote_connection_handshake_done(p: event_peer) + { + print fmt("handshake done with peer: %s", p$host); + event my_event("hello world"); + } + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +@TEST-END-FILE diff --git a/testing/btest/istate/broccoli-ipv6-socket.bro b/testing/btest/istate/broccoli-ipv6-socket.bro new file mode 100644 index 0000000000..21067c1b23 --- /dev/null +++ b/testing/btest/istate/broccoli-ipv6-socket.bro @@ -0,0 +1,10 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib +# @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1" +# +# @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ipv6=T" +# @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs -6 ::1 +# @TEST-EXEC: btest-bg-wait -k 20 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-diff broccoli/.stdout diff --git a/testing/btest/istate/broccoli-ipv6.bro b/testing/btest/istate/broccoli-ipv6.bro index cd0b546ce7..ba181d4987 100644 --- a/testing/btest/istate/broccoli-ipv6.bro +++ b/testing/btest/istate/broccoli-ipv6.bro @@ -2,14 +2,8 @@ # # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib # -# @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broccoli-v6addrs.bro +# @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs # @TEST-EXEC: btest-bg-wait -k 20 # @TEST-EXEC: btest-diff bro/.stdout # @TEST-EXEC: btest-diff broccoli/.stdout - -event remote_connection_closed(p: event_peer) - { - terminate(); - } - diff --git a/testing/btest/istate/broccoli-ssl.bro b/testing/btest/istate/broccoli-ssl.bro new file mode 100644 index 0000000000..4465cd1bb3 --- /dev/null +++ b/testing/btest/istate/broccoli-ssl.bro @@ -0,0 +1,68 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib +# +# @TEST-EXEC: chmod 600 broccoli.conf +# @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ssl=T" "ssl_ca_certificate=../ca_cert.pem" "ssl_private_key=../bro.pem" +# @TEST-EXEC: btest-bg-run broccoli BROCCOLI_CONFIG_FILE=../broccoli.conf $BUILD/aux/broccoli/test/broccoli-v6addrs +# @TEST-EXEC: btest-bg-wait -k 20 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-diff broccoli/.stdout + +@TEST-START-FILE broccoli.conf +/broccoli/use_ssl yes +/broccoli/ca_cert ../ca_cert.pem +/broccoli/host_cert ../bro.pem +/broccoli/host_key ../bro.pem +@TEST-END-FILE + +@TEST-START-FILE bro.pem +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQD17FE8UVaO224Y8UL2bH1okCYxr5dVytTQ93uE5J9caGADzPZe +qYPuvtPt9ivhBtf2L9odK7unQU60v6RsO3bb9bQktQbEdh0FEjnso2UHe/nLreYn +VyLCEp9Sh1OFQnMhJNYuzNwVzWOqH/TYNy3ODueZTS4YBsRyEkpEfgeoaQIDAQAB +AoGAJ/S1Xi94+Mz+Hl9UmeUWmx6QlhIJbI7/9NPA5d6fZcwvjW6HuOmh3fBzTn5o +sq8B96Xesk6gtpQNzaA1fsBKlzDSpGRDVg2odN9vIT3jd0Dub2F47JHdFCqtMUIV +rCsO+fpGtavv1zJ/rzlJz7rx4cRP+/Gwd5YlH0q5cFuHhAECQQD9q328Ye4A7o2e +cLOhzuWUZszqdIY7ZTgDtk06F57VrjLVERrZjrtAwbs77m+ybw4pDKKU7H5inhQQ +03PU40ARAkEA+C6cCM6E4hRwuR+QyIqpNC4CzgPaKlF+VONZLYYvHEwFvx2/EPtX +zOZdE4HdJwnXBYx7+AGFeq8uHhrN2Tq62QJBAMory2JAinejqKsGF6R2SPMlm1ug +0vqziRksShBqkuSqmUjHASczYnoR7S+usMb9S8PblhgrA++FHWjrnf2lwIECQQCj ++/AfpY2J8GWW/HNm/q/UiX5S75qskZI+tsXK3bmtIdI+OIJxzxFxktj3NbyRud+4 +i92xvhebO7rmK2HOYg7pAkEA2wrwY1E237twoYXuUInv9F9kShKLQs19nup/dfmF +xfoVqYjJwidzPfgngowJZij7SoTaIBKv/fKp5Tq6xW3AEg== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICZDCCAc2gAwIBAgIJAKoxR9yFGsk8MA0GCSqGSIb3DQEBBQUAMCsxKTAnBgNV +BAMTIEJybyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MCAXDTExMDYxNTIx +MjgxNVoYDzIxMTEwNTIyMjEyODE1WjArMSkwJwYDVQQDEyBCcm8gUm9vdCBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA +9exRPFFWjttuGPFC9mx9aJAmMa+XVcrU0Pd7hOSfXGhgA8z2XqmD7r7T7fYr4QbX +9i/aHSu7p0FOtL+kbDt22/W0JLUGxHYdBRI57KNlB3v5y63mJ1ciwhKfUodThUJz +ISTWLszcFc1jqh/02Dctzg7nmU0uGAbEchJKRH4HqGkCAwEAAaOBjTCBijAdBgNV +HQ4EFgQU2vIsKYuGhHP8c7GeJLfWAjbKCFgwWwYDVR0jBFQwUoAU2vIsKYuGhHP8 +c7GeJLfWAjbKCFihL6QtMCsxKTAnBgNVBAMTIEJybyBSb290IENlcnRpZmljYXRp +b24gQXV0aG9yaXR5ggkAqjFH3IUayTwwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B +AQUFAAOBgQAF2oceL61dA7WxA9lxcxsA/Fccr7+J6sO+pLXoZtx5tpknEuIUebkm +UfMGAiyYIenHi8u0Sia8KrIfuCDc2dG3DYmfX7/faCEbtSx8KtNQFIs3aXr1zhsw +3sX9fLS0gp/qHoPMuhbhlvTlMFSE/Mih3KDsZEGcifzI6ooLF0YP5A== +-----END CERTIFICATE----- +@TEST-END-FILE + +@TEST-START-FILE ca_cert.pem +-----BEGIN CERTIFICATE----- +MIICZDCCAc2gAwIBAgIJAKoxR9yFGsk8MA0GCSqGSIb3DQEBBQUAMCsxKTAnBgNV +BAMTIEJybyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MCAXDTExMDYxNTIx +MjgxNVoYDzIxMTEwNTIyMjEyODE1WjArMSkwJwYDVQQDEyBCcm8gUm9vdCBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA +9exRPFFWjttuGPFC9mx9aJAmMa+XVcrU0Pd7hOSfXGhgA8z2XqmD7r7T7fYr4QbX +9i/aHSu7p0FOtL+kbDt22/W0JLUGxHYdBRI57KNlB3v5y63mJ1ciwhKfUodThUJz +ISTWLszcFc1jqh/02Dctzg7nmU0uGAbEchJKRH4HqGkCAwEAAaOBjTCBijAdBgNV +HQ4EFgQU2vIsKYuGhHP8c7GeJLfWAjbKCFgwWwYDVR0jBFQwUoAU2vIsKYuGhHP8 +c7GeJLfWAjbKCFihL6QtMCsxKTAnBgNVBAMTIEJybyBSb290IENlcnRpZmljYXRp +b24gQXV0aG9yaXR5ggkAqjFH3IUayTwwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B +AQUFAAOBgQAF2oceL61dA7WxA9lxcxsA/Fccr7+J6sO+pLXoZtx5tpknEuIUebkm +UfMGAiyYIenHi8u0Sia8KrIfuCDc2dG3DYmfX7/faCEbtSx8KtNQFIs3aXr1zhsw +3sX9fLS0gp/qHoPMuhbhlvTlMFSE/Mih3KDsZEGcifzI6ooLF0YP5A== +-----END CERTIFICATE----- +@TEST-END-FILE diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index 03784addef..d87d014a21 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -8,8 +8,10 @@ # @TEST-EXEC: btest-diff receiver/http.log # @TEST-EXEC: cmp sender/http.log receiver/http.log # -# @TEST-EXEC: bro -x sender/events.bst http/base | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.snd.log -# @TEST-EXEC: bro -x receiver/events.bst http/base | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.rec.log +# @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.snd.log +# @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.rec.log +# @TEST-EXEC: btest-diff events.rec.log +# @TEST-EXEC: btest-diff events.snd.log # @TEST-EXEC: cmp events.rec.log events.snd.log # # We don't compare the transmitted event paramerters anymore. With the dynamic diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index 81d9cc61b6..fe588b5c3b 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -10,6 +10,8 @@ # # @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.snd.log # @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.rec.log +# @TEST-EXEC: btest-diff events.rec.log +# @TEST-EXEC: btest-diff events.snd.log # @TEST-EXEC: cmp events.rec.log events.snd.log # # We don't compare the transmitted event paramerters anymore. With the dynamic diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro new file mode 100644 index 0000000000..8d4028a12e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -0,0 +1,50 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); +} + +event Input::update_finished(name: string, source:string) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro new file mode 100644 index 0000000000..77659d13ec --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +##types bool int +T 1 +- 2 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); +} + +event Input::update_finished(name: string, source:string) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro new file mode 100644 index 0000000000..dca75334d0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + + +module A; + +type Val: record { + i: int; + b: bool; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) { + print description; + print tpe; + print i; + print b; +} + +event bro_init() +{ + Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro new file mode 100644 index 0000000000..6df28d08ea --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +@load frameworks/communication/listen + +global outfile: file; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + close(outfile); +} + +event bro_init() +{ + outfile = open ("../out.tmp"); + Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/executestreamraw.bro b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro new file mode 100644 index 0000000000..d97a7b26a0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro @@ -0,0 +1,58 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. + +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + try = try + 1; + + if ( try == 9 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro new file mode 100644 index 0000000000..d6c81cb2db --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -0,0 +1,36 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} + diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro new file mode 100644 index 0000000000..ca1e956f35 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -0,0 +1,36 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + Input::add_table([$name="input", $source="input.log", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} + diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro new file mode 100644 index 0000000000..c354f7c3ab --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -0,0 +1,45 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + notb: bool &optional; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro new file mode 100644 index 0000000000..88e86eb5dc --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -0,0 +1,40 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#fields i p t +1.2.3.4 80 tcp +1.2.3.5 52 udp +1.2.3.6 30 unknown +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: addr; +}; + +type Val: record { + p: port &type_column="t"; +}; + +global servers: table[addr] of Val = table(); + +event bro_init() +{ + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers]); + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; +} + diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro new file mode 100644 index 0000000000..20c69131cb --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -0,0 +1,80 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out +# +# only difference from predicate.bro is, that this one uses a stream source. +# the reason is, that the code-paths are quite different, because then the ascii reader uses the put and not the sendevent interface + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); +global ct: int; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { + ct = ct + 1; + if ( ct < 3 ) { + return; + } + if ( ct > 3 ) { + print "Too many events"; + return; + } + + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} + +event bro_init() +{ + ct = 0; + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); + Input::remove("input"); + +} + diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro new file mode 100644 index 0000000000..278ac7418e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -0,0 +1,64 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro new file mode 100644 index 0000000000..c3198d8483 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro @@ -0,0 +1,50 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; + ss: string; +}; + +type Val: record { + b: bool; + s: string; +}; + +global servers: table[int, string] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { + if ( left$i == 1 ) { + right$s = "testmodified"; + } + + if ( left$i == 2 ) { + left$ss = "idxmodified"; + } + return T; + } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro new file mode 100644 index 0000000000..1606ff6a27 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -0,0 +1,107 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input4.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input5.log input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out +# + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 F test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 F test1 idx1 +2 F test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input4.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +2 F test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input5.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; + ss: string; +}; + +type Val: record { + b: bool; + s: string; +}; + +global servers: table[int, string] of Val = table(); +global outfile: file; +global try: count; + +event bro_init() +{ + try = 0; + outfile = open ("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $mode=Input::REREAD, + $pred(typ: Input::Event, left: Idx, right: Val) = { + if ( left$i == 1 ) { + right$s = "testmodified"; + } + + if ( left$i == 2 ) { + left$ss = "idxmodified"; + } + return T; + } + ]); +} + +event Input::update_finished(name: string, source: string) { + try = try + 1; + print outfile, fmt("Update_finished for %s, try %d", name, try); + print outfile, servers; + + if ( try == 5 ) { + close (outfile); + Input::remove("input"); + } +} diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro new file mode 100644 index 0000000000..8ec6c12a78 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro new file mode 100644 index 0000000000..58ce9a1675 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -0,0 +1,41 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32}; + +event bro_init() +{ + for ( i in one_to_32 ) { + Input::add_table([$source="input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]); + Input::remove(fmt("input%d", i)); + } +} + +event Input::update_finished(name: string, source: string) { + print name; + print source; + print destination; +} diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro new file mode 100644 index 0000000000..f33b060fe0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -0,0 +1,132 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input4.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input5.log input.log +# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input4.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -45 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -46 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -47 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input5.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; + print outfile, tpe; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); +} + + +event Input::update_finished(name: string, source: string) { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 5 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro new file mode 100644 index 0000000000..33361ad27e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line]); + Input::force_update("input"); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro new file mode 100644 index 0000000000..571a2273c1 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -0,0 +1,83 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, tpe; + print outfile, left; + print outfile, right; + print outfile, "============SERVERS============"; + print outfile, servers; + + try = try + 1; + + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); +} + diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro new file mode 100644 index 0000000000..cc0afd5ae8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -0,0 +1,56 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro new file mode 100644 index 0000000000..e40485dd12 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -0,0 +1,42 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { + print description; + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro new file mode 100644 index 0000000000..1413275e63 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -0,0 +1,116 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; +# print outfile, "Description"; +# print outfile, description; +# print outfile, "Type"; +# print outfile, tpe; +# print outfile, "Left"; +# print outfile, left; +# print outfile, "Right"; +# print outfile, right; +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); + Input::add_table([$source="../input2.log", $mode=Input::REREAD, $name="ssh2", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE 2============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); +} + + +event Input::update_finished(name: string, source: string) { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 3 ) { + print outfile, "done"; + print outfile, servers; + close(outfile); + Input::remove("input"); + Input::remove("input2"); + terminate(); + } +}