From 3fe930dbf2a704d46c56ca574a75bf838856c233 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 30 Jun 2022 17:01:13 +0200 Subject: [PATCH] Introduce telemetry framework Adds base/frameworks/telemetry with wrappers around telemetry.bif and updates telemetry/Manager to support collecting metrics from script land. Add policy/frameworks/telemetry/log for logging of metrics data into a new telemetry.log and telemetry_histogram.log and add into local.zeek by default. --- NEWS | 17 + .../base/frameworks/telemetry/__load__.zeek | 1 + scripts/base/frameworks/telemetry/main.zeek | 610 ++++++++++++++++++ scripts/base/init-default.zeek | 1 + scripts/policy/frameworks/telemetry/log.zeek | 199 ++++++ scripts/site/local.zeek | 4 + scripts/test-all-policy.zeek | 1 + src/telemetry/CMakeLists.txt | 2 +- src/telemetry/Manager.cc | 457 +++++++++++++ src/telemetry/Manager.h | 135 ++++ src/telemetry/telemetry.bif | 36 ++ .../canonified_loaded_scripts.log | 4 +- .../btest/Baseline/coverage.find-bro-logs/out | 2 + testing/btest/Baseline/plugins.hooks/output | 99 ++- .../out | 25 + .../out | 13 + .../out | 26 + .../telemetry.log | 11 + .../telemetry_histogram.log | 12 + .../telemetry_histogram.log.filtered | 3 + .../telemetry.log.filtered | 5 + .../telemetry_histogram.log.filtered | 3 + .../btest/plugins/hooks-plugin/src/Plugin.cc | 30 +- .../plugins/unprocessed-packet-hook.zeek | 1 - .../base/frameworks/telemetry/basic.zeek | 116 ++++ .../telemetry/conn-duration-histogram.zeek | 48 ++ .../telemetry/internal-metrics.zeek | 43 ++ .../frameworks/telemetry/log-prefixes.zeek | 34 + .../policy/frameworks/telemetry/log.zeek | 25 + testing/external/commit-hash.zeek-testing | 2 +- .../external/commit-hash.zeek-testing-private | 2 +- testing/external/scripts/testing-setup.zeek | 10 + 32 files changed, 1950 insertions(+), 27 deletions(-) create mode 100644 scripts/base/frameworks/telemetry/__load__.zeek create mode 100644 scripts/base/frameworks/telemetry/main.zeek create mode 100644 scripts/policy/frameworks/telemetry/log.zeek create mode 100644 testing/btest/Baseline/scripts.base.frameworks.telemetry.basic/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.telemetry.conn-duration-histogram/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.telemetry.internal-metrics/out create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log.filtered create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered create mode 100644 testing/btest/scripts/base/frameworks/telemetry/basic.zeek create mode 100644 testing/btest/scripts/base/frameworks/telemetry/conn-duration-histogram.zeek create mode 100644 testing/btest/scripts/base/frameworks/telemetry/internal-metrics.zeek create mode 100644 testing/btest/scripts/policy/frameworks/telemetry/log-prefixes.zeek create mode 100644 testing/btest/scripts/policy/frameworks/telemetry/log.zeek diff --git a/NEWS b/NEWS index a00c8cc7d2..ea9ebc1277 100644 --- a/NEWS +++ b/NEWS @@ -28,6 +28,23 @@ New Functionality patterns in Zeek scripts will cause the '.' character to also match newline characters. +- Added a new telemetry framework for providing high-level access to Broker's + metric subsystem. This framework allows script writers to use different + metric types (counters, gauges and histograms) for tracking metrics without + using lower-level BIFs from ``telemetry.bif``. Additionally, metrics can + now be accessed from script land using ``Telemetry::collect_metrics() and + ``Telemetry::collect_histogram_metrics()``. + + The framework is located in ``base/frameworks/telemetry``. + + In addition to the Prometheus endpoint for metrics export that has existed + since Zeek 4.1, two new log streams, ``telemetry.log`` and ``telemetry_histogram.log``, + can be enabled by loading ``policy/frameworks/telemetry/log``. This policy + script is included in ``local.zeek`` by default. + + For further details on the framework and examples, please refer to the + Zeek documentation. + Changed Functionality --------------------- diff --git a/scripts/base/frameworks/telemetry/__load__.zeek b/scripts/base/frameworks/telemetry/__load__.zeek new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/frameworks/telemetry/__load__.zeek @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/frameworks/telemetry/main.zeek b/scripts/base/frameworks/telemetry/main.zeek new file mode 100644 index 0000000000..ec14a3e887 --- /dev/null +++ b/scripts/base/frameworks/telemetry/main.zeek @@ -0,0 +1,610 @@ +##! Module for recording and querying metrics. This modules wraps +##! the lower-level telemetry.bif functions. +##! +##! Metrics will be exposed through a Prometheus HTTP endpoint when +##! enabled by setting :zeek:see:`Broker::metrics_port` or using the +##! `BROKER_METRICS_PORT` environment variable. + +@load base/misc/version + +module Telemetry; + +export { + ## Alias for a vector of label values. + type labels_vector: vector of string; + + ## Type that captures options used to create metrics. + type MetricOpts: record { + ## The prefix (namespace) of the metric. + prefix: string; + + ## The human-readable name of the metric. + name: string; + + ## The unit of the metric. Use the pseudo-unit "1" if this is a unit-less metric. + unit: string; + + ## Documentation for this metric. + help_text: string; + + ## The label names (also called dimensions) of the metric. When + ## instantiating or working with concrete metrics, corresponding + ## label values have to be provided. + labels: vector of string &default=vector(); + + ## Whether the metric represents something that is accumulating. + ## Defaults to ``T`` for counters and ``F`` for gauges and + ## histograms. + is_total: bool &optional; + + ## When creating a :zeek:see:`Telemetry::HistogramFamily`, + ## describes the number and bounds of the individual buckets. + bounds: vector of double &optional; + + ## The same meaning as *bounds*, but as :zeek:type:`count`. + ## Only set in the return value of + ## :zeek:see:`Telemetry::collect_histogram_metrics`. + ## for histograms when the underlying type is ``int64_t``, + ## otherwise ignored. + count_bounds: vector of count &optional; + + ## Describes the underlying metric type. + ## Only set in the return value of + ## :zeek:see:`Telemetry::collect_metrics` or + ## :zeek:see:`Telemetry::collect_histogram_metrics`, + ## otherwise ignored. + metric_type: MetricType &optional; + }; + + ## Type representing a family of counters with uninitialized label values. + ## + ## To create concrete :zeek:see:`Telemetry::Counter` instances, use + ## :zeek:see:`Telemetry::counter_with`. To modify counters directly + ## use :zeek:see:`Telemetry::counter_family_inc`. + type CounterFamily: record { + __family: opaque of dbl_counter_metric_family; + __labels: vector of string; + }; + + ## Type representing a counter metric with initialized label values. + ## + ## Counter metrics only ever go up and reset when the process + ## restarts. Use :zeek:see:`Telemetry::counter_inc` or + ## :zeek:see:`Telemetry::counter_set` to modify counters. + ## An example for a counter is the number of log writes + ## per :zeek:see:`Log::Stream` or number connections broken down + ## by protocol and service. + type Counter: record { + __metric: opaque of dbl_counter_metric; + }; + + ## Register a counter family. + global register_counter_family: function(opts: MetricOpts): CounterFamily; + + ## Get a :zeek:see:`Telemetry::Counter` instance given family and label values. + global counter_with: function(cf: CounterFamily, + label_values: labels_vector &default=vector()): Counter; + + ## Increment a :zeek:see:`Telemetry::Counter` by `amount`. + ## Using a negative `amount` is an error. + ## + ## c: The counter instance. + ## + ## amount: The amount by which to increment the counter. + ## + ## Returns: True if the counter was incremented successfully. + global counter_inc: function(c: Counter, amount: double &default=1.0): bool; + + ## Helper to set a :zeek:see:`Telemetry::Counter` to the given `value`. + ## This can be useful for mirroring counter metrics in an + ## :zeek:see:`Telemetry::sync` hook implementation. + ## Setting a value that is less than the current value of the + ## metric is an error and will be ignored. + ## + ## c: The counter instance. + ## + ## value: The value to set the counter to. + ## + ## Returns: True if the counter value was set successfully. + global counter_set: function(c: Counter, value: double): bool; + + ## Increment a :zeek:see:`Telemetry::Counter` through the :zeek:see:`Telemetry::CounterFamily`. + ## This is a short-cut for :zeek:see:`Telemetry::counter_inc`. + ## Using a negative amount is an error. + ## + ## cf: The counter family to use. + ## + ## label_values: The label values to use for the counter. + ## + ## amount: The amount by which to increment the counter. + ## + ## Returns: True if the counter was incremented successfully. + global counter_family_inc: function(cf: CounterFamily, + label_values: labels_vector &default=vector(), + amount: double &default=1.0): bool; + + ## Set a :zeek:see:`Telemetry::Counter` through the :zeek:see:`Telemetry::CounterFamily`. + ## This is a short-cut for :zeek:see:`Telemetry::counter_set`. + ## Setting a value that is less than the current value of the + ## metric is an error and will be ignored. + ## + ## cf: The counter family to use. + ## + ## label_values: The label values to use for the counter. + ## + ## value: The value to set the counter to. + ## + ## Returns: True if the counter value was set successfully. + global counter_family_set: function(cf: CounterFamily, + label_values: labels_vector, + value: double): bool; + + ## Type representing a family of gauges with uninitialized label values. + ## + ## Create concrete :zeek:see:`Telemetry::Gauge` instances with + ## :zeek:see:`Telemetry::gauge_with`, or use + ## :zeek:see:`Telemetry::gauge_family_inc` or + ## :zeek:see:`Telemetry::gauge_family_set` directly. + type GaugeFamily: record { + __family: opaque of dbl_gauge_metric_family; + __labels: vector of string; + }; + + ## Type representing a gauge metric with initialized label values. + ## + ## Use :zeek:see:`Telemetry::gauge_inc`, :zeek:see:`Telemetry::gauge_dec`, + ## or :zeek:see:`Telemetry::gauge_set` to modify the gauge. + ## Example for gauges are process memory usage, table sizes + ## or footprints of long-lived values as determined by + ## :zeek:see:`val_footprint`. + type Gauge: record { + __metric: opaque of dbl_gauge_metric; + }; + + ## Register a gauge family. + global register_gauge_family: function(opts: MetricOpts): GaugeFamily; + + + ## Get a :zeek:see:`Telemetry::Gauge` instance given family and label values. + global gauge_with: function(gf: GaugeFamily, + label_values: labels_vector &default=vector()): Gauge; + + ## Increment a :zeek:see:`Telemetry::Gauge` by `amount`. + ## + ## g: The gauge instance. + ## + ## amount: The amount by which to increment the gauge. + ## + ## Returns: True if the gauge was incremented successfully. + global gauge_inc: function(g: Gauge, amount: double &default=1.0): bool; + + ## Decrement a :zeek:see:`Telemetry::Gauge` by `amount`. + ## + ## g: The gauge instance. + ## + ## amount: The amount by which to decrement the gauge. + ## + ## Returns: True if the gauge was incremented successfully. + global gauge_dec: function(g: Gauge, amount: double &default=1.0): bool; + + ## Helper to set a :zeek:see:`Telemetry::Gauge` to the given `value`. + ## + ## g: The gauge instance. + ## + ## value: The value to set the gauge to. + ## + ## Returns: True if the gauge value was set successfully. + global gauge_set: function(g: Gauge, value: double): bool; + + ## Increment a :zeek:see:`Telemetry::Gauge` by the given `amount` through + ## the :zeek:see:`Telemetry::GaugeFamily`. + ## This is a short-cut for :zeek:see:`Telemetry::gauge_inc`. + ## Using a negative amount is an error. + ## + ## gf: The gauge family to use. + ## + ## label_values: The label values to use for the gauge. + ## + ## amount: The amount by which to increment the gauge. + ## + ## Returns: True if the gauge was incremented successfully. + global gauge_family_inc: function(gf: GaugeFamily, + label_values: labels_vector &default=vector(), + amount: double &default=1.0): bool; + + ## Decrement a :zeek:see:`Telemetry::Gauge` by the given `amount` through + ## the :zeek:see:`Telemetry::GaugeFamily`. + ## This is a short-cut for :zeek:see:`Telemetry::gauge_dec`. + ## + ## gf: The gauge family to use. + ## + ## label_values: The label values to use for the gauge. + ## + ## amount: The amount by which to increment the gauge. + ## + ## Returns: True if the gauge was incremented successfully. + global gauge_family_dec: function(gf: GaugeFamily, + label_values: labels_vector &default=vector(), + amount: double &default=1.0): bool; + + ## Set a :zeek:see:`Telemetry::Gauge` to the given `value` through + ## the :zeek:see:`Telemetry::GaugeFamily`. + ## This is a short-cut for :zeek:see:`Telemetry::gauge_set`. + ## + ## gf: The gauge family to use. + ## + ## label_values: The label values to use for the gauge. + ## + ## value: The value to set the gauge to. + ## + ## Returns: True if the gauge value was set successfully. + global gauge_family_set: function(g: GaugeFamily, + label_values: labels_vector, + value: double): bool; + + ## Type representing a family of histograms with uninitialized label values. + ## Create concrete :zeek:see:`Telemetry::Histogram` instances with + ## :zeek:see:`Telemetry::histogram_with` or use + ## :zeek:see:`Telemetry::histogram_family_observe` directly. + type HistogramFamily: record { + __family: opaque of dbl_histogram_metric_family; + __labels: vector of string; + }; + + ## Type representing a histogram metric with initialized label values. + ## Use :zeek:see:`Telemetry::histogram_observe` to make observations. + type Histogram: record { + __metric: opaque of dbl_histogram_metric; + }; + + ## Register a histogram family. + global register_histogram_family: function(opts: MetricOpts): HistogramFamily; + + ## Get a :zeek:see:`Telemetry::Histogram` instance given family and label values. + global histogram_with: function(hf: HistogramFamily, + label_values: labels_vector &default=vector()): Histogram; + + ## Observe a measurement for a :zeek:see:`Telemetry::Histogram`. + ## + ## h: The histogram instance. + ## + ## measurement: The value for this observations. + ## + ## Returns: True if measurement was observed successfully. + global histogram_observe: function(h: Histogram, measurement: double): bool; + + ## Observe a measurement for a :zeek:see:`Telemetry::Histogram` through + ## the :zeek:see:`Telemetry::HistogramFamily`. + ## This is a short-cut for :zeek:see:`Telemetry::histogram_observe`. + ## + ## hf: The histogram family to use. + ## + ## label_values: The label values to use for the histogram. + ## + ## measurement: The value for this observations. + ## + ## Returns: True if measurement was observed successfully. + global histogram_family_observe: function(hf: HistogramFamily, + label_values: labels_vector, + measurement: double): bool; + + ## Telemetry sync hook. + ## + ## This hook is invoked every :zeek:see:`Telemetry::sync_interval` + ## for script writers to synchronize or mirror metrics with the + ## telemetry subsystem. For example, when tracking table or value + ## footprints with gauges, the value in question can be set on an actual + ## :zeek:see:`Telemetry::Gauge` instance during execution of this hook. + ## + ## Implementations should be lightweight, this hook may be called + ## multiple times per minute. The interval can increased by changing + ## :zeek:see:`Telemetry::sync_interval` at the cost of delaying + ## metric updates and thereby reducing granularity. + global sync: hook(); + + ## Interval at which the :zeek:see:`Telemetry::sync` hook is invoked. + option sync_interval = 10sec; + + ## Type of elements returned by the :zeek:see:`Telemetry::collect_metrics` function. + type Metric: record { + ## A :zeek:see:`Telemetry::MetricOpts` record describing this metric. + opts: MetricOpts; + + ## The label values associated with this metric, if any. + labels: vector of string; + + ## The value of gauge or counter cast to a double + ## independent of the underlying data type. + ## This value is set for all counter and gauge metrics, + ## it is unset for histograms. + value: double &optional; + + ## The value of the underlying gauge or counter as a double + ## if the underlying metric type uses ``int64_t``. + ## Only counters and gauges created with the C++ API may + ## have this value set. + count_value: count &optional; + }; + + ## Type of elements returned by the :zeek:see:`Telemetry::collect_histogram_metrics` function. + type HistogramMetric: record { + ## A :zeek:see:`Telemetry::MetricOpts` record describing this histogram. + opts: MetricOpts; + + ## The label values associated with this histogram, if any. + labels: vector of string; + + ## Individual counters for each of the buckets as + ## described by the *bounds* field in *opts*; + values: vector of double; + + ## If the underlying data type of the histogram is ``int64_t``, + ## this vector will hold the values as counts, otherwise it + ## is unset. Only histograms created with the C++ API have + ## may have this value set. + count_values: vector of count &optional; + + ## The number of observations made for this histogram. + observations: double; + + ## The sum of all observations for this histogram. + sum: double; + + ## If the underlying data type of the histogram is ``int64_t``, + ## the number of observations as :zeek:type:`count`, otherwise + ## unset. + count_observations: count &optional; + + ## If the underlying data type of the histogram is ``int64_t``, + ## the sum of all observations as :zeek:type:`count`, otherwise + ## unset. + count_sum: count &optional; + }; + + ## Collect all counter and gauge metrics matching the given *name* and *prefix*. + ## + ## For histogram metrics, use the :zeek:see:`Telemetry::collect_histogram_metrics`. + ## + ## The *prefix* and *name* parameters support globbing. By default, + ## all counters and gauges are returned. + global collect_metrics: function(prefix: string &default="*", + name: string &default="*"): vector of Metric; + + ## Collect all histograms and their observations matching the given + ## *prefix* and *name*. + ## + ## The *prefix* and *name* parameters support globbing. By default, + ## all histogram metrics are returned. + global collect_histogram_metrics: function(prefix: string &default="*", + name: string &default="*"): vector of HistogramMetric; +} + + +## Internal helper to create the labels table. +function make_labels(keys: vector of string, values: labels_vector): table[string] of string + { + local labels: table[string] of string; + for ( i in keys ) + labels[keys[i]] = values[i]; + + return labels; + } + +function register_counter_family(opts: MetricOpts): CounterFamily + { + local f = Telemetry::__dbl_counter_family( + opts$prefix, + opts$name, + opts$labels, + opts$help_text, + opts$unit, + opts?$is_total ? opts$is_total : T + ); + return CounterFamily($__family=f, $__labels=opts$labels); + } + +# Fallback Counter returned when there are issues with the labels. +global error_counter_cf = register_counter_family([ + $prefix="zeek", + $name="telemetry_counter_usage_error", + $unit="1", + $help_text="This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero." +]); + +function counter_with(cf: CounterFamily, label_values: labels_vector): Counter + { + if ( |cf$__labels| != |label_values| ) + { + Reporter::error(fmt("Invalid label values expected %s, have %s", |cf$__labels|, |label_values|)); + return counter_with(error_counter_cf); + } + + local labels = make_labels(cf$__labels, label_values); + local m = Telemetry::__dbl_counter_metric_get_or_add(cf$__family, labels); + return Counter($__metric=m); + } + +function counter_inc(c: Counter, amount: double): bool + { + return Telemetry::__dbl_counter_inc(c$__metric, amount); + } + +function counter_set(c: Counter, value: double): bool + { + local cur_value: double = Telemetry::__dbl_counter_value(c$__metric); + if (value < cur_value) + { + Reporter::error(fmt("Attempted to set lower counter value=%s cur_value=%s", value, cur_value)); + return F; + } + return Telemetry::__dbl_counter_inc(c$__metric, value - cur_value); + } + +function counter_family_inc(cf: CounterFamily, label_values: labels_vector, amount: double): bool + { + return counter_inc(counter_with(cf, label_values), amount); + } + +function counter_family_set(cf: CounterFamily, label_values: labels_vector, value: double): bool + { + return counter_set(counter_with(cf, label_values), value); + } + +function register_gauge_family(opts: MetricOpts): GaugeFamily + { + local f = Telemetry::__dbl_gauge_family( + opts$prefix, + opts$name, + opts$labels, + opts$help_text, + opts$unit, + opts?$is_total ? opts$is_total : F + ); + return GaugeFamily($__family=f, $__labels=opts$labels); + } + +# Fallback Gauge returned when there are issues with the label usage. +global error_gauge_cf = register_gauge_family([ + $prefix="zeek", + $name="telemetry_gauge_usage_error", + $unit="1", + $help_text="This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero." +]); + +function gauge_with(gf: GaugeFamily, label_values: labels_vector): Gauge + { + if ( |gf$__labels| != |label_values| ) + { + Reporter::error(fmt("Invalid label values expected %s, have %s", |gf$__labels|, |label_values|)); + return gauge_with(error_gauge_cf); + } + local labels = make_labels(gf$__labels, label_values); + local m = Telemetry::__dbl_gauge_metric_get_or_add(gf$__family, labels); + return Gauge($__metric=m); + } + +function gauge_inc(g: Gauge, amount: double &default=1.0): bool + { + return Telemetry::__dbl_gauge_inc(g$__metric, amount); + } + +function gauge_dec(g: Gauge, amount: double &default=1.0): bool + { + return Telemetry::__dbl_gauge_dec(g$__metric, amount); + } + +function gauge_set(g: Gauge, value: double): bool + { + # Telemetry currently does not implement __dbl_gauge_set(), do + # it by hand here. + local cur_value: double = Telemetry::__dbl_gauge_value(g$__metric); + if (value > cur_value) + return Telemetry::__dbl_gauge_inc(g$__metric, value - cur_value); + + return Telemetry::__dbl_gauge_dec(g$__metric, cur_value - value); + } + +function gauge_family_inc(gf: GaugeFamily, label_values: labels_vector, value: double): bool + { + return gauge_inc(gauge_with(gf, label_values), value); + } + +function gauge_family_dec(gf: GaugeFamily, label_values: labels_vector, value: double): bool + { + return gauge_dec(gauge_with(gf, label_values), value); + } + +function gauge_family_set(gf: GaugeFamily, label_values: labels_vector, value: double): bool + { + return gauge_set(gauge_with(gf, label_values), value); + } + +function register_histogram_family(opts: MetricOpts): HistogramFamily + { + local f = Telemetry::__dbl_histogram_family( + opts$prefix, + opts$name, + opts$labels, + opts$bounds, + opts$help_text, + opts$unit, + opts?$is_total ? opts$is_total : F + ); + return HistogramFamily($__family=f, $__labels=opts$labels); + } + +# Fallback Histogram when there are issues with the labels. +global error_histogram_hf = register_histogram_family([ + $prefix="zeek", + $name="telemetry_histogram_usage_error", + $unit="1", + $help_text="This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero.", + $bounds=vector(1.0) +]); + +function histogram_with(hf: HistogramFamily, label_values: labels_vector): Histogram + { + if ( |hf$__labels| != |label_values| ) + { + Reporter::error(fmt("Invalid label values expected %s, have %s", |hf$__labels|, |label_values|)); + return histogram_with(error_histogram_hf); + } + + local labels = make_labels(hf$__labels, label_values); + local m = Telemetry::__dbl_histogram_metric_get_or_add(hf$__family, labels); + return Histogram($__metric=m); + } + +function histogram_observe(h: Histogram, measurement: double): bool + { + return Telemetry::__dbl_histogram_observe(h$__metric, measurement); + } + +function histogram_family_observe(hf: HistogramFamily, label_values: labels_vector, measurement: double): bool + { + return histogram_observe(histogram_with(hf, label_values), measurement); + } + +function collect_metrics(prefix: string, name: string): vector of Metric + { + return Telemetry::__collect_metrics(prefix, name); + } + +function collect_histogram_metrics(prefix: string, name: string): vector of HistogramMetric + { + return Telemetry::__collect_histogram_metrics(prefix, name); + } + +event run_sync_hook() + { + hook Telemetry::sync(); + schedule sync_interval { run_sync_hook() }; + } + +event zeek_init() + { + schedule sync_interval { run_sync_hook() }; + } + +# Expose the Zeek version as Prometheus style info metric +global version_gauge_family = Telemetry::register_gauge_family([ + $prefix="zeek", + $name="version_info", + $unit="1", + $help_text="The Zeek version", + $labels=vector("version_number", "major", "minor", "patch", "commit", + "beta", "debug","version_string") +]); + +event zeek_init() + { + local v = Version::info; + local labels = vector(cat(v$version_number), + cat(v$major), cat(v$minor), cat (v$patch), + cat(v$commit), + v$beta ? "true" : "false", + v$debug ? "true" : "false", + v$version_string); + + Telemetry::gauge_family_set(version_gauge_family, labels, 1.0); + } diff --git a/scripts/base/init-default.zeek b/scripts/base/init-default.zeek index 32274faca6..081b102fe5 100644 --- a/scripts/base/init-default.zeek +++ b/scripts/base/init-default.zeek @@ -43,6 +43,7 @@ @load base/frameworks/tunnels @load base/frameworks/openflow @load base/frameworks/netcontrol +@load base/frameworks/telemetry @load base/protocols/conn @load base/protocols/dce-rpc diff --git a/scripts/policy/frameworks/telemetry/log.zeek b/scripts/policy/frameworks/telemetry/log.zeek new file mode 100644 index 0000000000..42ecb0cbc4 --- /dev/null +++ b/scripts/policy/frameworks/telemetry/log.zeek @@ -0,0 +1,199 @@ +##! Implementation of a telemetry.log and telemetry_histogram.log file +##! using metrics accessible via the Telemetry module. + +@load base/frameworks/telemetry + +module Telemetry; + +export { + redef enum Log::ID += { LOG, LOG_HISTOGRAM }; + + ## How often metrics are reported. + option log_interval = 60sec; + + ## Only metrics with prefixes in this set will be included in the + ## `telemetry.log` and `telemetry_histogram.log` files by default. + ## Setting this option to an empty set includes all prefixes. + ## + ## For more fine-grained customization, setting this option to an + ## empty set and implementing the :zeek:see:`Telemetry::log_policy` + ## and :zeek:see:`Telemetry::log_policy_histogram` hooks to filter + ## individual records is recommended. + option log_prefixes: set[string] = {"process", "zeek"}; + + ## Record type used for logging counter and gauge metrics. + type Info: record { + ## Timestamp of reporting. + ts: time &log; + + ## Peer that generated this log. + peer: string &log; + + ## Contains the value "counter" or "gauge" depending on + ## the underlying metric type. + metric_type: string &log; + + ## The prefix (namespace) of the metric. + prefix: string &log; + + ## The name of the metric. + name: string &log; + + ## The unit of this metric, or unset if unit-less. + unit: string &log &optional; + + ## The names of the individual labels. + labels: vector of string &log; + + ## The values of the labels as listed in ``labels``. + label_values: vector of string &log; + + ## The value of this metric. + value: double &log; + }; + + ## Record type used for logging histogram metrics. + type HistogramInfo: record { + ## Timestamp of reporting. + ts: time &log; + + ## Peer that generated this log. + peer: string &log; + + ## The prefix (namespace) of the metric. + prefix: string &log; + + ## The name of the metric. + name: string &log; + + ## The unit of this metric, or unset if unit-less. + unit: string &log &optional; + + ## The names of the individual labels. + labels: vector of string &log; + + ## The values of the labels as listed in ``labels``. + label_values: vector of string &log; + + ## The bounds of the individual buckets + bounds: vector of double &log; + + ## The number of observations within each individual bucket. + values: vector of double &log; + + ## The sum over all observations + sum: double &log; + + ## The total number of observations. + observations: double &log; + }; + + ## A default logging policy hook for the stream. + global log_policy: Log::PolicyHook; + + ## A default logging policy hook for the histogram stream. + global log_policy_histogram: Log::PolicyHook; + + ## Event triggered for every record in the stream. + global log_telemetry: event(rec: Info); + + ## Event triggered for every record in the histogram stream. + global log_telemetry_histogram: event(rec: HistogramInfo); +} + +function do_log() + { + local ts = network_time(); + local metrics = Telemetry::collect_metrics(); + + for ( i in metrics ) + { + local m = metrics[i]; + + # Histograms don't have single values, skip over them. + if ( m$opts$metric_type == DOUBLE_HISTOGRAM || m$opts$metric_type == INT_HISTOGRAM ) + next; + + if ( |log_prefixes| > 0 && m$opts$prefix !in log_prefixes ) + next; + + # Render the metric_type as a short string. Unknown + # shouldn't really happen, but lets have a fallback. + local metric_type = "unknown"; + switch ( m$opts$metric_type ) { + case DOUBLE_COUNTER, INT_COUNTER: + metric_type = "counter"; + break; + case DOUBLE_GAUGE, INT_GAUGE: + metric_type = "gauge"; + break; + } + + local rec = Info($ts=ts, + $peer=peer_description, + $metric_type=metric_type, + $prefix=m$opts$prefix, + $name=m$opts$name, + $labels=m$opts$labels, + $label_values=m$labels, + $value=m$value); + + if ( m$opts$unit != "1" ) + rec$unit = m$opts$unit; + + Log::write(LOG, rec); + } + + # Logging of histograms. + ts = network_time(); + local histogram_metrics = Telemetry::collect_histogram_metrics(); + for ( i in histogram_metrics ) + { + local hm = histogram_metrics[i]; + + if ( |log_prefixes| > 0 && hm$opts$prefix !in log_prefixes ) + next; + + local hrec = HistogramInfo($ts=ts, + $peer=peer_description, + $prefix=hm$opts$prefix, + $name=hm$opts$name, + $labels=hm$opts$labels, + $label_values=hm$labels, + $bounds=hm$opts$bounds, + $values=hm$values, + $sum=hm$sum, + $observations=hm$observations); + + if ( hm$opts$unit != "1" ) + hrec$unit = hm$opts$unit; + + Log::write(LOG_HISTOGRAM, hrec); + } + } + +event Telemetry::log() + { + # We explicitly log once during zeek_done(), so short-circuit + # here when we're already in the process of shutting down. + if ( zeek_is_terminating() ) + return; + + do_log(); + schedule log_interval { Telemetry::log() }; + } + +event zeek_init() &priority=5 + { + Log::create_stream(LOG, [$columns=Info, $ev=log_telemetry, $path="telemetry", $policy=log_policy]); + Log::create_stream(LOG_HISTOGRAM, [$columns=HistogramInfo, $ev=log_telemetry_histogram, $path="telemetry_histogram", $policy=log_policy_histogram]); + + schedule log_interval { Telemetry::log() }; + } + +# Log late during zeek_done() once more. Any metric updates +# afterwards won't be visible in the log. +event zeek_done() &priority=-1000 + { + do_log(); + } diff --git a/scripts/site/local.zeek b/scripts/site/local.zeek index 87838e4bee..9bf2ffc000 100644 --- a/scripts/site/local.zeek +++ b/scripts/site/local.zeek @@ -89,6 +89,10 @@ redef digest_salt = "Please change this value."; # Extend email alerting to include hostnames @load policy/frameworks/notice/extend-email/hostnames +# Enable logging of telemetry data into telemetry.log and +# telemetry_histogram.log. +@load frameworks/telemetry/log + # Uncomment the following line to enable detection of the heartbleed attack. Enabling # this might impact performance a bit. # @load policy/protocols/ssl/heartbleed diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 902a5163d8..ef752bece9 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -71,6 +71,7 @@ @load frameworks/software/version-changes.zeek @load frameworks/software/vulnerable.zeek @load frameworks/software/windows-version-detection.zeek +@load frameworks/telemetry/log.zeek @load integration/barnyard2/__load__.zeek @load integration/barnyard2/main.zeek @load integration/barnyard2/types.zeek diff --git a/src/telemetry/CMakeLists.txt b/src/telemetry/CMakeLists.txt index 542b7a8528..c4b8359a48 100644 --- a/src/telemetry/CMakeLists.txt +++ b/src/telemetry/CMakeLists.txt @@ -12,4 +12,4 @@ set(telemetry_SRCS bif_target(telemetry.bif) bro_add_subdir_library(telemetry ${telemetry_SRCS}) - +add_dependencies(bro_telemetry generate_outputs) diff --git a/src/telemetry/Manager.cc b/src/telemetry/Manager.cc index 66fb3d5b94..2ff0a5063a 100644 --- a/src/telemetry/Manager.cc +++ b/src/telemetry/Manager.cc @@ -2,11 +2,15 @@ #include "zeek/telemetry/Manager.h" +#include #include +#include #include "zeek/3rdparty/doctest.h" +#include "zeek/ID.h" #include "zeek/broker/Manager.h" #include "zeek/telemetry/Timer.h" +#include "zeek/telemetry/telemetry.bif.h" #include "broker/telemetry/metric_registry.hh" @@ -15,6 +19,33 @@ namespace using NativeManager = broker::telemetry::metric_registry; using NativeManagerImpl = broker::telemetry::metric_registry_impl; using NativeManagerImplPtr = zeek::IntrusivePtr; +using DoubleValPtr = zeek::IntrusivePtr; + +std::vector extract_label_values(broker::telemetry::const_label_list labels) + { + auto get_value = [](const auto& label) + { + return label.second; + }; + std::vector v; + std::transform(labels.begin(), labels.end(), std::back_inserter(v), get_value); + return v; + } + +// Convert an int64_t or double to a DoubleValPtr. int64_t is casted. +template DoubleValPtr as_double_val(T val) + { + if constexpr ( std::is_same_v ) + { + return zeek::make_intrusive(static_cast(val)); + } + else + { + static_assert(std::is_same_v); + return zeek::make_intrusive(val); + } + }; + } namespace zeek::telemetry @@ -38,6 +69,432 @@ void Manager::InitPostBrokerSetup(broker::endpoint& ep) pimpl.swap(ptr); } +// -- collect metric stuff ----------------------------------------------------- + +template +zeek::RecordValPtr Manager::GetMetricOptsRecord(Manager::MetricType metric_type, + const broker::telemetry::metric_family_hdl* family) + { + static auto string_vec_type = zeek::id::find_type("string_vec"); + static auto metric_opts_type = zeek::id::find_type("Telemetry::MetricOpts"); + + static auto prefix_idx = metric_opts_type->FieldOffset("prefix"); + static auto name_idx = metric_opts_type->FieldOffset("name"); + static auto help_text_idx = metric_opts_type->FieldOffset("help_text"); + static auto unit_idx = metric_opts_type->FieldOffset("unit"); + static auto is_total_idx = metric_opts_type->FieldOffset("is_total"); + static auto labels_idx = metric_opts_type->FieldOffset("labels"); + static auto bounds_idx = metric_opts_type->FieldOffset("bounds"); + static auto metric_type_idx = metric_opts_type->FieldOffset("metric_type"); + + if ( const auto& it = metric_opts_cache.find(family); it != metric_opts_cache.end() ) + return it->second; + + auto r = make_intrusive(metric_opts_type); + r->Assign(prefix_idx, make_intrusive(broker::telemetry::prefix(family))); + r->Assign(name_idx, make_intrusive(broker::telemetry::name(family))); + r->Assign(help_text_idx, make_intrusive(broker::telemetry::helptext(family))); + r->Assign(unit_idx, make_intrusive(broker::telemetry::unit(family))); + r->Assign(is_total_idx, val_mgr->Bool(broker::telemetry::is_sum(family))); + + auto label_names_vec = make_intrusive(string_vec_type); + for ( const auto& l : broker::telemetry::label_names(family) ) + label_names_vec->Append(make_intrusive(l)); + + r->Assign(labels_idx, label_names_vec); + + // This is mapping Manager.h enums to bif values depending on + // the template type and whether this is a counter, gauge or + // histogram. + zeek_int_t metric_type_int = -1; + if constexpr ( std::is_same::value ) + { + switch ( metric_type ) + { + case MetricType::Counter: + metric_type_int = BifEnum::Telemetry::MetricType::DOUBLE_COUNTER; + break; + case MetricType::Gauge: + metric_type_int = BifEnum::Telemetry::MetricType::DOUBLE_GAUGE; + break; + case MetricType::Histogram: + metric_type_int = BifEnum::Telemetry::MetricType::DOUBLE_HISTOGRAM; + break; + } + } + else + { + switch ( metric_type ) + { + case MetricType::Counter: + metric_type_int = BifEnum::Telemetry::MetricType::INT_COUNTER; + break; + case MetricType::Gauge: + metric_type_int = BifEnum::Telemetry::MetricType::INT_GAUGE; + break; + case MetricType::Histogram: + metric_type_int = BifEnum::Telemetry::MetricType::INT_HISTOGRAM; + break; + } + } + + if ( metric_type_int < 0 ) + reporter->FatalError("Unable to lookup metric type %d", int(metric_type)); + + r->Assign(metric_type_idx, + zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(metric_type_int)); + + metric_opts_cache.insert({family, r}); + + return r; + } + +zeek::RecordValPtr Manager::CollectedValueMetric::AsMetricRecord() const + { + static auto string_vec_type = zeek::id::find_type("string_vec"); + static auto metric_record_type = zeek::id::find_type("Telemetry::Metric"); + static auto opts_idx = metric_record_type->FieldOffset("opts"); + static auto labels_idx = metric_record_type->FieldOffset("labels"); + static auto value_idx = metric_record_type->FieldOffset("value"); + static auto count_value_idx = metric_record_type->FieldOffset("count_value"); + + auto r = make_intrusive(metric_record_type); + + auto label_values_vec = make_intrusive(string_vec_type); + for ( const auto& l : label_values ) + label_values_vec->Append(make_intrusive(l)); + + r->Assign(labels_idx, label_values_vec); + + auto fn = [&](auto val) + { + using val_t = decltype(val); + auto opts_record = telemetry_mgr->GetMetricOptsRecord(metric_type, family); + r->Assign(opts_idx, opts_record); + r->Assign(value_idx, as_double_val(val)); + if constexpr ( std::is_same_v ) + r->Assign(count_value_idx, val_mgr->Count(val)); + }; + + std::visit(fn, value); + + return r; + } + +zeek::RecordValPtr Manager::CollectedHistogramMetric::AsHistogramMetricRecord() const + { + static auto string_vec_type = zeek::id::find_type("string_vec"); + static auto double_vec_type = zeek::id::find_type("double_vec"); + static auto count_vec_type = zeek::id::find_type("index_vec"); + static auto histogram_metric_type = zeek::id::find_type( + "Telemetry::HistogramMetric"); + static auto opts_idx = histogram_metric_type->FieldOffset("opts"); + static auto labels_idx = histogram_metric_type->FieldOffset("labels"); + static auto values_idx = histogram_metric_type->FieldOffset("values"); + static auto count_values_idx = histogram_metric_type->FieldOffset("count_values"); + static auto observations_idx = histogram_metric_type->FieldOffset("observations"); + static auto sum_idx = histogram_metric_type->FieldOffset("sum"); + static auto count_observations_idx = histogram_metric_type->FieldOffset("count_observations"); + static auto count_sum_idx = histogram_metric_type->FieldOffset("count_sum"); + + zeek::RecordValPtr opts_record; + + auto r = make_intrusive(histogram_metric_type); + + auto label_values_vec = make_intrusive(string_vec_type); + for ( const auto& l : label_values ) + label_values_vec->Append(make_intrusive(l)); + + r->Assign(labels_idx, label_values_vec); + + auto fn = [&](const auto& histogram_data) + { + using val_t = std::decay_t; + opts_record = telemetry_mgr->GetMetricOptsRecord(MetricType::Histogram, family); + r->Assign(opts_idx, opts_record); + + val_t observations = 0; + auto values_vec = make_intrusive(double_vec_type); + auto count_values_vec = make_intrusive(count_vec_type); + + for ( const auto& b : histogram_data.buckets ) + { + observations += b.count; + values_vec->Append(as_double_val(b.count)); + if constexpr ( std::is_same_v ) + count_values_vec->Append(val_mgr->Count(b.count)); + } + + r->Assign(values_idx, values_vec); + r->Assign(sum_idx, as_double_val(histogram_data.sum)); + r->Assign(observations_idx, as_double_val(observations)); + + // Add extra fields just for int64_t based histograms with type count + if constexpr ( std::is_same_v ) + { + r->Assign(count_values_idx, count_values_vec); + r->Assign(count_sum_idx, val_mgr->Count(histogram_data.sum)); + r->Assign(count_observations_idx, val_mgr->Count(observations)); + } + }; + + std::visit(fn, histogram); + + // This is a bit annoying. The bounds / upper bounds of the individual + // buckets are not accessible through the family hdl. We set the + // opts$bounds field based on bounds collected from an individual + // metric instance. + // + // As the opts_record is cached by the manager this should only happen + // once and not over and over again. + // + // This is probably working around something that should be fixed + // on the broker::telemetry level. + static auto opts_rt = zeek::id::find_type("Telemetry::MetricOpts"); + static auto opts_rt_off_bounds = opts_rt->FieldOffset("bounds"); + static auto opts_rt_off_count_bounds = opts_rt->FieldOffset("count_bounds"); + static auto bounds_default_val = opts_rt->FieldDefault(opts_rt_off_bounds); + + auto bounds_field_val = opts_record->GetField(opts_rt_off_bounds); + if ( ! bounds_field_val || bounds_field_val == bounds_default_val ) + { + auto fn = [&opts_record](const auto& histogram_data) + { + auto bounds_vec = make_intrusive(double_vec_type); + for ( const auto& b : histogram_data.buckets ) + bounds_vec->Append(as_double_val(b.upper_bound)); + + opts_record->Assign(opts_rt_off_bounds, bounds_vec); + + // If this is an int64_t histogram, also fill count_bounds + using HistogramDataType = std::decay_t; + if constexpr ( std::is_same_v ) + { + auto count_bounds_vec = make_intrusive(count_vec_type); + for ( const auto& b : histogram_data.buckets ) + count_bounds_vec->Append(val_mgr->Count(b.upper_bound)); + + opts_record->Assign(opts_rt_off_count_bounds, count_bounds_vec); + } + }; + + std::visit(fn, histogram); + } + + return r; + } + +/** + * Encapsulate matching of prefix and name against a broker::telemetry::metric_family_hdl + */ +class MetricFamilyMatcher + { +public: + MetricFamilyMatcher(std::string_view prefix, std::string_view name) + : prefix_pattern(prefix), name_pattern(name) + { + } + + /** + * @return true if the given family's prefix and name match, else false; + */ + bool operator()(const broker::telemetry::metric_family_hdl* family) + { + auto prefix = std::string{broker::telemetry::prefix(family)}; + auto name = std::string{broker::telemetry::name(family)}; + + return fnmatch(prefix_pattern.c_str(), prefix.c_str(), 0) != FNM_NOMATCH && + fnmatch(name_pattern.c_str(), name.c_str(), 0) != FNM_NOMATCH; + } + +private: + std::string prefix_pattern; + std::string name_pattern; + }; + +/** + * A collector implementation for counters and gauges. + */ +class MetricsCollector : public broker::telemetry::metrics_collector + { + using MetricType = Manager::MetricType; + +public: + MetricsCollector(std::string_view prefix, std::string_view name) : matches(prefix, name) { } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::dbl_counter_hdl* counter, + broker::telemetry::const_label_list labels) + { + if ( matches(family) ) + metrics.emplace_back(MetricType::Counter, family, extract_label_values(labels), + broker::telemetry::value(counter)); + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::int_counter_hdl* counter, + broker::telemetry::const_label_list labels) + { + if ( matches(family) ) + metrics.emplace_back(MetricType::Counter, family, extract_label_values(labels), + broker::telemetry::value(counter)); + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::dbl_gauge_hdl* gauge, + broker::telemetry::const_label_list labels) + { + if ( matches(family) ) + metrics.emplace_back(MetricType::Gauge, family, extract_label_values(labels), + broker::telemetry::value(gauge)); + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::int_gauge_hdl* gauge, + broker::telemetry::const_label_list labels) + { + if ( matches(family) ) + metrics.emplace_back(MetricType::Gauge, family, extract_label_values(labels), + broker::telemetry::value(gauge)); + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::dbl_histogram_hdl* histogram, + broker::telemetry::const_label_list labels) + { + // Ignored + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::int_histogram_hdl* histogram, + broker::telemetry::const_label_list labels) + { + // Ignored + } + + std::vector& GetResult() { return metrics; } + +private: + MetricFamilyMatcher matches; + std::vector metrics; + }; + +std::vector Manager::CollectMetrics(std::string_view prefix, + std::string_view name) + { + auto collector = MetricsCollector(prefix, name); + + pimpl->collect(collector); + + return std::move(collector.GetResult()); + } + +/** + * A collector implementation for histograms. + */ +class HistogramMetricsCollector : public broker::telemetry::metrics_collector + { + using MetricType = Manager::MetricType; + +public: + HistogramMetricsCollector(std::string_view prefix, std::string_view name) + : matches(prefix, name) + { + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::dbl_counter_hdl* counter, + broker::telemetry::const_label_list labels) + { + // Ignored + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::int_counter_hdl* counter, + broker::telemetry::const_label_list labels) + { + // Ignored + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::dbl_gauge_hdl* gauge, + broker::telemetry::const_label_list labels) + { + // Ignored + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::int_gauge_hdl* gauge, + broker::telemetry::const_label_list labels) + { + // Ignored + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::dbl_histogram_hdl* histogram, + broker::telemetry::const_label_list labels) + { + if ( ! matches(family) ) + return; + + size_t num_buckets = broker::telemetry::num_buckets(histogram); + + Manager::CollectedHistogramMetric::DblHistogramData histogram_data; + histogram_data.buckets.reserve(num_buckets); + + for ( size_t i = 0; i < num_buckets; i++ ) + { + double c = broker::telemetry::count_at(histogram, i); + double ub = broker::telemetry::upper_bound_at(histogram, i); + histogram_data.buckets.emplace_back(c, ub); + } + + histogram_data.sum = broker::telemetry::sum(histogram); + + metrics.emplace_back(family, extract_label_values(labels), std::move(histogram_data)); + } + + void operator()(const broker::telemetry::metric_family_hdl* family, + const broker::telemetry::int_histogram_hdl* histogram, + broker::telemetry::const_label_list labels) + { + if ( ! matches(family) ) + return; + + size_t num_buckets = broker::telemetry::num_buckets(histogram); + + Manager::CollectedHistogramMetric::IntHistogramData histogram_data; + histogram_data.buckets.reserve(num_buckets); + + for ( size_t i = 0; i < num_buckets; i++ ) + { + int64_t c = broker::telemetry::count_at(histogram, i); + int64_t ub = broker::telemetry::upper_bound_at(histogram, i); + histogram_data.buckets.emplace_back(c, ub); + } + + histogram_data.sum = broker::telemetry::sum(histogram); + + metrics.emplace_back(family, extract_label_values(labels), std::move(histogram_data)); + } + + std::vector& GetResult() { return metrics; } + +private: + MetricFamilyMatcher matches; + std::vector metrics; + }; + +std::vector +Manager::CollectHistogramMetrics(std::string_view prefix, std::string_view name) + { + auto collector = HistogramMetricsCollector(prefix, name); + + pimpl->collect(collector); + + return std::move(collector.GetResult()); + } + } // namespace zeek::telemetry // -- unit tests --------------------------------------------------------------- diff --git a/src/telemetry/Manager.h b/src/telemetry/Manager.h index b260838778..a0d4d763ba 100644 --- a/src/telemetry/Manager.h +++ b/src/telemetry/Manager.h @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include "zeek/IntrusivePtr.h" @@ -20,6 +22,12 @@ namespace broker class endpoint; } +namespace zeek + { +class RecordVal; +using RecordValPtr = IntrusivePtr; + } + namespace zeek::Broker { class Manager; @@ -50,6 +58,128 @@ public: */ virtual void InitPostScript(); + /** + * Supported metric types. + */ + enum class MetricType + { + Counter, + Gauge, + Histogram + }; + + /** + * Captures information about counter and gauge metrics. + */ + struct CollectedValueMetric + { + /** + * Constructor. + * @param metric_type The type of this metric. + * @param family Broker layer family handle for this metric. + * @param label_values The string values for each of the metric's labels. + * @param value The metric's current value. + */ + CollectedValueMetric(MetricType metric_type, + const broker::telemetry::metric_family_hdl* family, + std::vector label_values, + std::variant value) + : metric_type(metric_type), family(family), label_values(std::move(label_values)), + value(value) + { + } + + /** + * @return A script layer Telemetry::Metric record for this metric. + */ + zeek::RecordValPtr AsMetricRecord() const; + + enum MetricType metric_type; + const broker::telemetry::metric_family_hdl* family; + std::vector label_values; + std::variant value; + }; + + /** + * Captures information about histogram metrics. + */ + struct CollectedHistogramMetric + { + /** + * Helper struct representing a single bucket of a histogram. + * @tparam T The data type used by the histogram (double or int64_t). + */ + template struct Bucket + { + Bucket(T count, T upper_bound) : count(count), upper_bound(upper_bound) { } + + T count; + T upper_bound; + }; + + /** + * Helper struct representing a histogram as sum and buckets. + * @tparam T The data type used by the histogram (double or int64_t). + */ + template struct HistogramData + { + T sum; + std::vector> buckets; + }; + + using DblHistogramData = HistogramData; + using IntHistogramData = HistogramData; + + /** + * Constructor. + * @param family Broker layer family handle for this metric. + * @param label_values The string values for each of the metric's labels. + * @param histogram The histogram's data (sum and individual buckets). + */ + CollectedHistogramMetric(const broker::telemetry::metric_family_hdl* family, + std::vector label_values, + std::variant histogram) + + : family(family), label_values(std::move(label_values)), histogram(std::move(histogram)) + { + } + + const broker::telemetry::metric_family_hdl* family; + std::vector label_values; + std::variant histogram; + + /** + * @return A script layer Telemetry::HistogramMetric record for this histogram. + */ + zeek::RecordValPtr AsHistogramMetricRecord() const; + }; + + /** + * @return A script layer Telemetry::MetricOpts record for the given metric family. + * @param metric_typ The type of metric. + * @param family Broker layer family handle for the family. + * @tparam T The underlying data type (double or int64_t) + */ + template + zeek::RecordValPtr GetMetricOptsRecord(MetricType metric_type, + const broker::telemetry::metric_family_hdl* family); + + /** + * @return All counter and gauge metrics and their values matching prefix and name. + * @param prefix The prefix pattern to use for filtering. Supports globbing. + * @param name The name pattern to use for filtering. Supports globbing. + */ + std::vector CollectMetrics(std::string_view prefix, + std::string_view name); + + /** + * @return All histogram metrics and their data matching prefix and name. + * @param prefix The prefix pattern to use for filtering. Supports globbing. + * @param name The name pattern to use for filtering. Supports globbing. + */ + std::vector CollectHistogramMetrics(std::string_view prefix, + std::string_view name); + /** * @return A counter metric family. Creates the family lazily if necessary. * @param prefix The prefix (namespace) this family belongs to. @@ -414,6 +544,11 @@ protected: void InitPostBrokerSetup(broker::endpoint&); IntrusivePtr pimpl; + +private: + // Caching of metric_family_hdl instances to their Zeek record representation. + std::unordered_map + metric_opts_cache; }; } // namespace zeek::telemetry diff --git a/src/telemetry/telemetry.bif b/src/telemetry/telemetry.bif index 5aa23abae0..a56191e93c 100644 --- a/src/telemetry/telemetry.bif +++ b/src/telemetry/telemetry.bif @@ -3,6 +3,15 @@ module Telemetry; +enum MetricType %{ + DOUBLE_COUNTER, + INT_COUNTER, + DOUBLE_GAUGE, + INT_GAUGE, + DOUBLE_HISTOGRAM, + INT_HISTOGRAM, +%} + %%{ #include "zeek/telemetry/Counter.h" #include "zeek/telemetry/Gauge.h" @@ -548,3 +557,30 @@ function Telemetry::__dbl_histogram_sum%(val: opaque of dbl_histogram_metric%): return zeek::make_intrusive(0.0); } %} + + +function Telemetry::__collect_metrics%(prefix: string, name: string%): any_vec + %{ + auto metrics = telemetry_mgr->CollectMetrics(sv(prefix), sv(name)); + + static auto metrics_vector_type = zeek::id::find_type("any_vec"); + auto vec = zeek::make_intrusive(metrics_vector_type); + + for ( const auto& m : metrics ) + vec->Append(m.AsMetricRecord()); + + return vec; + %} + +function Telemetry::__collect_histogram_metrics%(prefix: string, name: string%): any_vec + %{ + auto metrics = telemetry_mgr->CollectHistogramMetrics(sv(prefix), sv(name)); + + static auto metrics_vector_type = zeek::id::find_type("any_vec"); + auto vec = zeek::make_intrusive(metrics_vector_type); + + for ( const auto& m : metrics ) + vec->Append(m.AsHistogramMetricRecord()); + + return vec; + %} diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index ecf792fbc6..307c31c4ac 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -336,6 +336,9 @@ scripts/base/init-default.zeek scripts/base/frameworks/netcontrol/drop.zeek scripts/base/frameworks/netcontrol/shunt.zeek scripts/base/frameworks/netcontrol/non-cluster.zeek + scripts/base/frameworks/telemetry/__load__.zeek + scripts/base/frameworks/telemetry/main.zeek + scripts/base/misc/version.zeek scripts/base/protocols/conn/__load__.zeek scripts/base/protocols/conn/main.zeek scripts/base/protocols/conn/contents.zeek @@ -445,7 +448,6 @@ scripts/base/init-default.zeek scripts/base/misc/find-checksum-offloading.zeek scripts/base/misc/find-filtered-trace.zeek build/scripts/base/misc/installation.zeek - scripts/base/misc/version.zeek build/scripts/builtin-plugins/__preload__.zeek build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek build/scripts/builtin-plugins/__load__.zeek diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out index aa85391f6e..b2cb315069 100644 --- a/testing/btest/Baseline/coverage.find-bro-logs/out +++ b/testing/btest/Baseline/coverage.find-bro-logs/out @@ -57,6 +57,8 @@ ssh ssl stats syslog +telemetry +telemetry_histogram traceroute tunnel unified2 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index fb698f57cf..669e025f64 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -547,6 +547,7 @@ 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Site::neighbor_zones, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Site::private_address_space, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Software::asset_tracking, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Telemetry::sync_interval, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Weird::ignore_hosts, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Weird::sampling_duration, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Weird::sampling_duration, Config::weird_option_change_interval{ if (Weird::sampling_duration == Config::ID) { Reporter::set_weird_sampling_duration(Config::new_value)}return (Config::new_value)}, 5)) -> @@ -675,10 +676,26 @@ 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(Supervisor::__is_supervisor, , ()) -> 0.000000 MetaHookPost CallFunction(Supervisor::is_supervisor, , ()) -> +0.000000 MetaHookPost CallFunction(Telemetry::__dbl_counter_family, , (zeek, telemetry_counter_usage_error, [], This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero., 1, T)) -> +0.000000 MetaHookPost CallFunction(Telemetry::__dbl_gauge_family, , (zeek, telemetry_gauge_usage_error, [], This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero., 1, F)) -> +0.000000 MetaHookPost CallFunction(Telemetry::__dbl_gauge_family, , (zeek, version_info, [version_number, major, minor, patch, commit, beta, debug, version_string], The Zeek version, 1, F)) -> +0.000000 MetaHookPost CallFunction(Telemetry::__dbl_gauge_inc, , (, 1.0)) -> +0.000000 MetaHookPost CallFunction(Telemetry::__dbl_gauge_metric_get_or_add, ..., ...) -> +0.000000 MetaHookPost CallFunction(Telemetry::__dbl_gauge_value, , ()) -> +0.000000 MetaHookPost CallFunction(Telemetry::__dbl_histogram_family, , (zeek, telemetry_histogram_usage_error, [], [1.0], This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero., 1, F)) -> +0.000000 MetaHookPost CallFunction(Telemetry::gauge_family_set, ..., ...) -> +0.000000 MetaHookPost CallFunction(Telemetry::gauge_set, , ([__metric=], 1.0)) -> +0.000000 MetaHookPost CallFunction(Telemetry::gauge_with, ..., ...) -> +0.000000 MetaHookPost CallFunction(Telemetry::make_labels, ..., ...) -> +0.000000 MetaHookPost CallFunction(Telemetry::register_counter_family, , ([prefix=zeek, name=telemetry_counter_usage_error, unit=1, help_text=This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=, count_bounds=, metric_type=])) -> +0.000000 MetaHookPost CallFunction(Telemetry::register_gauge_family, , ([prefix=zeek, name=telemetry_gauge_usage_error, unit=1, help_text=This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=, count_bounds=, metric_type=])) -> +0.000000 MetaHookPost CallFunction(Telemetry::register_gauge_family, , ([prefix=zeek, name=version_info, unit=1, help_text=The Zeek version, labels=[version_number, major, minor, patch, commit, beta, debug, version_string], is_total=, bounds=, count_bounds=, metric_type=])) -> +0.000000 MetaHookPost CallFunction(Telemetry::register_histogram_family, , ([prefix=zeek, name=telemetry_histogram_usage_error, unit=1, help_text=This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=[1.0], count_bounds=, metric_type=])) -> 0.000000 MetaHookPost CallFunction(Version::parse, ..., ...) -> 0.000000 MetaHookPost CallFunction(__init_primary_bifs, , ()) -> 0.000000 MetaHookPost CallFunction(__init_secondary_bifs, , ()) -> 0.000000 MetaHookPost CallFunction(bare_mode, , ()) -> +0.000000 MetaHookPost CallFunction(cat, ..., ...) -> 0.000000 MetaHookPost CallFunction(current_time, , ()) -> 0.000000 MetaHookPost CallFunction(filter_change_tracking, , ()) -> 0.000000 MetaHookPost CallFunction(getenv, , (CLUSTER_NODE)) -> @@ -1059,6 +1076,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/syslog, <...>/syslog) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/tcp, <...>/tcp) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/telemetry, <...>/telemetry) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/teredo, <...>/teredo) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/thresholds, <...>/thresholds.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/time, <...>/time.zeek) -> -1 @@ -1444,6 +1462,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/syslog, <...>/syslog) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/tcp, <...>/tcp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/telemetry, <...>/telemetry) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/teredo, <...>/teredo) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/thresholds, <...>/thresholds.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/time, <...>/time.zeek) -> (-1, ) @@ -2029,6 +2048,7 @@ 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Site::neighbor_zones, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Site::private_address_space, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Software::asset_tracking, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Telemetry::sync_interval, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Weird::ignore_hosts, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Weird::sampling_duration, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Weird::sampling_duration, Config::weird_option_change_interval{ if (Weird::sampling_duration == Config::ID) { Reporter::set_weird_sampling_duration(Config::new_value)}return (Config::new_value)}, 5)) @@ -2157,10 +2177,26 @@ 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugins, , ()) 0.000000 MetaHookPre CallFunction(Supervisor::__is_supervisor, , ()) 0.000000 MetaHookPre CallFunction(Supervisor::is_supervisor, , ()) +0.000000 MetaHookPre CallFunction(Telemetry::__dbl_counter_family, , (zeek, telemetry_counter_usage_error, [], This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero., 1, T)) +0.000000 MetaHookPre CallFunction(Telemetry::__dbl_gauge_family, , (zeek, telemetry_gauge_usage_error, [], This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero., 1, F)) +0.000000 MetaHookPre CallFunction(Telemetry::__dbl_gauge_family, , (zeek, version_info, [version_number, major, minor, patch, commit, beta, debug, version_string], The Zeek version, 1, F)) +0.000000 MetaHookPre CallFunction(Telemetry::__dbl_gauge_inc, , (, 1.0)) +0.000000 MetaHookPre CallFunction(Telemetry::__dbl_gauge_metric_get_or_add, ..., ...) +0.000000 MetaHookPre CallFunction(Telemetry::__dbl_gauge_value, , ()) +0.000000 MetaHookPre CallFunction(Telemetry::__dbl_histogram_family, , (zeek, telemetry_histogram_usage_error, [], [1.0], This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero., 1, F)) +0.000000 MetaHookPre CallFunction(Telemetry::gauge_family_set, ..., ...) +0.000000 MetaHookPre CallFunction(Telemetry::gauge_set, , ([__metric=], 1.0)) +0.000000 MetaHookPre CallFunction(Telemetry::gauge_with, ..., ...) +0.000000 MetaHookPre CallFunction(Telemetry::make_labels, ..., ...) +0.000000 MetaHookPre CallFunction(Telemetry::register_counter_family, , ([prefix=zeek, name=telemetry_counter_usage_error, unit=1, help_text=This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=, count_bounds=, metric_type=])) +0.000000 MetaHookPre CallFunction(Telemetry::register_gauge_family, , ([prefix=zeek, name=telemetry_gauge_usage_error, unit=1, help_text=This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=, count_bounds=, metric_type=])) +0.000000 MetaHookPre CallFunction(Telemetry::register_gauge_family, , ([prefix=zeek, name=version_info, unit=1, help_text=The Zeek version, labels=[version_number, major, minor, patch, commit, beta, debug, version_string], is_total=, bounds=, count_bounds=, metric_type=])) +0.000000 MetaHookPre CallFunction(Telemetry::register_histogram_family, , ([prefix=zeek, name=telemetry_histogram_usage_error, unit=1, help_text=This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=[1.0], count_bounds=, metric_type=])) 0.000000 MetaHookPre CallFunction(Version::parse, ..., ...) 0.000000 MetaHookPre CallFunction(__init_primary_bifs, , ()) 0.000000 MetaHookPre CallFunction(__init_secondary_bifs, , ()) 0.000000 MetaHookPre CallFunction(bare_mode, , ()) +0.000000 MetaHookPre CallFunction(cat, ..., ...) 0.000000 MetaHookPre CallFunction(current_time, , ()) 0.000000 MetaHookPre CallFunction(filter_change_tracking, , ()) 0.000000 MetaHookPre CallFunction(getenv, , (CLUSTER_NODE)) @@ -2541,6 +2577,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/syslog, <...>/syslog) 0.000000 MetaHookPre LoadFile(0, base<...>/tcp, <...>/tcp) +0.000000 MetaHookPre LoadFile(0, base<...>/telemetry, <...>/telemetry) 0.000000 MetaHookPre LoadFile(0, base<...>/teredo, <...>/teredo) 0.000000 MetaHookPre LoadFile(0, base<...>/thresholds, <...>/thresholds.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/time, <...>/time.zeek) @@ -2926,6 +2963,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/syslog, <...>/syslog) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/tcp, <...>/tcp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/telemetry, <...>/telemetry) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/teredo, <...>/teredo) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/thresholds, <...>/thresholds.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/time, <...>/time.zeek) @@ -3510,6 +3548,7 @@ 0.000000 | HookCallFunction Option::set_change_handler(Site::neighbor_zones, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(Site::private_address_space, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(Software::asset_tracking, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(Telemetry::sync_interval, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(Weird::ignore_hosts, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_duration, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_duration, Config::weird_option_change_interval{ if (Weird::sampling_duration == Config::ID) { Reporter::set_weird_sampling_duration(Config::new_value)}return (Config::new_value)}, 5) @@ -3638,10 +3677,26 @@ 0.000000 | HookCallFunction SumStats::register_observe_plugins() 0.000000 | HookCallFunction Supervisor::__is_supervisor() 0.000000 | HookCallFunction Supervisor::is_supervisor() +0.000000 | HookCallFunction Telemetry::__dbl_counter_family(zeek, telemetry_counter_usage_error, [], This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero., 1, T) +0.000000 | HookCallFunction Telemetry::__dbl_gauge_family(zeek, telemetry_gauge_usage_error, [], This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero., 1, F) +0.000000 | HookCallFunction Telemetry::__dbl_gauge_family(zeek, version_info, [version_number, major, minor, patch, commit, beta, debug, version_string], The Zeek version, 1, F) +0.000000 | HookCallFunction Telemetry::__dbl_gauge_inc(, 1.0) +0.000000 | HookCallFunction Telemetry::__dbl_gauge_metric_get_or_add(...) +0.000000 | HookCallFunction Telemetry::__dbl_gauge_value() +0.000000 | HookCallFunction Telemetry::__dbl_histogram_family(zeek, telemetry_histogram_usage_error, [], [1.0], This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero., 1, F) +0.000000 | HookCallFunction Telemetry::gauge_family_set(...) +0.000000 | HookCallFunction Telemetry::gauge_set([__metric=], 1.0) +0.000000 | HookCallFunction Telemetry::gauge_with(...) +0.000000 | HookCallFunction Telemetry::make_labels(...) +0.000000 | HookCallFunction Telemetry::register_counter_family([prefix=zeek, name=telemetry_counter_usage_error, unit=1, help_text=This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=, count_bounds=, metric_type=]) +0.000000 | HookCallFunction Telemetry::register_gauge_family([prefix=zeek, name=telemetry_gauge_usage_error, unit=1, help_text=This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=, count_bounds=, metric_type=]) +0.000000 | HookCallFunction Telemetry::register_gauge_family([prefix=zeek, name=version_info, unit=1, help_text=The Zeek version, labels=[version_number, major, minor, patch, commit, beta, debug, version_string], is_total=, bounds=, count_bounds=, metric_type=]) +0.000000 | HookCallFunction Telemetry::register_histogram_family([prefix=zeek, name=telemetry_histogram_usage_error, unit=1, help_text=This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero., labels=[], is_total=, bounds=[1.0], count_bounds=, metric_type=]) 0.000000 | HookCallFunction Version::parse(...) 0.000000 | HookCallFunction __init_primary_bifs() 0.000000 | HookCallFunction __init_secondary_bifs() 0.000000 | HookCallFunction bare_mode() +0.000000 | HookCallFunction cat(...) 0.000000 | HookCallFunction current_time() 0.000000 | HookCallFunction filter_change_tracking() 0.000000 | HookCallFunction getenv(CLUSTER_NODE) @@ -4034,6 +4089,7 @@ 0.000000 | HookLoadFile base<...>/supervisor.bif <...>/supervisor.bif.zeek 0.000000 | HookLoadFile base<...>/syslog <...>/syslog 0.000000 | HookLoadFile base<...>/tcp <...>/tcp +0.000000 | HookLoadFile base<...>/telemetry <...>/telemetry 0.000000 | HookLoadFile base<...>/teredo <...>/teredo 0.000000 | HookLoadFile base<...>/thresholds <...>/thresholds.zeek 0.000000 | HookLoadFile base<...>/time <...>/time.zeek @@ -4419,6 +4475,7 @@ 0.000000 | HookLoadFileExtended base<...>/supervisor.bif <...>/supervisor.bif.zeek 0.000000 | HookLoadFileExtended base<...>/syslog <...>/syslog 0.000000 | HookLoadFileExtended base<...>/tcp <...>/tcp +0.000000 | HookLoadFileExtended base<...>/telemetry <...>/telemetry 0.000000 | HookLoadFileExtended base<...>/teredo <...>/teredo 0.000000 | HookLoadFileExtended base<...>/thresholds <...>/thresholds.zeek 0.000000 | HookLoadFileExtended base<...>/time <...>/time.zeek @@ -4449,15 +4506,18 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Broker::__flush_logs, , () XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Broker::flush_logs, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Broker::log_flush, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(ChecksumOffloading::check, , ()) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Telemetry::sync, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(filter_change_tracking, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(get_net_stats, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={}, history=, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(run_sync_hook, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost DrainEvents() -> XXXXXXXXXX.XXXXXX MetaHookPost ObjDtor() -> XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(Broker::log_flush()) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(ChecksumOffloading::check()) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(filter_change_tracking()) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(new_connection([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={}, history=, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> false +XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(run_sync_hook()) -> false XXXXXXXXXX.XXXXXX MetaHookPost SetupAnalyzerTree(XXXXXXXXXX.XXXXXX(XXXXXXXXXX.XXXXXX) TCP 141.142.228.5:59856 -> 192.150.187.43:80) -> XXXXXXXXXX.XXXXXX MetaHookPost UpdateNetworkTime(XXXXXXXXXX.XXXXXX) -> XXXXXXXXXX.XXXXXX MetaHookPre BroObjDtor() @@ -4465,15 +4525,18 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::__flush_logs, , () XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::flush_logs, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::log_flush, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(ChecksumOffloading::check, , ()) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Telemetry::sync, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(filter_change_tracking, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(get_net_stats, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={}, history=, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(run_sync_hook, , ()) XXXXXXXXXX.XXXXXX MetaHookPre DrainEvents() XXXXXXXXXX.XXXXXX MetaHookPre ObjDtor() XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(Broker::log_flush()) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(ChecksumOffloading::check()) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(filter_change_tracking()) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(new_connection([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={}, history=, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) +XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(run_sync_hook()) XXXXXXXXXX.XXXXXX MetaHookPre SetupAnalyzerTree(XXXXXXXXXX.XXXXXX(XXXXXXXXXX.XXXXXX) TCP 141.142.228.5:59856 -> 192.150.187.43:80) XXXXXXXXXX.XXXXXX MetaHookPre UpdateNetworkTime(XXXXXXXXXX.XXXXXX) XXXXXXXXXX.XXXXXX | HookBroObjDtor @@ -4483,14 +4546,17 @@ XXXXXXXXXX.XXXXXX | HookCallFunction Broker::__flush_logs() XXXXXXXXXX.XXXXXX | HookCallFunction Broker::flush_logs() XXXXXXXXXX.XXXXXX | HookCallFunction Broker::log_flush() XXXXXXXXXX.XXXXXX | HookCallFunction ChecksumOffloading::check() +XXXXXXXXXX.XXXXXX | HookCallFunction Telemetry::sync() XXXXXXXXXX.XXXXXX | HookCallFunction filter_change_tracking() XXXXXXXXXX.XXXXXX | HookCallFunction get_net_stats() XXXXXXXXXX.XXXXXX | HookCallFunction new_connection([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={}, history=, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]) +XXXXXXXXXX.XXXXXX | HookCallFunction run_sync_hook() XXXXXXXXXX.XXXXXX | HookDrainEvents XXXXXXXXXX.XXXXXX | HookQueueEvent Broker::log_flush() XXXXXXXXXX.XXXXXX | HookQueueEvent ChecksumOffloading::check() XXXXXXXXXX.XXXXXX | HookQueueEvent filter_change_tracking() XXXXXXXXXX.XXXXXX | HookQueueEvent new_connection([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={}, history=, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]) +XXXXXXXXXX.XXXXXX | HookQueueEvent run_sync_hook() XXXXXXXXXX.XXXXXX | HookSetupAnalyzerTree XXXXXXXXXX.XXXXXX(XXXXXXXXXX.XXXXXX) TCP 141.142.228.5:59856 -> 192.150.187.43:80 XXXXXXXXXX.XXXXXX | RequestObjDtor Broker::log_flush() XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(connection_established, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=0, state=4, num_pkts=1, num_bytes_ip=64, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=69.0 msecs 740.056992 usecs, service={}, history=Sh, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> @@ -4523,7 +4589,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(HTTP::set_state, , ([id=[o XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(analyzer_confirmation, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) -> -XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, T, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, ..., ...) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(fmt, , (-%s, HTTP)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> @@ -4562,7 +4628,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(HTTP::set_state, , ([id=[o XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(analyzer_confirmation, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3)) -XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, T, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, ..., ...) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(fmt, , (-%s, HTTP)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) @@ -4602,7 +4668,7 @@ XXXXXXXXXX.XXXXXX | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, XXXXXXXXXX.XXXXXX | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) XXXXXXXXXX.XXXXXX | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) XXXXXXXXXX.XXXXXX | HookCallFunction analyzer_confirmation([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], Analyzer::ANALYZER_HTTP, 3) -XXXXXXXXXX.XXXXXX | HookCallFunction cat(Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, T, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) +XXXXXXXXXX.XXXXXX | HookCallFunction cat(...) XXXXXXXXXX.XXXXXX | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp) XXXXXXXXXX.XXXXXX | HookCallFunction fmt(-%s, HTTP) XXXXXXXXXX.XXXXXX | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=0, state=4, num_pkts=1, num_bytes_ip=60, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=70.0 msecs 183.038712 usecs, service={HTTP}, history=ShAD, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) @@ -4645,7 +4711,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(HTTP::get_file_handle, , ( XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> -XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, ..., ...) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(file_new, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=, ftp=, http=, irc=, pe=])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(file_over_new_connection, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={}, rx_hosts={}, conn_uids={}, source=HTTP, depth=0, analyzers={}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=, irc=, pe=], [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) -> @@ -4690,7 +4756,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(HTTP::get_file_handle, , ( XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(HTTP::set_state, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, ..., ...) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(file_new, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=, ftp=, http=, irc=, pe=])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(file_over_new_connection, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={}, rx_hosts={}, conn_uids={}, source=HTTP, depth=0, analyzers={}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=, irc=, pe=], [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) @@ -4736,7 +4802,7 @@ XXXXXXXXXX.XXXXXX | HookCallFunction HTTP::get_file_handle([id=[orig_h=141.142.2 XXXXXXXXXX.XXXXXX | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) XXXXXXXXXX.XXXXXX | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) XXXXXXXXXX.XXXXXX | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) -XXXXXXXXXX.XXXXXX | HookCallFunction cat(Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) +XXXXXXXXXX.XXXXXX | HookCallFunction cat(...) XXXXXXXXXX.XXXXXX | HookCallFunction file_new([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=, ftp=, http=, irc=, pe=]) XXXXXXXXXX.XXXXXX | HookCallFunction file_over_new_connection([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={}, rx_hosts={}, conn_uids={}, source=HTTP, depth=0, analyzers={}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=, irc=, pe=], [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=, resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) XXXXXXXXXX.XXXXXX | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp) @@ -4794,7 +4860,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Log::log_stream_policy, , ( XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Log::log_stream_policy, , ([ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], Files::LOG)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Log::write, , (Files::LOG, [ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Log::write, , (HTTP::LOG, [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -> -XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, ..., ...) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(file_sniff, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=4042, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=\x0a0.26 | 2012-08-24 15:10:04 -0700\x0a\x0a * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)\x0a\x0a * Fixing GPG signing script. (Robin Sommer)\x0a\x0a0.25 | 2012-08-01 13:55:46 -0500\x0a\x0a * Fix configure script to exit with non-zero status on error (Jon Siwek)\x0a\x0a0.24 | 2012-07-05 12:50:43 -0700\x0a\x0a * Raise minimum required CMake version to 2.6.3 (Jon Siwek)\x0a\x0a * Adding script to delete old fully-merged branches. (Robin Sommer)\x0a\x0a0.23-2 | 2012-01-25 13:24:01 -0800\x0a\x0a * Fix a bro-cut error message. (Daniel Thayer)\x0a\x0a0.23 | 2012-01-11 12:16:11 -0800\x0a\x0a * Tweaks to release scripts, plus a new one for signing files.\x0a (Robin Sommer)\x0a\x0a0.22 | 2012-01-10 16:45:19 -0800\x0a\x0a * Tweaks for OpenBSD support. (Jon Siwek)\x0a\x0a * bro-cut extensions and fixes. (Robin Sommer)\x0a \x0a - If no field names are given on the command line, we now pass through\x0a all fields. Adresses #657.\x0a\x0a - Removing some GNUism from awk script. Addresses #653.\x0a\x0a - Added option for time output in UTC. Addresses #668.\x0a\x0a - Added output field separator option -F. Addresses #649.\x0a\x0a - Fixing option -c: only some header lines were passed through\x0a rather than all. (Robin Sommer)\x0a\x0a * Fix parallel make portability. (Jon Siwek)\x0a\x0a0.21-9 | 2011-11-07 05:44:14 -0800\x0a\x0a * Fixing compiler warnings. Addresses #388. (Jon Siwek)\x0a\x0a0.21-2 | 2011-11-02 18:12:13 -0700\x0a\x0a * Fix for misnaming temp file in update-changes script. (Robin Sommer)\x0a\x0a0.21-1 | 2011-11-02 18:10:39 -0700\x0a\x0a * Little fix for make-release script, which could pick out the wrong\x0a tag. (Robin Sommer)\x0a\x0a0.21 | 2011-10-27 17:40:45 -0700\x0a\x0a * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)\x0a\x0a * Bugfix in update-changes script. (Robin Sommer)\x0a\x0a * update-changes now ignores commits it did itself. (Robin Sommer)\x0a\x0a * Fix a bug in the update-changes script. (Robin Sommer)\x0a\x0a * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)\x0a\x0a * Options to adjust time format for bro-cut. (Robin Sommer)\x0a\x0a The default with -d is now ISO format. The new option "-D "\x0a specifies a custom strftime()-style format string. Alternatively,\x0a the environment variable BRO_CUT_TIMEFMT can set the format as\x0a well.\x0a\x0a * bro-cut now understands the field separator header. (Robin Sommer)\x0a\x0a * Renaming options -h/-H -> -c/-C, and doing some general cleanup.\x0a\x0a0.2 | 2011-10-25 19:53:57 -0700\x0a\x0a * Adding support for replacing version string in a setup.py. (Robin\x0a Sommer)\x0a\x0a * Change generated root cert DN indices format for RFC2253\x0a compliance. (Jon Siwek)\x0a\x0a * New tool devel-tools/check-release to run before making releases.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes gets a new option -a to amend to\x0a previous commit if possible. Default is now not to (used to be the\x0a opposite). (Robin Sommer)\x0a\x0a * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)\x0a\x0a * Change distclean to only remove build dir. (Jon Siwek)\x0a\x0a * Make dist now cleans the copied source (Jon Siwek)\x0a\x0a * Small tweak to make-release for forced git-clean. (Jon Siwek)\x0a\x0a * Fix to not let updates scripts loose their executable permissions.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes now looks for a 'release' tag to\x0a idenfify the stable version, and 'beta' for the beta versions.\x0a (Robin Sommer).\x0a\x0a * Distribution cleanup. (Robin Sommer)\x0a\x0a * New script devel-tools/make-release to create source tar balls.\x0a (Robin Sommer)\x0a\x0a * Removing bdcat. With the new log format, this isn't very useful\x0a anymore. (Robin Sommer)\x0a\x0a * Adding script that shows all pending git fastpath commits. (Robin\x0a Sommer)\x0a\x0a * Script to measure CPU time by loading an increasing set of\x0a scripts. (Robin Sommer)\x0a\x0a * extract-conn script now deals wit *.gz files. (Robin Sommer)\x0a\x0a * Tiny update to output a valid CA list file for SSL cert\x0a validation. (Seth Hall)\x0a\x0a * Adding "install-aux" target. Addresses #622. (Jon Siwek)\x0a\x0a * Distribution cleanup. (Jon Siwek and Robin Sommer)\x0a\x0a * FindPCAP now links against thread library when necessary (e.g.\x0a PF_RING's libpcap) (Jon Siwek)\x0a\x0a * Install binaries with an RPATH (Jon Siwek)\x0a\x0a * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)\x0a\x0a * Rewrite of the update-changes script. (Robin Sommer)\x0a\x0a0.1-1 | 2011-06-14 21:12:41 -0700\x0a\x0a * Add a script for generating Mozilla's CA list for the SSL analyzer.\x0a (Seth Hall)\x0a\x0a0.1 | 2011-04-01 16:28:22 -0700\x0a\x0a * Converting build process to CMake. (Jon Siwek)\x0a\x0a * Removing cf<...>/ca-* from distribution. The README has a note where\x0a to find them now. (Robin Sommer)\x0a\x0a * General cleanup. (Robin Sommer)\x0a\x0a * Initial import of bro/aux from SVN r7088. (Jon Siwek)\x0a, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, pe=], [mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], inferred=T])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(file_state_remove, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=4, num_pkts=5, num_bytes_ip=4612, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 430.927277 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=\x0a0.26 | 2012-08-24 15:10:04 -0700\x0a\x0a * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)\x0a\x0a * Fixing GPG signing script. (Robin Sommer)\x0a\x0a0.25 | 2012-08-01 13:55:46 -0500\x0a\x0a * Fix configure script to exit with non-zero status on error (Jon Siwek)\x0a\x0a0.24 | 2012-07-05 12:50:43 -0700\x0a\x0a * Raise minimum required CMake version to 2.6.3 (Jon Siwek)\x0a\x0a * Adding script to delete old fully-merged branches. (Robin Sommer)\x0a\x0a0.23-2 | 2012-01-25 13:24:01 -0800\x0a\x0a * Fix a bro-cut error message. (Daniel Thayer)\x0a\x0a0.23 | 2012-01-11 12:16:11 -0800\x0a\x0a * Tweaks to release scripts, plus a new one for signing files.\x0a (Robin Sommer)\x0a\x0a0.22 | 2012-01-10 16:45:19 -0800\x0a\x0a * Tweaks for OpenBSD support. (Jon Siwek)\x0a\x0a * bro-cut extensions and fixes. (Robin Sommer)\x0a \x0a - If no field names are given on the command line, we now pass through\x0a all fields. Adresses #657.\x0a\x0a - Removing some GNUism from awk script. Addresses #653.\x0a\x0a - Added option for time output in UTC. Addresses #668.\x0a\x0a - Added output field separator option -F. Addresses #649.\x0a\x0a - Fixing option -c: only some header lines were passed through\x0a rather than all. (Robin Sommer)\x0a\x0a * Fix parallel make portability. (Jon Siwek)\x0a\x0a0.21-9 | 2011-11-07 05:44:14 -0800\x0a\x0a * Fixing compiler warnings. Addresses #388. (Jon Siwek)\x0a\x0a0.21-2 | 2011-11-02 18:12:13 -0700\x0a\x0a * Fix for misnaming temp file in update-changes script. (Robin Sommer)\x0a\x0a0.21-1 | 2011-11-02 18:10:39 -0700\x0a\x0a * Little fix for make-release script, which could pick out the wrong\x0a tag. (Robin Sommer)\x0a\x0a0.21 | 2011-10-27 17:40:45 -0700\x0a\x0a * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)\x0a\x0a * Bugfix in update-changes script. (Robin Sommer)\x0a\x0a * update-changes now ignores commits it did itself. (Robin Sommer)\x0a\x0a * Fix a bug in the update-changes script. (Robin Sommer)\x0a\x0a * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)\x0a\x0a * Options to adjust time format for bro-cut. (Robin Sommer)\x0a\x0a The default with -d is now ISO format. The new option "-D "\x0a specifies a custom strftime()-style format string. Alternatively,\x0a the environment variable BRO_CUT_TIMEFMT can set the format as\x0a well.\x0a\x0a * bro-cut now understands the field separator header. (Robin Sommer)\x0a\x0a * Renaming options -h/-H -> -c/-C, and doing some general cleanup.\x0a\x0a0.2 | 2011-10-25 19:53:57 -0700\x0a\x0a * Adding support for replacing version string in a setup.py. (Robin\x0a Sommer)\x0a\x0a * Change generated root cert DN indices format for RFC2253\x0a compliance. (Jon Siwek)\x0a\x0a * New tool devel-tools/check-release to run before making releases.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes gets a new option -a to amend to\x0a previous commit if possible. Default is now not to (used to be the\x0a opposite). (Robin Sommer)\x0a\x0a * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)\x0a\x0a * Change distclean to only remove build dir. (Jon Siwek)\x0a\x0a * Make dist now cleans the copied source (Jon Siwek)\x0a\x0a * Small tweak to make-release for forced git-clean. (Jon Siwek)\x0a\x0a * Fix to not let updates scripts loose their executable permissions.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes now looks for a 'release' tag to\x0a idenfify the stable version, and 'beta' for the beta versions.\x0a (Robin Sommer).\x0a\x0a * Distribution cleanup. (Robin Sommer)\x0a\x0a * New script devel-tools/make-release to create source tar balls.\x0a (Robin Sommer)\x0a\x0a * Removing bdcat. With the new log format, this isn't very useful\x0a anymore. (Robin Sommer)\x0a\x0a * Adding script that shows all pending git fastpath commits. (Robin\x0a Sommer)\x0a\x0a * Script to measure CPU time by loading an increasing set of\x0a scripts. (Robin Sommer)\x0a\x0a * extract-conn script now deals wit *.gz files. (Robin Sommer)\x0a\x0a * Tiny update to output a valid CA list file for SSL cert\x0a validation. (Seth Hall)\x0a\x0a * Adding "install-aux" target. Addresses #622. (Jon Siwek)\x0a\x0a * Distribution cleanup. (Jon Siwek and Robin Sommer)\x0a\x0a * FindPCAP now links against thread library when necessary (e.g.\x0a PF_RING's libpcap) (Jon Siwek)\x0a\x0a * Install binaries with an RPATH (Jon Siwek)\x0a\x0a * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)\x0a\x0a * Rewrite of the update-changes script. (Robin Sommer)\x0a\x0a0.1-1 | 2011-06-14 21:12:41 -0700\x0a\x0a * Add a script for generating Mozilla's CA list for the SSL analyzer.\x0a (Seth Hall)\x0a\x0a0.1 | 2011-04-01 16:28:22 -0700\x0a\x0a * Converting build process to CMake. (Jon Siwek)\x0a\x0a * Removing cf<...>/ca-* from distribution. The README has a note where\x0a to find them now. (Robin Sommer)\x0a\x0a * General cleanup. (Robin Sommer)\x0a\x0a * Initial import of bro/aux from SVN r7088. (Jon Siwek)\x0a, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4042, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, pe=])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) -> @@ -4827,7 +4893,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Log::log_stream_policy, , ( XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Log::log_stream_policy, , ([ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], Files::LOG)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Log::write, , (Files::LOG, [ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Log::write, , (HTTP::LOG, [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, ..., ...) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(file_sniff, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=4042, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=\x0a0.26 | 2012-08-24 15:10:04 -0700\x0a\x0a * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)\x0a\x0a * Fixing GPG signing script. (Robin Sommer)\x0a\x0a0.25 | 2012-08-01 13:55:46 -0500\x0a\x0a * Fix configure script to exit with non-zero status on error (Jon Siwek)\x0a\x0a0.24 | 2012-07-05 12:50:43 -0700\x0a\x0a * Raise minimum required CMake version to 2.6.3 (Jon Siwek)\x0a\x0a * Adding script to delete old fully-merged branches. (Robin Sommer)\x0a\x0a0.23-2 | 2012-01-25 13:24:01 -0800\x0a\x0a * Fix a bro-cut error message. (Daniel Thayer)\x0a\x0a0.23 | 2012-01-11 12:16:11 -0800\x0a\x0a * Tweaks to release scripts, plus a new one for signing files.\x0a (Robin Sommer)\x0a\x0a0.22 | 2012-01-10 16:45:19 -0800\x0a\x0a * Tweaks for OpenBSD support. (Jon Siwek)\x0a\x0a * bro-cut extensions and fixes. (Robin Sommer)\x0a \x0a - If no field names are given on the command line, we now pass through\x0a all fields. Adresses #657.\x0a\x0a - Removing some GNUism from awk script. Addresses #653.\x0a\x0a - Added option for time output in UTC. Addresses #668.\x0a\x0a - Added output field separator option -F. Addresses #649.\x0a\x0a - Fixing option -c: only some header lines were passed through\x0a rather than all. (Robin Sommer)\x0a\x0a * Fix parallel make portability. (Jon Siwek)\x0a\x0a0.21-9 | 2011-11-07 05:44:14 -0800\x0a\x0a * Fixing compiler warnings. Addresses #388. (Jon Siwek)\x0a\x0a0.21-2 | 2011-11-02 18:12:13 -0700\x0a\x0a * Fix for misnaming temp file in update-changes script. (Robin Sommer)\x0a\x0a0.21-1 | 2011-11-02 18:10:39 -0700\x0a\x0a * Little fix for make-release script, which could pick out the wrong\x0a tag. (Robin Sommer)\x0a\x0a0.21 | 2011-10-27 17:40:45 -0700\x0a\x0a * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)\x0a\x0a * Bugfix in update-changes script. (Robin Sommer)\x0a\x0a * update-changes now ignores commits it did itself. (Robin Sommer)\x0a\x0a * Fix a bug in the update-changes script. (Robin Sommer)\x0a\x0a * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)\x0a\x0a * Options to adjust time format for bro-cut. (Robin Sommer)\x0a\x0a The default with -d is now ISO format. The new option "-D "\x0a specifies a custom strftime()-style format string. Alternatively,\x0a the environment variable BRO_CUT_TIMEFMT can set the format as\x0a well.\x0a\x0a * bro-cut now understands the field separator header. (Robin Sommer)\x0a\x0a * Renaming options -h/-H -> -c/-C, and doing some general cleanup.\x0a\x0a0.2 | 2011-10-25 19:53:57 -0700\x0a\x0a * Adding support for replacing version string in a setup.py. (Robin\x0a Sommer)\x0a\x0a * Change generated root cert DN indices format for RFC2253\x0a compliance. (Jon Siwek)\x0a\x0a * New tool devel-tools/check-release to run before making releases.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes gets a new option -a to amend to\x0a previous commit if possible. Default is now not to (used to be the\x0a opposite). (Robin Sommer)\x0a\x0a * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)\x0a\x0a * Change distclean to only remove build dir. (Jon Siwek)\x0a\x0a * Make dist now cleans the copied source (Jon Siwek)\x0a\x0a * Small tweak to make-release for forced git-clean. (Jon Siwek)\x0a\x0a * Fix to not let updates scripts loose their executable permissions.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes now looks for a 'release' tag to\x0a idenfify the stable version, and 'beta' for the beta versions.\x0a (Robin Sommer).\x0a\x0a * Distribution cleanup. (Robin Sommer)\x0a\x0a * New script devel-tools/make-release to create source tar balls.\x0a (Robin Sommer)\x0a\x0a * Removing bdcat. With the new log format, this isn't very useful\x0a anymore. (Robin Sommer)\x0a\x0a * Adding script that shows all pending git fastpath commits. (Robin\x0a Sommer)\x0a\x0a * Script to measure CPU time by loading an increasing set of\x0a scripts. (Robin Sommer)\x0a\x0a * extract-conn script now deals wit *.gz files. (Robin Sommer)\x0a\x0a * Tiny update to output a valid CA list file for SSL cert\x0a validation. (Seth Hall)\x0a\x0a * Adding "install-aux" target. Addresses #622. (Jon Siwek)\x0a\x0a * Distribution cleanup. (Jon Siwek and Robin Sommer)\x0a\x0a * FindPCAP now links against thread library when necessary (e.g.\x0a PF_RING's libpcap) (Jon Siwek)\x0a\x0a * Install binaries with an RPATH (Jon Siwek)\x0a\x0a * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)\x0a\x0a * Rewrite of the update-changes script. (Robin Sommer)\x0a\x0a0.1-1 | 2011-06-14 21:12:41 -0700\x0a\x0a * Add a script for generating Mozilla's CA list for the SSL analyzer.\x0a (Seth Hall)\x0a\x0a0.1 | 2011-04-01 16:28:22 -0700\x0a\x0a * Converting build process to CMake. (Jon Siwek)\x0a\x0a * Removing cf<...>/ca-* from distribution. The README has a note where\x0a to find them now. (Robin Sommer)\x0a\x0a * General cleanup. (Robin Sommer)\x0a\x0a * Initial import of bro/aux from SVN r7088. (Jon Siwek)\x0a, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, pe=], [mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], inferred=T])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(file_state_remove, , ([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=4, num_pkts=5, num_bytes_ip=4612, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 430.927277 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=\x0a0.26 | 2012-08-24 15:10:04 -0700\x0a\x0a * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)\x0a\x0a * Fixing GPG signing script. (Robin Sommer)\x0a\x0a0.25 | 2012-08-01 13:55:46 -0500\x0a\x0a * Fix configure script to exit with non-zero status on error (Jon Siwek)\x0a\x0a0.24 | 2012-07-05 12:50:43 -0700\x0a\x0a * Raise minimum required CMake version to 2.6.3 (Jon Siwek)\x0a\x0a * Adding script to delete old fully-merged branches. (Robin Sommer)\x0a\x0a0.23-2 | 2012-01-25 13:24:01 -0800\x0a\x0a * Fix a bro-cut error message. (Daniel Thayer)\x0a\x0a0.23 | 2012-01-11 12:16:11 -0800\x0a\x0a * Tweaks to release scripts, plus a new one for signing files.\x0a (Robin Sommer)\x0a\x0a0.22 | 2012-01-10 16:45:19 -0800\x0a\x0a * Tweaks for OpenBSD support. (Jon Siwek)\x0a\x0a * bro-cut extensions and fixes. (Robin Sommer)\x0a \x0a - If no field names are given on the command line, we now pass through\x0a all fields. Adresses #657.\x0a\x0a - Removing some GNUism from awk script. Addresses #653.\x0a\x0a - Added option for time output in UTC. Addresses #668.\x0a\x0a - Added output field separator option -F. Addresses #649.\x0a\x0a - Fixing option -c: only some header lines were passed through\x0a rather than all. (Robin Sommer)\x0a\x0a * Fix parallel make portability. (Jon Siwek)\x0a\x0a0.21-9 | 2011-11-07 05:44:14 -0800\x0a\x0a * Fixing compiler warnings. Addresses #388. (Jon Siwek)\x0a\x0a0.21-2 | 2011-11-02 18:12:13 -0700\x0a\x0a * Fix for misnaming temp file in update-changes script. (Robin Sommer)\x0a\x0a0.21-1 | 2011-11-02 18:10:39 -0700\x0a\x0a * Little fix for make-release script, which could pick out the wrong\x0a tag. (Robin Sommer)\x0a\x0a0.21 | 2011-10-27 17:40:45 -0700\x0a\x0a * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)\x0a\x0a * Bugfix in update-changes script. (Robin Sommer)\x0a\x0a * update-changes now ignores commits it did itself. (Robin Sommer)\x0a\x0a * Fix a bug in the update-changes script. (Robin Sommer)\x0a\x0a * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)\x0a\x0a * Options to adjust time format for bro-cut. (Robin Sommer)\x0a\x0a The default with -d is now ISO format. The new option "-D "\x0a specifies a custom strftime()-style format string. Alternatively,\x0a the environment variable BRO_CUT_TIMEFMT can set the format as\x0a well.\x0a\x0a * bro-cut now understands the field separator header. (Robin Sommer)\x0a\x0a * Renaming options -h/-H -> -c/-C, and doing some general cleanup.\x0a\x0a0.2 | 2011-10-25 19:53:57 -0700\x0a\x0a * Adding support for replacing version string in a setup.py. (Robin\x0a Sommer)\x0a\x0a * Change generated root cert DN indices format for RFC2253\x0a compliance. (Jon Siwek)\x0a\x0a * New tool devel-tools/check-release to run before making releases.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes gets a new option -a to amend to\x0a previous commit if possible. Default is now not to (used to be the\x0a opposite). (Robin Sommer)\x0a\x0a * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)\x0a\x0a * Change distclean to only remove build dir. (Jon Siwek)\x0a\x0a * Make dist now cleans the copied source (Jon Siwek)\x0a\x0a * Small tweak to make-release for forced git-clean. (Jon Siwek)\x0a\x0a * Fix to not let updates scripts loose their executable permissions.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes now looks for a 'release' tag to\x0a idenfify the stable version, and 'beta' for the beta versions.\x0a (Robin Sommer).\x0a\x0a * Distribution cleanup. (Robin Sommer)\x0a\x0a * New script devel-tools/make-release to create source tar balls.\x0a (Robin Sommer)\x0a\x0a * Removing bdcat. With the new log format, this isn't very useful\x0a anymore. (Robin Sommer)\x0a\x0a * Adding script that shows all pending git fastpath commits. (Robin\x0a Sommer)\x0a\x0a * Script to measure CPU time by loading an increasing set of\x0a scripts. (Robin Sommer)\x0a\x0a * extract-conn script now deals wit *.gz files. (Robin Sommer)\x0a\x0a * Tiny update to output a valid CA list file for SSL cert\x0a validation. (Seth Hall)\x0a\x0a * Adding "install-aux" target. Addresses #622. (Jon Siwek)\x0a\x0a * Distribution cleanup. (Jon Siwek and Robin Sommer)\x0a\x0a * FindPCAP now links against thread library when necessary (e.g.\x0a PF_RING's libpcap) (Jon Siwek)\x0a\x0a * Install binaries with an RPATH (Jon Siwek)\x0a\x0a * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)\x0a\x0a * Rewrite of the update-changes script. (Robin Sommer)\x0a\x0a0.1-1 | 2011-06-14 21:12:41 -0700\x0a\x0a * Add a script for generating Mozilla's CA list for the SSL analyzer.\x0a (Seth Hall)\x0a\x0a0.1 | 2011-04-01 16:28:22 -0700\x0a\x0a * Converting build process to CMake. (Jon Siwek)\x0a\x0a * Removing cf<...>/ca-* from distribution. The README has a note where\x0a to find them now. (Robin Sommer)\x0a\x0a * General cleanup. (Robin Sommer)\x0a\x0a * Initial import of bro/aux from SVN r7088. (Jon Siwek)\x0a, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4042, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, pe=])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) @@ -4861,7 +4927,7 @@ XXXXXXXXXX.XXXXXX | HookCallFunction Log::log_stream_policy([ts=XXXXXXXXXX.XXXXX XXXXXXXXXX.XXXXXX | HookCallFunction Log::log_stream_policy([ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], Files::LOG) XXXXXXXXXX.XXXXXX | HookCallFunction Log::write(Files::LOG, [ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=]) XXXXXXXXXX.XXXXXX | HookCallFunction Log::write(HTTP::LOG, [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]) -XXXXXXXXXX.XXXXXX | HookCallFunction cat(Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) +XXXXXXXXXX.XXXXXX | HookCallFunction cat(...) XXXXXXXXXX.XXXXXX | HookCallFunction file_sniff([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=112, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 167.951584 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=4042, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=\x0a0.26 | 2012-08-24 15:10:04 -0700\x0a\x0a * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)\x0a\x0a * Fixing GPG signing script. (Robin Sommer)\x0a\x0a0.25 | 2012-08-01 13:55:46 -0500\x0a\x0a * Fix configure script to exit with non-zero status on error (Jon Siwek)\x0a\x0a0.24 | 2012-07-05 12:50:43 -0700\x0a\x0a * Raise minimum required CMake version to 2.6.3 (Jon Siwek)\x0a\x0a * Adding script to delete old fully-merged branches. (Robin Sommer)\x0a\x0a0.23-2 | 2012-01-25 13:24:01 -0800\x0a\x0a * Fix a bro-cut error message. (Daniel Thayer)\x0a\x0a0.23 | 2012-01-11 12:16:11 -0800\x0a\x0a * Tweaks to release scripts, plus a new one for signing files.\x0a (Robin Sommer)\x0a\x0a0.22 | 2012-01-10 16:45:19 -0800\x0a\x0a * Tweaks for OpenBSD support. (Jon Siwek)\x0a\x0a * bro-cut extensions and fixes. (Robin Sommer)\x0a \x0a - If no field names are given on the command line, we now pass through\x0a all fields. Adresses #657.\x0a\x0a - Removing some GNUism from awk script. Addresses #653.\x0a\x0a - Added option for time output in UTC. Addresses #668.\x0a\x0a - Added output field separator option -F. Addresses #649.\x0a\x0a - Fixing option -c: only some header lines were passed through\x0a rather than all. (Robin Sommer)\x0a\x0a * Fix parallel make portability. (Jon Siwek)\x0a\x0a0.21-9 | 2011-11-07 05:44:14 -0800\x0a\x0a * Fixing compiler warnings. Addresses #388. (Jon Siwek)\x0a\x0a0.21-2 | 2011-11-02 18:12:13 -0700\x0a\x0a * Fix for misnaming temp file in update-changes script. (Robin Sommer)\x0a\x0a0.21-1 | 2011-11-02 18:10:39 -0700\x0a\x0a * Little fix for make-release script, which could pick out the wrong\x0a tag. (Robin Sommer)\x0a\x0a0.21 | 2011-10-27 17:40:45 -0700\x0a\x0a * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)\x0a\x0a * Bugfix in update-changes script. (Robin Sommer)\x0a\x0a * update-changes now ignores commits it did itself. (Robin Sommer)\x0a\x0a * Fix a bug in the update-changes script. (Robin Sommer)\x0a\x0a * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)\x0a\x0a * Options to adjust time format for bro-cut. (Robin Sommer)\x0a\x0a The default with -d is now ISO format. The new option "-D "\x0a specifies a custom strftime()-style format string. Alternatively,\x0a the environment variable BRO_CUT_TIMEFMT can set the format as\x0a well.\x0a\x0a * bro-cut now understands the field separator header. (Robin Sommer)\x0a\x0a * Renaming options -h/-H -> -c/-C, and doing some general cleanup.\x0a\x0a0.2 | 2011-10-25 19:53:57 -0700\x0a\x0a * Adding support for replacing version string in a setup.py. (Robin\x0a Sommer)\x0a\x0a * Change generated root cert DN indices format for RFC2253\x0a compliance. (Jon Siwek)\x0a\x0a * New tool devel-tools/check-release to run before making releases.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes gets a new option -a to amend to\x0a previous commit if possible. Default is now not to (used to be the\x0a opposite). (Robin Sommer)\x0a\x0a * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)\x0a\x0a * Change distclean to only remove build dir. (Jon Siwek)\x0a\x0a * Make dist now cleans the copied source (Jon Siwek)\x0a\x0a * Small tweak to make-release for forced git-clean. (Jon Siwek)\x0a\x0a * Fix to not let updates scripts loose their executable permissions.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes now looks for a 'release' tag to\x0a idenfify the stable version, and 'beta' for the beta versions.\x0a (Robin Sommer).\x0a\x0a * Distribution cleanup. (Robin Sommer)\x0a\x0a * New script devel-tools/make-release to create source tar balls.\x0a (Robin Sommer)\x0a\x0a * Removing bdcat. With the new log format, this isn't very useful\x0a anymore. (Robin Sommer)\x0a\x0a * Adding script that shows all pending git fastpath commits. (Robin\x0a Sommer)\x0a\x0a * Script to measure CPU time by loading an increasing set of\x0a scripts. (Robin Sommer)\x0a\x0a * extract-conn script now deals wit *.gz files. (Robin Sommer)\x0a\x0a * Tiny update to output a valid CA list file for SSL cert\x0a validation. (Seth Hall)\x0a\x0a * Adding "install-aux" target. Addresses #622. (Jon Siwek)\x0a\x0a * Distribution cleanup. (Jon Siwek and Robin Sommer)\x0a\x0a * FindPCAP now links against thread library when necessary (e.g.\x0a PF_RING's libpcap) (Jon Siwek)\x0a\x0a * Install binaries with an RPATH (Jon Siwek)\x0a\x0a * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)\x0a\x0a * Rewrite of the update-changes script. (Robin Sommer)\x0a\x0a0.1-1 | 2011-06-14 21:12:41 -0700\x0a\x0a * Add a script for generating Mozilla's CA list for the SSL analyzer.\x0a (Seth Hall)\x0a\x0a0.1 | 2011-04-01 16:28:22 -0700\x0a\x0a * Converting build process to CMake. (Jon Siwek)\x0a\x0a * Removing cf<...>/ca-* from distribution. The README has a note where\x0a to find them now. (Robin Sommer)\x0a\x0a * General cleanup. (Robin Sommer)\x0a\x0a * Initial import of bro/aux from SVN r7088. (Jon Siwek)\x0a, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, pe=], [mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], inferred=T]) XXXXXXXXXX.XXXXXX | HookCallFunction file_state_remove([id=FMnxxt3xjVcWNS2141, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]] = [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=4, num_pkts=3, num_bytes_ip=304, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=4, num_pkts=5, num_bytes_ip=4612, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=140.0 msecs 430.927277 usecs, service={HTTP}, history=ShADad, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={[1] = [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]}, last_active=XXXXXXXXXX.XXXXXX, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=\x0a0.26 | 2012-08-24 15:10:04 -0700\x0a\x0a * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)\x0a\x0a * Fixing GPG signing script. (Robin Sommer)\x0a\x0a0.25 | 2012-08-01 13:55:46 -0500\x0a\x0a * Fix configure script to exit with non-zero status on error (Jon Siwek)\x0a\x0a0.24 | 2012-07-05 12:50:43 -0700\x0a\x0a * Raise minimum required CMake version to 2.6.3 (Jon Siwek)\x0a\x0a * Adding script to delete old fully-merged branches. (Robin Sommer)\x0a\x0a0.23-2 | 2012-01-25 13:24:01 -0800\x0a\x0a * Fix a bro-cut error message. (Daniel Thayer)\x0a\x0a0.23 | 2012-01-11 12:16:11 -0800\x0a\x0a * Tweaks to release scripts, plus a new one for signing files.\x0a (Robin Sommer)\x0a\x0a0.22 | 2012-01-10 16:45:19 -0800\x0a\x0a * Tweaks for OpenBSD support. (Jon Siwek)\x0a\x0a * bro-cut extensions and fixes. (Robin Sommer)\x0a \x0a - If no field names are given on the command line, we now pass through\x0a all fields. Adresses #657.\x0a\x0a - Removing some GNUism from awk script. Addresses #653.\x0a\x0a - Added option for time output in UTC. Addresses #668.\x0a\x0a - Added output field separator option -F. Addresses #649.\x0a\x0a - Fixing option -c: only some header lines were passed through\x0a rather than all. (Robin Sommer)\x0a\x0a * Fix parallel make portability. (Jon Siwek)\x0a\x0a0.21-9 | 2011-11-07 05:44:14 -0800\x0a\x0a * Fixing compiler warnings. Addresses #388. (Jon Siwek)\x0a\x0a0.21-2 | 2011-11-02 18:12:13 -0700\x0a\x0a * Fix for misnaming temp file in update-changes script. (Robin Sommer)\x0a\x0a0.21-1 | 2011-11-02 18:10:39 -0700\x0a\x0a * Little fix for make-release script, which could pick out the wrong\x0a tag. (Robin Sommer)\x0a\x0a0.21 | 2011-10-27 17:40:45 -0700\x0a\x0a * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)\x0a\x0a * Bugfix in update-changes script. (Robin Sommer)\x0a\x0a * update-changes now ignores commits it did itself. (Robin Sommer)\x0a\x0a * Fix a bug in the update-changes script. (Robin Sommer)\x0a\x0a * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)\x0a\x0a * Options to adjust time format for bro-cut. (Robin Sommer)\x0a\x0a The default with -d is now ISO format. The new option "-D "\x0a specifies a custom strftime()-style format string. Alternatively,\x0a the environment variable BRO_CUT_TIMEFMT can set the format as\x0a well.\x0a\x0a * bro-cut now understands the field separator header. (Robin Sommer)\x0a\x0a * Renaming options -h/-H -> -c/-C, and doing some general cleanup.\x0a\x0a0.2 | 2011-10-25 19:53:57 -0700\x0a\x0a * Adding support for replacing version string in a setup.py. (Robin\x0a Sommer)\x0a\x0a * Change generated root cert DN indices format for RFC2253\x0a compliance. (Jon Siwek)\x0a\x0a * New tool devel-tools/check-release to run before making releases.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes gets a new option -a to amend to\x0a previous commit if possible. Default is now not to (used to be the\x0a opposite). (Robin Sommer)\x0a\x0a * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)\x0a\x0a * Change distclean to only remove build dir. (Jon Siwek)\x0a\x0a * Make dist now cleans the copied source (Jon Siwek)\x0a\x0a * Small tweak to make-release for forced git-clean. (Jon Siwek)\x0a\x0a * Fix to not let updates scripts loose their executable permissions.\x0a (Robin Sommer)\x0a\x0a * devel-tools/update-changes now looks for a 'release' tag to\x0a idenfify the stable version, and 'beta' for the beta versions.\x0a (Robin Sommer).\x0a\x0a * Distribution cleanup. (Robin Sommer)\x0a\x0a * New script devel-tools/make-release to create source tar balls.\x0a (Robin Sommer)\x0a\x0a * Removing bdcat. With the new log format, this isn't very useful\x0a anymore. (Robin Sommer)\x0a\x0a * Adding script that shows all pending git fastpath commits. (Robin\x0a Sommer)\x0a\x0a * Script to measure CPU time by loading an increasing set of\x0a scripts. (Robin Sommer)\x0a\x0a * extract-conn script now deals wit *.gz files. (Robin Sommer)\x0a\x0a * Tiny update to output a valid CA list file for SSL cert\x0a validation. (Seth Hall)\x0a\x0a * Adding "install-aux" target. Addresses #622. (Jon Siwek)\x0a\x0a * Distribution cleanup. (Jon Siwek and Robin Sommer)\x0a\x0a * FindPCAP now links against thread library when necessary (e.g.\x0a PF_RING's libpcap) (Jon Siwek)\x0a\x0a * Install binaries with an RPATH (Jon Siwek)\x0a\x0a * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)\x0a\x0a * Rewrite of the update-changes script. (Robin Sommer)\x0a\x0a0.1-1 | 2011-06-14 21:12:41 -0700\x0a\x0a * Add a script for generating Mozilla's CA list for the SSL analyzer.\x0a (Seth Hall)\x0a\x0a0.1 | 2011-04-01 16:28:22 -0700\x0a\x0a * Converting build process to CMake. (Jon Siwek)\x0a\x0a * Removing cf<...>/ca-* from distribution. The README has a note where\x0a to find them now. (Robin Sommer)\x0a\x0a * General cleanup. (Robin Sommer)\x0a\x0a * Initial import of bro/aux from SVN r7088. (Jon Siwek)\x0a, info=[ts=XXXXXXXXXX.XXXXXX, fuid=FMnxxt3xjVcWNS2141, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CHhAvVGS1DHFjwGM9}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.975693 usecs, local_orig=, is_orig=F, seen_bytes=4042, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=, extracted_cutoff=, extracted_size=], ftp=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, pe=]) XXXXXXXXXX.XXXXXX | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp) @@ -4919,7 +4985,8 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Log::log_stream_policy, , ( XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Log::write, , (Conn::LOG, [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], proto=tcp, service=http, duration=211.0 msecs 483.955383 usecs, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents=])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(PacketAnalyzer::GTPV1::remove_gtpv1_connection, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(PacketAnalyzer::TEREDO::remove_teredo_connection, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) -> -XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, T, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(Telemetry::sync, , ()) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(cat, ..., ...) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(connection_state_remove, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(filter_change_tracking, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) -> @@ -4929,6 +4996,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(get_port_transport_proto, XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(is_tcp_port, , (59856/tcp)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(net_done, , (XXXXXXXXXX.XXXXXX)) -> +XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(run_sync_hook, , ()) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(sub_bytes, , (HTTP, 0, 1)) -> XXXXXXXXXX.XXXXXX MetaHookPost CallFunction(to_lower, , (HTTP)) -> @@ -4941,6 +5009,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(ChecksumOffloading::check()) -> fal XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(filter_change_tracking()) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> false +XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(run_sync_hook()) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(zeek_done()) -> false XXXXXXXXXX.XXXXXX MetaHookPost UpdateNetworkTime(XXXXXXXXXX.XXXXXX) -> XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::__flush_logs, , ()) @@ -4958,7 +5027,8 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Log::log_stream_policy, , ( XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Log::write, , (Conn::LOG, [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], proto=tcp, service=http, duration=211.0 msecs 483.955383 usecs, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents=])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(PacketAnalyzer::GTPV1::remove_gtpv1_connection, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(PacketAnalyzer::TEREDO::remove_teredo_connection, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) -XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, , (Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, T, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Telemetry::sync, , ()) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(cat, ..., ...) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(connection_state_remove, , ([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(filter_change_tracking, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp)) @@ -4968,6 +5038,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(get_port_transport_proto, XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp])) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(is_tcp_port, , (59856/tcp)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(net_done, , (XXXXXXXXXX.XXXXXX)) +XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(run_sync_hook, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(sub_bytes, , (HTTP, 0, 1)) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(to_lower, , (HTTP)) @@ -4980,6 +5051,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(ChecksumOffloading::check()) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=])) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(filter_change_tracking()) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) +XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(run_sync_hook()) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(zeek_done()) XXXXXXXXXX.XXXXXX MetaHookPre UpdateNetworkTime(XXXXXXXXXX.XXXXXX) XXXXXXXXXX.XXXXXX | HookUpdateNetworkTime XXXXXXXXXX.XXXXXX @@ -4998,7 +5070,8 @@ XXXXXXXXXX.XXXXXX | HookCallFunction Log::log_stream_policy([ts=XXXXXXXXXX.XXXXX XXXXXXXXXX.XXXXXX | HookCallFunction Log::write(Conn::LOG, [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], proto=tcp, service=http, duration=211.0 msecs 483.955383 usecs, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents=]) XXXXXXXXXX.XXXXXX | HookCallFunction PacketAnalyzer::GTPV1::remove_gtpv1_connection([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]) XXXXXXXXXX.XXXXXX | HookCallFunction PacketAnalyzer::TEREDO::remove_teredo_connection([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]) -XXXXXXXXXX.XXXXXX | HookCallFunction cat(Analyzer::ANALYZER_HTTP, XXXXXXXXXX.XXXXXX, T, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) +XXXXXXXXXX.XXXXXX | HookCallFunction Telemetry::sync() +XXXXXXXXXX.XXXXXX | HookCallFunction cat(...) XXXXXXXXXX.XXXXXX | HookCallFunction connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]) XXXXXXXXXX.XXXXXX | HookCallFunction filter_change_tracking() XXXXXXXXXX.XXXXXX | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856/tcp, 192.150.187.43, 80/tcp) @@ -5008,6 +5081,7 @@ XXXXXXXXXX.XXXXXX | HookCallFunction get_port_transport_proto(80/tcp) XXXXXXXXXX.XXXXXX | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]) XXXXXXXXXX.XXXXXX | HookCallFunction is_tcp_port(59856/tcp) XXXXXXXXXX.XXXXXX | HookCallFunction net_done(XXXXXXXXXX.XXXXXX) +XXXXXXXXXX.XXXXXX | HookCallFunction run_sync_hook() XXXXXXXXXX.XXXXXX | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTPXXXXXXXXXX.XXXXXXT11141.142.228.5:59856 > 192.150.187.43:80) XXXXXXXXXX.XXXXXX | HookCallFunction sub_bytes(HTTP, 0, 1) XXXXXXXXXX.XXXXXX | HookCallFunction to_lower(HTTP) @@ -5020,4 +5094,5 @@ XXXXXXXXXX.XXXXXX | HookQueueEvent ChecksumOffloading::check() XXXXXXXXXX.XXXXXX | HookQueueEvent connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=]) XXXXXXXXXX.XXXXXX | HookQueueEvent filter_change_tracking() XXXXXXXXXX.XXXXXX | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, removal_hooks={HTTP::finalize_http{ HTTP::r, HTTP::info{ if (HTTP::c?$http_state) { for ([HTTP::r] in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerceHTTP::info)}}}}}, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) +XXXXXXXXXX.XXXXXX | HookQueueEvent run_sync_hook() XXXXXXXXXX.XXXXXX | HookQueueEvent zeek_done() diff --git a/testing/btest/Baseline/scripts.base.frameworks.telemetry.basic/out b/testing/btest/Baseline/scripts.base.frameworks.telemetry.basic/out new file mode 100644 index 0000000000..c8c07fb8c5 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.telemetry.basic/out @@ -0,0 +1,25 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +### zeek_session_metrics |2| +Telemetry::INT_GAUGE, zeek, active-sessions, [protocol], [tcp], 500.0 +count_value, 500 +Telemetry::INT_COUNTER, zeek, total-sessions, [protocol], [tcp], 500.0 +count_value, 500 +### bt* metrics |5| +Telemetry::DOUBLE_COUNTER, btest, a_test, [x, y], [a, b], 1.0 +Telemetry::DOUBLE_COUNTER, btest, a_test, [x, y], [a, c], 2.0 +Telemetry::DOUBLE_COUNTER, btest, b_test, [x, y], [a, b], 10.0 +Telemetry::DOUBLE_COUNTER, btest, b_test, [x, y], [a, c], 20.0 +Telemetry::DOUBLE_COUNTER, btest, c_test, [x, y], [a, b], 200.0 +### btest_a_metrics |2| +Telemetry::DOUBLE_COUNTER, btest, a_test, [x, y], [a, b], 1.0 +Telemetry::DOUBLE_COUNTER, btest, a_test, [x, y], [a, c], 2.0 +### btest_b_metrics |2| +Telemetry::DOUBLE_COUNTER, btest, b_test, [x, y], [a, b], 10.0 +Telemetry::DOUBLE_COUNTER, btest, b_test, [x, y], [a, c], 20.0 +### system_metrics |3| +Telemetry::DOUBLE_GAUGE, system, sensor_temperature, [name], [cpu0], 43.0 +Telemetry::DOUBLE_GAUGE, system, sensor_temperature, [name], [cpu1], 44.1 +Telemetry::DOUBLE_GAUGE, system, sensor_temperature, [name], [cpu3], 42.2 +### btest_histogram_metrics |2| +Telemetry::DOUBLE_HISTOGRAM, btest, sample_histogram, [1.0, 2.0, 3.0, 4.0, 5.0, inf], [dim], [a], [2.0, 2.0, 0.0, 0.0, 0.0, 1.0], 11.5, 5.0 +Telemetry::DOUBLE_HISTOGRAM, btest, sample_histogram, [1.0, 2.0, 3.0, 4.0, 5.0, inf], [dim], [b], [1.0, 0.0, 0.0, 0.0, 0.0, 1.0], 7.5, 2.0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.telemetry.conn-duration-histogram/out b/testing/btest/Baseline/scripts.base.frameworks.telemetry.conn-duration-histogram/out new file mode 100644 index 0000000000..6857acd723 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.telemetry.conn-duration-histogram/out @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Telemetry::DOUBLE_HISTOGRAM, zeek, connection_duration +[] +[] +[2.0, 3.0, 4.0, 5.0, 6.0, 10.0, inf] +[0.0, 322.0, 90.0, 5.0, 76.0, 7.0, 0.0] +500.0, 1650.264644 +Telemetry::DOUBLE_HISTOGRAM, zeek, realistic_connection_duration +[proto] +[tcp] +[0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 900.0, 1800.0, inf] +[0.0, 0.0, 500.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +500.0, 1650.264644 diff --git a/testing/btest/Baseline/scripts.base.frameworks.telemetry.internal-metrics/out b/testing/btest/Baseline/scripts.base.frameworks.telemetry.internal-metrics/out new file mode 100644 index 0000000000..fdae1e75f3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.telemetry.internal-metrics/out @@ -0,0 +1,26 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +### broker |5| +Telemetry::INT_COUNTER, broker, processed-elements, [type], [data], 0.0 +count_value, 0 +Telemetry::INT_COUNTER, broker, processed-elements, [type], [command], 0.0 +count_value, 0 +Telemetry::INT_COUNTER, broker, processed-elements, [type], [routing-update], 0.0 +count_value, 0 +Telemetry::INT_COUNTER, broker, processed-elements, [type], [ping], 0.0 +count_value, 0 +Telemetry::INT_COUNTER, broker, processed-elements, [type], [pong], 0.0 +count_value, 0 +### caf |5| +Telemetry::INT_COUNTER, caf.system, rejected-messages, [], [], 0.0 +count_value, 0 +Telemetry::INT_COUNTER, caf.system, processed-messages, [], [], 7.0 +count_value, 7 +Telemetry::INT_GAUGE, caf.system, running-actors, [], [], 2.0 +count_value, 2 +Telemetry::INT_GAUGE, caf.system, queued-messages, [], [], 0.0 +count_value, 0 +Telemetry::INT_GAUGE, caf.actor, mailbox-size, [name], [broker.core], 0.0 +count_value, 0 +### caf |2| +Telemetry::DOUBLE_HISTOGRAM, caf.actor, processing-time, [0.00001, 0.0001, 0.0005, 0.001, 0.01, 0.1, 0.5, 1.0, 5.0, inf], [name], [broker.core] +Telemetry::DOUBLE_HISTOGRAM, caf.actor, mailbox-time, [0.00001, 0.0001, 0.0005, 0.001, 0.01, 0.1, 0.5, 1.0, 5.0, inf], [name], [broker.core] diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log new file mode 100644 index 0000000000..be6b64fe0c --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path telemetry +#open XXXX-XX-XX-XX-XX-XX +#fields ts peer metric_type prefix name unit labels label_values value +#types time string string string string string vector[string] vector[string] double +XXXXXXXXXX.XXXXXX zeek counter btest connections - proto tcp 500.0 +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log new file mode 100644 index 0000000000..412f007793 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path telemetry_histogram +#open XXXX-XX-XX-XX-XX-XX +#fields ts peer prefix name unit labels label_values bounds values sum observations +#types time string string string string vector[string] vector[string] vector[double] vector[double] double double +XXXXXXXXXX.XXXXXX zeek btest connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,0.0,0.0,0.0,0.0,0.0,0.0 0.0 0.0 +XXXXXXXXXX.XXXXXX zeek btest connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,322.0,90.0,5.0,76.0,7.0,0.0 1650.264644 500.0 +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log.filtered b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log.filtered new file mode 100644 index 0000000000..08154c52e4 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log.filtered @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +XXXXXXXXXX.XXXXXX - _zeek connection_duration (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,0.0,0.0,0.0,0.0,0.0,0.0 0.0 0.0 +XXXXXXXXXX.XXXXXX - _zeek connection_duration (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,322.0,90.0,5.0,76.0,7.0,0.0 1650.264644 500.0 diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered new file mode 100644 index 0000000000..654ba3bb99 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +XXXXXXXXXX.XXXXXX zeek gauge zeek active-sessions - protocol tcp 1.0 +XXXXXXXXXX.XXXXXX zeek counter zeek total-sessions - protocol tcp 1.0 +XXXXXXXXXX.XXXXXX zeek gauge zeek active-sessions - protocol tcp 500.0 +XXXXXXXXXX.XXXXXX zeek counter zeek total-sessions - protocol tcp 500.0 diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered new file mode 100644 index 0000000000..c0bf74163d --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +XXXXXXXXXX.XXXXXX zeek zeek connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,0.0,0.0,0.0,0.0,0.0,0.0 0.0 0.0 +XXXXXXXXXX.XXXXXX zeek zeek connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,322.0,90.0,5.0,76.0,7.0,0.0 1650.264644 500.0 diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.cc b/testing/btest/plugins/hooks-plugin/src/Plugin.cc index e4d64ab0fc..785ac764a9 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.cc +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.cc @@ -1,4 +1,3 @@ - #include "Plugin.h" #include @@ -8,6 +7,7 @@ #include #include #include +#include namespace btest::plugin::Demo_Hooks { @@ -16,6 +16,22 @@ Plugin plugin; using namespace btest::plugin::Demo_Hooks; +// Sanitize arguments for the following functions with (...). These +// receiving the current version string or parts of it and make the +// baseline non-deterministic. +static std::set sanitized_functions = { + "Version::parse", + "gsub", + "split_string1", + "lstrip", + "to_count", + "cat", + "Telemetry::__dbl_gauge_metric_get_or_add", + "Telemetry::gauge_with", + "Telemetry::make_labels", + "Telemetry::gauge_family_set", +}; + zeek::plugin::Configuration Plugin::Configure() { EnableHook(zeek::plugin::HOOK_LOAD_FILE); @@ -58,11 +74,7 @@ static void describe_hook_args(const zeek::plugin::HookArgumentList& args, zeek: // For function calls we remove args for unstable arguments // from parsing the version in `base/misc/version`. if ( i->GetType() == zeek::plugin::HookArgument::FUNC && - (::strcmp(d->Description(), "Version::parse") == 0 || - ::strcmp(d->Description(), "gsub") == 0 || - ::strcmp(d->Description(), "split_string1") == 0 || - ::strcmp(d->Description(), "lstrip") == 0 || - ::strcmp(d->Description(), "to_count") == 0) ) + sanitized_functions.count(d->Description()) != 0 ) serialize_args = false; continue; @@ -105,10 +117,8 @@ std::pair Plugin::HookFunctionCall(const zeek::Func* func, // For function calls we remove args for unstable arguments // from parsing the version in `base/misc/version`. - if ( ::strcmp(d.Description(), "Version::parse") == 0 || - ::strcmp(d.Description(), "gsub") == 0 || - ::strcmp(d.Description(), "split_string1") == 0 || - ::strcmp(d.Description(), "lstrip") == 0 || ::strcmp(d.Description(), "to_count") == 0 ) + // + if ( sanitized_functions.count(d.Description()) != 0 ) d.Add("(...)"); else zeek::plugin::HookArgument(args).Describe(&d); diff --git a/testing/btest/plugins/unprocessed-packet-hook.zeek b/testing/btest/plugins/unprocessed-packet-hook.zeek index f3e2f5da18..e64b5a759c 100644 --- a/testing/btest/plugins/unprocessed-packet-hook.zeek +++ b/testing/btest/plugins/unprocessed-packet-hook.zeek @@ -6,5 +6,4 @@ # @TEST-EXEC: hexdump -C unprocessed.pcap > unprocessed.pcap.hex # @TEST-EXEC: btest-diff unprocessed.pcap.hex -@unload base/misc/version @load base/init-default diff --git a/testing/btest/scripts/base/frameworks/telemetry/basic.zeek b/testing/btest/scripts/base/frameworks/telemetry/basic.zeek new file mode 100644 index 0000000000..32e85fa30b --- /dev/null +++ b/testing/btest/scripts/base/frameworks/telemetry/basic.zeek @@ -0,0 +1,116 @@ +# @TEST-DOC: Using and listing of counters and gauges using the telemetry module. +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC-FAIL: test -f reporter.log + +@load base/frameworks/telemetry + +global btest_a_cf = Telemetry::register_counter_family([ + $prefix="btest", + $name="a_test", + $unit="1", + $help_text="A btest metric", + $labels=vector("x", "y") +]); + +global btest_b_cf = Telemetry::register_counter_family([ + $prefix="btest", + $name="b_test", + $unit="1", + $help_text="Another btest metric", + $labels=vector("x", "y") +]); + +global btest_c_cf = Telemetry::register_counter_family([ + $prefix="btest", + $name="c_test", + $unit="1", + $help_text="The last btest metric", + $labels=vector("x", "y") +]); + +global system_sensor_temp_gf = Telemetry::register_gauge_family([ + $prefix="system", + $name="sensor_temperature", + $unit="celsius", + $help_text="Temperatures reported by sensors in the system", + $labels=vector("name") +]); + +global btest_sample_histogram_hf = Telemetry::register_histogram_family([ + $prefix="btest", + $name="sample_histogram", + $unit="1", + $help_text="A sample histogram that is not returned by Telemetry::collect_metrics", + $bounds=vector(1.0, 2.0, 3.0, 4.0, 5.0), + $labels=vector("dim") +]); + +function print_metrics(what: string, metrics: vector of Telemetry::Metric) + { + print fmt("### %s |%s|", what, |metrics|); + for (i in metrics) + { + local m = metrics[i]; + print m$opts$metric_type, m$opts$prefix, m$opts$name, m$opts$labels, m$labels, m$value; + + if (m?$count_value) + print "count_value", m$count_value; + } + } + +function print_histogram_metrics(what: string, metrics: vector of Telemetry::HistogramMetric) + { + print fmt("### %s |%s|", what, |metrics|); + for (i in metrics) + { + local m = metrics[i]; + print m$opts$metric_type, m$opts$prefix, m$opts$name, m$opts$bounds, m$opts$labels, m$labels, m$values, m$sum, m$observations; + } + } + +event zeek_done() &priority=-100 + { + Telemetry::counter_family_inc(btest_a_cf, vector("a", "b")); + Telemetry::counter_family_inc(btest_a_cf, vector("a", "c")); + Telemetry::counter_family_inc(btest_a_cf, vector("a", "c")); + + Telemetry::counter_family_inc(btest_b_cf, vector("a", "b"), 10.0); + Telemetry::counter_family_inc(btest_b_cf, vector("a", "c"), 20.0); + + Telemetry::counter_family_set(btest_c_cf, vector("a", "b"), 100.0); + Telemetry::counter_family_set(btest_c_cf, vector("a", "b"), 200.0); + + Telemetry::gauge_family_set(system_sensor_temp_gf, vector("cpu0"), 43.0); + Telemetry::gauge_family_set(system_sensor_temp_gf, vector("cpu1"), 43.1); + Telemetry::gauge_family_inc(system_sensor_temp_gf, vector("cpu1")); + Telemetry::gauge_family_set(system_sensor_temp_gf, vector("cpu3"), 43.2); + Telemetry::gauge_family_dec(system_sensor_temp_gf, vector("cpu3")); + + Telemetry::histogram_family_observe(btest_sample_histogram_hf, vector("a"), 0.5); + Telemetry::histogram_family_observe(btest_sample_histogram_hf, vector("a"), 0.9); + Telemetry::histogram_family_observe(btest_sample_histogram_hf, vector("a"), 1.1); + Telemetry::histogram_family_observe(btest_sample_histogram_hf, vector("a"), 2.0); + Telemetry::histogram_family_observe(btest_sample_histogram_hf, vector("a"), 7.0); + + Telemetry::histogram_family_observe(btest_sample_histogram_hf, vector("b"), 0.5); + Telemetry::histogram_family_observe(btest_sample_histogram_hf, vector("b"), 7.0); + + local zeek_session_metrics = Telemetry::collect_metrics("zeek", "*session*"); + print_metrics("zeek_session_metrics", zeek_session_metrics); + + local all_btest_metrics = Telemetry::collect_metrics("bt*", "*"); + print_metrics("bt* metrics", all_btest_metrics); + + local btest_a_metrics = Telemetry::collect_metrics("btest", "a_*"); + print_metrics("btest_a_metrics", btest_a_metrics); + + local btest_b_metrics = Telemetry::collect_metrics("btest", "b_*"); + print_metrics("btest_b_metrics", btest_b_metrics); + + local system_metrics = Telemetry::collect_metrics("system"); + print_metrics("system_metrics", system_metrics); + + local histogram_metrics = Telemetry::collect_histogram_metrics("btest"); + print_histogram_metrics("btest_histogram_metrics", histogram_metrics); + } diff --git a/testing/btest/scripts/base/frameworks/telemetry/conn-duration-histogram.zeek b/testing/btest/scripts/base/frameworks/telemetry/conn-duration-histogram.zeek new file mode 100644 index 0000000000..23999206a0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/telemetry/conn-duration-histogram.zeek @@ -0,0 +1,48 @@ +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC-FAIL: test -f reporter.log + +@load base/frameworks/telemetry + +global connection_duration_hf = Telemetry::register_histogram_family([ + $prefix="zeek", + $name="connection_duration", + $unit="seconds", + $help_text="Monitored connection durations", + $bounds=vector(2.0, 3.0, 4.0, 5.0, 6.0, 10.0) +]); + +global realistic_connection_duration_hf = Telemetry::register_histogram_family([ + $prefix="zeek", + $name="realistic_connection_duration", + $labels=vector("proto"), + $unit="seconds", + $help_text="Monitored connection durations by protocol", + $bounds=vector(0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300, 900.0, 1800.0) +]); + +global connection_duration_h = Telemetry::histogram_with(connection_duration_hf); + +event connection_state_remove(c: connection) + { + Telemetry::histogram_observe(connection_duration_h, interval_to_double(c$duration)); + local proto = to_lower(cat(get_port_transport_proto(c$id$resp_p))); + Telemetry::histogram_family_observe(realistic_connection_duration_hf, + vector(proto), + interval_to_double(c$duration)); + } + +event zeek_done() &priority=-100 + { + local histogram_metrics = Telemetry::collect_histogram_metrics("zeek", "*connection_duration"); + for (i in histogram_metrics) + { + local hm = histogram_metrics[i]; + print hm$opts$metric_type, hm$opts$prefix, hm$opts$name; + print hm$opts$labels; + print hm$labels; + print hm$opts$bounds; + print hm$values; + print hm$observations, hm$sum; + } + } diff --git a/testing/btest/scripts/base/frameworks/telemetry/internal-metrics.zeek b/testing/btest/scripts/base/frameworks/telemetry/internal-metrics.zeek new file mode 100644 index 0000000000..9d7b549fc9 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/telemetry/internal-metrics.zeek @@ -0,0 +1,43 @@ +# @TEST-DOC: Query some internal broker/caf related metrics as they use the int64_t versions, too. +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC-FAIL: test -f reporter.log + +@load base/frameworks/telemetry + +function print_histogram_metrics(what: string, metrics: vector of Telemetry::HistogramMetric) + { + print fmt("### %s |%s|", what, |metrics|); + for (i in metrics) + { + local m = metrics[i]; + print m$opts$metric_type, m$opts$prefix, m$opts$name, m$opts$bounds, m$opts$labels, m$labels; + # Don't output actual values as they are runtime dependent. + # print m$values, m$sum, m$observations; + if ( m$opts?$count_bounds ) + print m$opts$count_bounds; + } + } + +function print_metrics(what: string, metrics: vector of Telemetry::Metric) + { + print fmt("### %s |%s|", what, |metrics|); + for (i in metrics) + { + local m = metrics[i]; + print m$opts$metric_type, m$opts$prefix, m$opts$name, m$opts$labels, m$labels, m$value; + + if (m?$count_value) + print "count_value", m$count_value; + } + } + +event zeek_done() &priority=-100 + { + local broker_metrics = Telemetry::collect_metrics("broker", "*"); + print_metrics("broker", broker_metrics); + local caf_metrics = Telemetry::collect_metrics("caf*", "*"); + print_metrics("caf", caf_metrics); + local caf_histogram_metrics = Telemetry::collect_histogram_metrics("caf*", "*"); + print_histogram_metrics("caf", caf_histogram_metrics); + } diff --git a/testing/btest/scripts/policy/frameworks/telemetry/log-prefixes.zeek b/testing/btest/scripts/policy/frameworks/telemetry/log-prefixes.zeek new file mode 100644 index 0000000000..1030f05c39 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/telemetry/log-prefixes.zeek @@ -0,0 +1,34 @@ +# @TEST-DOC: Tests that setting log_prefixes filters out the zeek metrics normally created. +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out + +# @TEST-EXEC: btest-diff telemetry.log +# @TEST-EXEC: btest-diff telemetry_histogram.log + +@load frameworks/telemetry/log + +redef Telemetry::log_prefixes = {"btest"}; + +global connections_by_proto_cf = Telemetry::register_counter_family([ + $prefix="btest", + $name="connections", + $unit="1", + $help_text="Total number of monitored connections", + $labels=vector("proto") +]); + +global connection_duration_hf = Telemetry::register_histogram_family([ + $prefix="btest", + $name="connection_duration", + $unit="seconds", + $help_text="Monitored connection duration", + $bounds=vector(2.0, 3.0, 4.0, 5.0, 6.0, 10.0) +]); + +global connection_duration_h = Telemetry::histogram_with(connection_duration_hf); + +event connection_state_remove(c: connection) + { + local proto = to_lower(cat(get_port_transport_proto(c$id$orig_p))); + Telemetry::counter_family_inc(connections_by_proto_cf, vector(proto)); + Telemetry::histogram_observe(connection_duration_h, interval_to_double(c$duration)); + } diff --git a/testing/btest/scripts/policy/frameworks/telemetry/log.zeek b/testing/btest/scripts/policy/frameworks/telemetry/log.zeek new file mode 100644 index 0000000000..ab74478913 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/telemetry/log.zeek @@ -0,0 +1,25 @@ +# @TEST-DOC: Test loading of telemetry/log and smoke check the telemetry.log file +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out +# @TEST-EXEC: grep 'zeek.*sessions' telemetry.log > telemetry.log.filtered +# @TEST-EXEC: grep 'zeek.*connection_duration' telemetry_histogram.log > telemetry_histogram.log.filtered + +# @TEST-EXEC: btest-diff telemetry.log.filtered +# @TEST-EXEC: btest-diff telemetry_histogram.log.filtered + +@load frameworks/telemetry/log + + +global connection_duration_hf = Telemetry::register_histogram_family([ + $prefix="zeek", + $name="connection_duration", + $unit="seconds", + $help_text="Monitored connection duration", + $bounds=vector(2.0, 3.0, 4.0, 5.0, 6.0, 10.0) +]); + +global connection_duration_h = Telemetry::histogram_with(connection_duration_hf); + +event connection_state_remove(c: connection) + { + Telemetry::histogram_observe(connection_duration_h, interval_to_double(c$duration)); + } diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 9ff3c836dd..8a8e99b426 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -5b2a6d78f789f1271b68123875ef66eaaba6f3e8 +eb531a695792f98f3e5e9fca7d0a922e5dc5d677 diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index db3b63b2c7..62a1333ba9 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -d8088ba741389aa092b5fb284d0849401234809f +de026605b1bd9650c8e9ea3e4448790366dfccce diff --git a/testing/external/scripts/testing-setup.zeek b/testing/external/scripts/testing-setup.zeek index 18e7c4783f..8fa050895e 100644 --- a/testing/external/scripts/testing-setup.zeek +++ b/testing/external/scripts/testing-setup.zeek @@ -12,3 +12,13 @@ # (json-logs.zeek activates this). redef LogAscii::use_json = F; @endif + +# Exclude process metrics, they are non-deterministic. +redef Telemetry::log_prefixes -= { "process" }; + +# Prevent the version_info metric from being logged as it's not deterministic. +hook Telemetry::log_policy(rec: Telemetry::Info, id: Log::ID, filter: Log::Filter) + { + if ( rec$prefix == "zeek" && rec$name == "version_info" ) + break; + }