diff --git a/scripts/policy/frameworks/telemetry/log.zeek b/scripts/policy/frameworks/telemetry/log.zeek index 42ecb0cbc4..d3a51d5db9 100644 --- a/scripts/policy/frameworks/telemetry/log.zeek +++ b/scripts/policy/frameworks/telemetry/log.zeek @@ -39,9 +39,6 @@ export { ## The name of the metric. name: string &log; - ## The unit of this metric, or unset if unit-less. - unit: string &log &optional; - ## The names of the individual labels. labels: vector of string &log; @@ -66,9 +63,6 @@ export { ## The name of the metric. name: string &log; - ## The unit of this metric, or unset if unit-less. - unit: string &log &optional; - ## The names of the individual labels. labels: vector of string &log; @@ -104,7 +98,21 @@ export { function do_log() { local ts = network_time(); - local metrics = Telemetry::collect_metrics(); + + ## TODO: this is potentially slow, since it requires looping over all of the metrics for each + ## prefix, and then doing it again for all of the histograms multiple times. + local metrics : vector of Telemetry::Metric; + if ( |log_prefixes| > 0 ) + { + for ( prefix in log_prefixes ) + { + metrics += Telemetry::collect_metrics(prefix, "*"); + } + } + else + { + metrics = Telemetry::collect_metrics(); + } for ( i in metrics ) { @@ -114,9 +122,6 @@ function do_log() if ( m$opts$metric_type == DOUBLE_HISTOGRAM || m$opts$metric_type == INT_HISTOGRAM ) next; - if ( |log_prefixes| > 0 && m$opts$prefix !in log_prefixes ) - next; - # Render the metric_type as a short string. Unknown # shouldn't really happen, but lets have a fallback. local metric_type = "unknown"; @@ -138,22 +143,29 @@ function do_log() $label_values=m$labels, $value=m$value); - if ( m$opts$unit != "1" ) - rec$unit = m$opts$unit; - Log::write(LOG, rec); } # Logging of histograms. ts = network_time(); - local histogram_metrics = Telemetry::collect_histogram_metrics(); + + local histogram_metrics : vector of Telemetry::HistogramMetric; + if ( |log_prefixes| > 0 ) + { + for ( prefix in log_prefixes ) + { + histogram_metrics += Telemetry::collect_histogram_metrics(prefix, "*"); + } + } + else + { + histogram_metrics = Telemetry::collect_histogram_metrics(); + } + for ( i in histogram_metrics ) { local hm = histogram_metrics[i]; - if ( |log_prefixes| > 0 && hm$opts$prefix !in log_prefixes ) - next; - local hrec = HistogramInfo($ts=ts, $peer=peer_description, $prefix=hm$opts$prefix, @@ -165,9 +177,6 @@ function do_log() $sum=hm$sum, $observations=hm$observations); - if ( hm$opts$unit != "1" ) - hrec$unit = hm$opts$unit; - Log::write(LOG_HISTOGRAM, hrec); } } diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.telemetry/telemetry.log b/testing/btest/Baseline/scripts.base.frameworks.logging.telemetry/telemetry.log index 16332e424f..6e0e60a2f8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.telemetry/telemetry.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.telemetry/telemetry.log @@ -5,12 +5,12 @@ #unset_field - #path telemetry #open XXXX-XX-XX-XX-XX-XX -#fields ts peer metric_type prefix name unit labels label_values value -#types time string string string string string vector[string] vector[string] double -XXXXXXXXXX.XXXXXX zeek counter zeek log-stream-writes - module,stream HTTP,HTTP::LOG 14.0 -XXXXXXXXXX.XXXXXX zeek counter zeek log-stream-writes - module,stream DNS,DNS::LOG 34.0 -XXXXXXXXXX.XXXXXX zeek counter zeek log-stream-writes - module,stream Conn,Conn::LOG 34.0 -XXXXXXXXXX.XXXXXX zeek counter zeek log-writer-writes - filter-name,module,path,stream,writer default,HTTP,http,HTTP::LOG,Log::WRITER_ASCII 10.0 -XXXXXXXXXX.XXXXXX zeek counter zeek log-writer-writes - filter-name,module,path,stream,writer default,DNS,dns,DNS::LOG,Log::WRITER_ASCII 23.0 -XXXXXXXXXX.XXXXXX zeek counter zeek log-writer-writes - filter-name,module,path,stream,writer default,Conn,conn,Conn::LOG,Log::WRITER_ASCII 30.0 +#fields ts peer metric_type prefix name labels label_values value +#types time string string string string vector[string] vector[string] double +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_log_stream_writes_total module,stream Conn,Conn::LOG 34.0 +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_log_stream_writes_total module,stream DNS,DNS::LOG 34.0 +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_log_stream_writes_total module,stream HTTP,HTTP::LOG 14.0 +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_log_writer_writes_total writer,module,stream,filter-name,path default,Conn,conn,Conn::LOG,Log::WRITER_ASCII 30.0 +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_log_writer_writes_total writer,module,stream,filter-name,path default,DNS,dns,DNS::LOG,Log::WRITER_ASCII 23.0 +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_log_writer_writes_total writer,module,stream,filter-name,path default,HTTP,http,HTTP::LOG,Log::WRITER_ASCII 10.0 #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log index be6b64fe0c..df759bf7a1 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry.log @@ -5,7 +5,7 @@ #unset_field - #path telemetry #open XXXX-XX-XX-XX-XX-XX -#fields ts peer metric_type prefix name unit labels label_values value -#types time string string string string string vector[string] vector[string] double -XXXXXXXXXX.XXXXXX zeek counter btest connections - proto tcp 500.0 +#fields ts peer metric_type prefix name labels label_values value +#types time string string string string vector[string] vector[string] double +XXXXXXXXXX.XXXXXX zeek counter btest btest_connections proto tcp 500.0 #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log index 412f007793..79adb57972 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log-prefixes/telemetry_histogram.log @@ -5,8 +5,8 @@ #unset_field - #path telemetry_histogram #open XXXX-XX-XX-XX-XX-XX -#fields ts peer prefix name unit labels label_values bounds values sum observations -#types time string string string string vector[string] vector[string] vector[double] vector[double] double double -XXXXXXXXXX.XXXXXX zeek btest connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,0.0,0.0,0.0,0.0,0.0,0.0 0.0 0.0 -XXXXXXXXXX.XXXXXX zeek btest connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,322.0,90.0,5.0,76.0,7.0,0.0 1650.264644 500.0 +#fields ts peer prefix name labels label_values bounds values sum observations +#types time string string string vector[string] vector[string] vector[double] vector[double] double double +XXXXXXXXXX.XXXXXX zeek btest btest_connection_duration_seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,0.0,0.0,0.0,0.0,0.0,0.0 0.0 0.0 +XXXXXXXXXX.XXXXXX zeek btest btest_connection_duration_seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,322.0,90.0,5.0,76.0,7.0,0.0 1650.264644 500.0 #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered index b7b4071151..19b4b90ddb 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry.log.filtered @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -XXXXXXXXXX.XXXXXX zeek gauge zeek active_sessions (empty) protocol tcp 1.0 -XXXXXXXXXX.XXXXXX zeek counter zeek total_sessions - protocol tcp 1.0 -XXXXXXXXXX.XXXXXX zeek gauge zeek active_sessions (empty) protocol tcp 500.0 -XXXXXXXXXX.XXXXXX zeek counter zeek total_sessions - protocol tcp 500.0 +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_total_sessions_total protocol tcp 1.0 +XXXXXXXXXX.XXXXXX zeek gauge zeek zeek_active_sessions protocol tcp 1.0 +XXXXXXXXXX.XXXXXX zeek counter zeek zeek_total_sessions_total protocol tcp 500.0 +XXXXXXXXXX.XXXXXX zeek gauge zeek zeek_active_sessions protocol tcp 500.0 diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered index c0bf74163d..b382cd5ca6 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.log/telemetry_histogram.log.filtered @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -XXXXXXXXXX.XXXXXX zeek zeek connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,0.0,0.0,0.0,0.0,0.0,0.0 0.0 0.0 -XXXXXXXXXX.XXXXXX zeek zeek connection_duration seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,322.0,90.0,5.0,76.0,7.0,0.0 1650.264644 500.0 +XXXXXXXXXX.XXXXXX zeek zeek zeek_connection_duration_seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,0.0,0.0,0.0,0.0,0.0,0.0 0.0 0.0 +XXXXXXXXXX.XXXXXX zeek zeek zeek_connection_duration_seconds (empty) (empty) 2.0,3.0,4.0,5.0,6.0,10.0,inf 0.0,322.0,90.0,5.0,76.0,7.0,0.0 1650.264644 500.0 diff --git a/testing/btest/scripts/base/frameworks/logging/telemetry.zeek b/testing/btest/scripts/base/frameworks/logging/telemetry.zeek index 4e7396d56d..a71ffd7d00 100644 --- a/testing/btest/scripts/base/frameworks/logging/telemetry.zeek +++ b/testing/btest/scripts/base/frameworks/logging/telemetry.zeek @@ -28,7 +28,7 @@ hook Log::log_stream_policy(rec: any, id: Log::ID) hook Telemetry::log_policy(rec: Telemetry::Info, id: Log::ID, filter: Log::Filter) { - if ( rec$prefix != "zeek" || /^log-/ !in rec$name ) + if ( rec$prefix != "zeek" || /^zeek_log_/ !in rec$name ) break; if ( /HTTP|DNS|Conn/ !in cat(rec$label_values) )