| and the next
- // literal block after the table is the literal block that we want
- // to show/hide
- $(this).parent().parent().parent().parent().next('.highlight-python').slideToggle('fast');
-
- // override default link behavior
- return false;
- });
-});
-
-// make "Private Interface" sections hidden by default
-$(document).ready(function() {
-
- var showText='Show Private Interface (for internal use)';
- var hideText='Hide Private Interface';
-
- var is_visible = false;
-
- // insert show/hide links
- $('#private-interface').children(":first-child").after(''+showText+'');
-
- // wrap all sub-sections in a new div that can be hidden/shown
- $('#private-interface').children(".section").wrapAll('');
-
- // hide the given class
- $('.private').hide();
-
- // register handler for clicking a "toggle" link
- $('a.privateToggle').click(function() {
- is_visible = !is_visible;
-
- $(this).html( (!is_visible) ? showText : hideText);
-
- $('.private').slideToggle('fast');
-
- // override default link behavior
- return false;
- });
-});
diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html
index cb6181361d..a4775f5870 100644
--- a/doc/_templates/layout.html
+++ b/doc/_templates/layout.html
@@ -73,6 +73,19 @@
{% endif %}
+ {%- if pagename != "search" %}
+
+
+
+
+
+ {%- endif %}
+
diff --git a/doc/scripts/example.bro b/doc/scripts/example.bro
index 20591072f2..9f6f656ee1 100644
--- a/doc/scripts/example.bro
+++ b/doc/scripts/example.bro
@@ -183,8 +183,7 @@ export {
## Summarize "an_event" here.
## Give more details about "an_event" here.
- ## Example::an_event should not be confused as a parameter
- ##
+ ## Example::an_event should not be confused as a parameter.
## name: describe the argument here
global an_event: event(name: string);
diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro
index 84d9767875..4e91036c55 100644
--- a/scripts/base/frameworks/cluster/setup-connections.bro
+++ b/scripts/base/frameworks/cluster/setup-connections.bro
@@ -41,7 +41,7 @@ event bro_init() &priority=9
{
if ( n$node_type == WORKER && n$proxy == node )
Communication::nodes[i] =
- [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events];
+ [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events];
# accepts connections from the previous one.
# (This is not ideal for setups with many proxies)
diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro
index 5c04fdd3d9..c285512dd5 100644
--- a/scripts/base/frameworks/logging/writers/ascii.bro
+++ b/scripts/base/frameworks/logging/writers/ascii.bro
@@ -19,8 +19,9 @@ export {
## Separator between set elements.
const set_separator = "," &redef;
- ## String to use for empty fields.
- const empty_field = "-" &redef;
+ ## String to use for empty fields. This should be different from
+ ## *unset_field* to make the output non-ambigious.
+ const empty_field = "(empty)" &redef;
## String to use for an unset &optional field.
const unset_field = "-" &redef;
diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro
index a73810c726..17a597678d 100644
--- a/scripts/base/frameworks/notice/extend-email/hostnames.bro
+++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro
@@ -2,31 +2,46 @@
module Notice;
-# This probably doesn't actually work due to the async lookup_addr.
+# We have to store references to the notices here because the when statement
+# clones the frame which doesn't give us access to modify values outside
+# of it's execution scope. (we get a clone of the notice instead of a
+# reference to the original notice)
+global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs;
+
event Notice::notice(n: Notice::Info) &priority=10
{
if ( ! n?$src && ! n?$dst )
return;
-
+
# This should only be done for notices that are being sent to email.
if ( ACTION_EMAIL !in n$actions )
return;
-
+
+ # I'm not recovering gracefully from the when statements because I want
+ # the notice framework to detect that something has exceeded the maximum
+ # allowed email delay and tell the user.
+ local uid = unique_id("");
+ tmp_notice_storage[uid] = n;
+
local output = "";
if ( n?$src )
{
+ add n$email_delay_tokens["hostnames-src"];
when ( local src_name = lookup_addr(n$src) )
{
- output = string_cat("orig_h/src hostname: ", src_name, "\n");
- n$email_body_sections[|n$email_body_sections|] = output;
+ output = string_cat("orig/src hostname: ", src_name, "\n");
+ tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output;
+ delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-src"];
}
}
if ( n?$dst )
{
+ add n$email_delay_tokens["hostnames-dst"];
when ( local dst_name = lookup_addr(n$dst) )
{
- output = string_cat("resp_h/dst hostname: ", dst_name, "\n");
- n$email_body_sections[|n$email_body_sections|] = output;
+ output = string_cat("resp/dst hostname: ", dst_name, "\n");
+ tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output;
+ delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-dst"];
}
}
}
diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro
index 7d98c6464c..72b87a8044 100644
--- a/scripts/base/frameworks/notice/main.bro
+++ b/scripts/base/frameworks/notice/main.bro
@@ -8,9 +8,9 @@
module Notice;
export {
- redef enum Log::ID += {
+ redef enum Log::ID += {
## This is the primary logging stream for notices.
- LOG,
+ LOG,
## This is the notice policy auditing log. It records what the current
## notice policy is at Bro init time.
POLICY_LOG,
@@ -18,25 +18,25 @@ export {
ALARM_LOG,
};
- ## Scripts creating new notices need to redef this enum to add their own
+ ## Scripts creating new notices need to redef this enum to add their own
## specific notice types which would then get used when they call the
## :bro:id:`NOTICE` function. The convention is to give a general category
## along with the specific notice separating words with underscores and using
## leading capitals on each word except for abbreviations which are kept in
- ## all capitals. For example, SSH::Login is for heuristically guessed
+ ## all capitals. For example, SSH::Login is for heuristically guessed
## successful SSH logins.
type Type: enum {
## Notice reporting a count of how often a notice occurred.
Tally,
};
-
+
## These are values representing actions that can be taken with notices.
type Action: enum {
## Indicates that there is no action to be taken.
ACTION_NONE,
## Indicates that the notice should be sent to the notice logging stream.
ACTION_LOG,
- ## Indicates that the notice should be sent to the email address(es)
+ ## Indicates that the notice should be sent to the email address(es)
## configured in the :bro:id:`Notice::mail_dest` variable.
ACTION_EMAIL,
## Indicates that the notice should be alarmed. A readable ASCII
@@ -47,30 +47,30 @@ export {
## duplicate notice suppression that the notice framework does.
ACTION_NO_SUPPRESS,
};
-
- ## The notice framework is able to do automatic notice supression by
+
+ ## The notice framework is able to do automatic notice supression by
## utilizing the $identifier field in :bro:type:`Info` records.
## Set this to "0secs" to completely disable automated notice suppression.
const default_suppression_interval = 1hrs &redef;
-
+
type Info: record {
ts: time &log &optional;
uid: string &log &optional;
id: conn_id &log &optional;
-
+
## These are shorthand ways of giving the uid and id to a notice. The
## reference to the actual connection will be deleted after applying
## the notice policy.
conn: connection &optional;
iconn: icmp_conn &optional;
-
+
## The :bro:enum:`Notice::Type` of the notice.
note: Type &log;
## The human readable message for the notice.
msg: string &log &optional;
## The human readable sub-message.
sub: string &log &optional;
-
+
## Source address, if we don't have a :bro:type:`conn_id`.
src: addr &log &optional;
## Destination address.
@@ -79,33 +79,39 @@ export {
p: port &log &optional;
## Associated count, or perhaps a status code.
n: count &log &optional;
-
+
## Peer that raised this notice.
src_peer: event_peer &optional;
## Textual description for the peer that raised this notice.
peer_descr: string &log &optional;
-
+
## The actions which have been applied to this notice.
actions: set[Notice::Action] &log &optional;
-
+
## These are policy items that returned T and applied their action
## to the notice.
policy_items: set[count] &log &optional;
-
+
## By adding chunks of text into this element, other scripts can
## expand on notices that are being emailed. The normal way to add text
## is to extend the vector by handling the :bro:id:`Notice::notice`
## event and modifying the notice in place.
- email_body_sections: vector of string &default=vector();
-
+ email_body_sections: vector of string &optional;
+
+ ## Adding a string "token" to this set will cause the notice framework's
+ ## built-in emailing functionality to delay sending the email until
+ ## either the token has been removed or the email has been delayed
+ ## for :bro:id:`max_email_delay`.
+ email_delay_tokens: set[string] &optional;
+
## This field is to be provided when a notice is generated for the
## purpose of deduplicating notices. The identifier string should
- ## be unique for a single instance of the notice. This field should be
- ## filled out in almost all cases when generating notices to define
+ ## be unique for a single instance of the notice. This field should be
+ ## filled out in almost all cases when generating notices to define
## when a notice is conceptually a duplicate of a previous notice.
- ##
- ## For example, an SSL certificate that is going to expire soon should
- ## always have the same identifier no matter the client IP address
+ ##
+ ## For example, an SSL certificate that is going to expire soon should
+ ## always have the same identifier no matter the client IP address
## that connected and resulted in the certificate being exposed. In
## this case, the resp_h, resp_p, and hash of the certificate would be
## used to create this value. The hash of the cert is included
@@ -114,19 +120,19 @@ export {
## Another example might be a host downloading a file which triggered
## a notice because the MD5 sum of the file it downloaded was known
## by some set of intelligence. In that case, the orig_h (client)
- ## and MD5 sum would be used in this field to dedup because if the
+ ## and MD5 sum would be used in this field to dedup because if the
## same file is downloaded over and over again you really only want to
## know about it a single time. This makes it possible to send those
## notices to email without worrying so much about sending thousands
## of emails.
identifier: string &optional;
-
+
## This field indicates the length of time that this
- ## unique notice should be suppressed. This field is automatically
+ ## unique notice should be suppressed. This field is automatically
## filled out and should not be written to by any other script.
suppress_for: interval &log &optional;
};
-
+
## Ignored notice types.
const ignored_types: set[Notice::Type] = {} &redef;
## Emailed notice types.
@@ -135,10 +141,10 @@ export {
const alarmed_types: set[Notice::Type] = {} &redef;
## Types that should be suppressed for the default suppression interval.
const not_suppressed_types: set[Notice::Type] = {} &redef;
- ## This table can be used as a shorthand way to modify suppression
+ ## This table can be used as a shorthand way to modify suppression
## intervals for entire notice types.
const type_suppression_intervals: table[Notice::Type] of interval = {} &redef;
-
+
## This is the record that defines the items that make up the notice policy.
type PolicyItem: record {
## This is the exact positional order in which the :bro:type:`PolicyItem`
@@ -149,20 +155,20 @@ export {
priority: count &log &default=5;
## An action given to the notice if the predicate return true.
action: Notice::Action &log &default=ACTION_NONE;
- ## The pred (predicate) field is a function that returns a boolean T
- ## or F value. If the predicate function return true, the action in
- ## this record is applied to the notice that is given as an argument
- ## to the predicate function. If no predicate is supplied, it's
+ ## The pred (predicate) field is a function that returns a boolean T
+ ## or F value. If the predicate function return true, the action in
+ ## this record is applied to the notice that is given as an argument
+ ## to the predicate function. If no predicate is supplied, it's
## assumed that the PolicyItem always applies.
pred: function(n: Notice::Info): bool &log &optional;
- ## Indicates this item should terminate policy processing if the
+ ## Indicates this item should terminate policy processing if the
## predicate returns T.
halt: bool &log &default=F;
## This defines the length of time that this particular notice should
## be supressed.
suppress_for: interval &log &optional;
};
-
+
## This is the where the :bro:id:`Notice::policy` is defined. All notice
## processing is done through this variable.
const policy: set[PolicyItem] = {
@@ -177,66 +183,68 @@ export {
[$pred(n: Notice::Info) = { return (n$note in Notice::emailed_types); },
$action = ACTION_EMAIL,
$priority = 8],
- [$pred(n: Notice::Info) = {
- if (n$note in Notice::type_suppression_intervals)
+ [$pred(n: Notice::Info) = {
+ if (n$note in Notice::type_suppression_intervals)
{
n$suppress_for=Notice::type_suppression_intervals[n$note];
return T;
}
- return F;
+ return F;
},
$action = ACTION_NONE,
$priority = 8],
[$action = ACTION_LOG,
$priority = 0],
} &redef;
-
+
## Local system sendmail program.
const sendmail = "/usr/sbin/sendmail" &redef;
## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action
## or to send bulk alarm logs on rotation with :bro:enum:`ACTION_ALARM`.
const mail_dest = "" &redef;
-
+
## Address that emails will be from.
const mail_from = "Big Brother " &redef;
## Reply-to address used in outbound email.
const reply_to = "" &redef;
## Text string prefixed to the subject of all emails sent out.
const mail_subject_prefix = "[Bro]" &redef;
+ ## The maximum amount of time a plugin can delay email from being sent.
+ const max_email_delay = 15secs &redef;
## A log postprocessing function that implements emailing the contents
## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`.
## The rotated log is removed upon being sent.
global log_mailing_postprocessor: function(info: Log::RotationInfo): bool;
- ## This is the event that is called as the entry point to the
- ## notice framework by the global :bro:id:`NOTICE` function. By the time
+ ## This is the event that is called as the entry point to the
+ ## notice framework by the global :bro:id:`NOTICE` function. By the time
## this event is generated, default values have already been filled out in
- ## the :bro:type:`Notice::Info` record and synchronous functions in the
+ ## the :bro:type:`Notice::Info` record and synchronous functions in the
## :bro:id:`Notice:sync_functions` have already been called. The notice
## policy has also been applied.
global notice: event(n: Info);
- ## This is a set of functions that provide a synchronous way for scripts
+ ## This is a set of functions that provide a synchronous way for scripts
## extending the notice framework to run before the normal event based
## notice pathway that most of the notice framework takes. This is helpful
## in cases where an action against a notice needs to happen immediately
## and can't wait the short time for the event to bubble up to the top of
- ## the event queue. An example is the IP address dropping script that
- ## can block IP addresses that have notices generated because it
+ ## the event queue. An example is the IP address dropping script that
+ ## can block IP addresses that have notices generated because it
## needs to operate closer to real time than the event queue allows it to.
- ## Normally the event based extension model using the
+ ## Normally the event based extension model using the
## :bro:id:`Notice::notice` event will work fine if there aren't harder
## real time constraints.
const sync_functions: set[function(n: Notice::Info)] = set() &redef;
-
+
## This event is generated when a notice begins to be suppressed.
global begin_suppression: event(n: Notice::Info);
## This event is generated on each occurence of an event being suppressed.
global suppressed: event(n: Notice::Info);
## This event is generated when a notice stops being suppressed.
global end_suppression: event(n: Notice::Info);
-
+
## Call this function to send a notice in an email. It is already used
## by default with the built in :bro:enum:`ACTION_EMAIL` and
## :bro:enum:`ACTION_PAGE` actions.
@@ -248,12 +256,12 @@ export {
## dest: recipient string to use for the mail
## Returns: a string of mail headers to which an email body can be appended
global email_headers: function(subject_desc: string, dest: string): string;
-
+
## This event can be handled to access the :bro:type:`Info`
## record as it is sent on to the logging framework.
global log_notice: event(rec: Info);
-
- ## This is an internal wrapper for the global NOTICE function. Please
+
+ ## This is an internal wrapper for the global NOTICE function. Please
## disregard.
global internal_NOTICE: function(n: Notice::Info);
}
@@ -264,22 +272,22 @@ function per_notice_suppression_interval(t: table[Notice::Type, string] of Notic
local n: Notice::Type;
local s: string;
[n,s] = idx;
-
+
local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts);
if ( suppress_time < 0secs )
suppress_time = 0secs;
-
+
# If there is no more suppression time left, the notice needs to be sent
# to the end_suppression event.
if ( suppress_time == 0secs )
event Notice::end_suppression(t[n,s]);
-
+
return suppress_time;
}
-# This is the internally maintained notice suppression table. It's
+# This is the internally maintained notice suppression table. It's
# indexed on the Notice::Type and the $identifier field from the notice.
-global suppressing: table[Type, string] of Notice::Info = {}
+global suppressing: table[Type, string] of Notice::Info = {}
&create_expire=0secs
&expire_func=per_notice_suppression_interval;
@@ -306,7 +314,7 @@ function log_mailing_postprocessor(info: Log::RotationInfo): bool
event bro_init() &priority=5
{
Log::create_stream(Notice::LOG, [$columns=Info, $ev=log_notice]);
-
+
Log::create_stream(Notice::ALARM_LOG, [$columns=Notice::Info]);
# If Bro is configured for mailing notices, set up mailing for alarms.
# Make sure that this alarm log is also output as text so that it can
@@ -347,25 +355,49 @@ function email_headers(subject_desc: string, dest: string): string
return header_text;
}
+event delay_sending_email(n: Notice::Info, dest: string, extend: bool)
+ {
+ email_notice_to(n, dest, extend);
+ }
+
function email_notice_to(n: Notice::Info, dest: string, extend: bool)
{
if ( reading_traces() || dest == "" )
return;
-
+
+ if ( extend )
+ {
+ if ( |n$email_delay_tokens| > 0 )
+ {
+ # If we still are within the max_email_delay, keep delaying.
+ if ( n$ts + max_email_delay > network_time() )
+ {
+ schedule 1sec { delay_sending_email(n, dest, extend) };
+ return;
+ }
+ else
+ {
+ event reporter_info(network_time(),
+ fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens),
+ "");
+ }
+ }
+ }
+
local email_text = email_headers(fmt("%s", n$note), dest);
-
+
# First off, finish the headers and include the human readable messages
# then leave a blank line after the message.
email_text = string_cat(email_text, "\nMessage: ", n$msg);
if ( n?$sub )
email_text = string_cat(email_text, "\nSub-message: ", n$sub);
-
+
email_text = string_cat(email_text, "\n\n");
-
+
# Next, add information about the connection if it exists.
if ( n?$id )
{
- email_text = string_cat(email_text, "Connection: ",
+ email_text = string_cat(email_text, "Connection: ",
fmt("%s", n$id$orig_h), ":", fmt("%d", n$id$orig_p), " -> ",
fmt("%s", n$id$resp_h), ":", fmt("%d", n$id$resp_p), "\n");
if ( n?$uid )
@@ -373,17 +405,18 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)
}
else if ( n?$src )
email_text = string_cat(email_text, "Address: ", fmt("%s", n$src), "\n");
-
+
# Add the extended information if it's requested.
if ( extend )
{
+ email_text = string_cat(email_text, "\nEmail Extensions\n");
+ email_text = string_cat(email_text, "----------------\n");
for ( i in n$email_body_sections )
{
- email_text = string_cat(email_text, "******************\n");
email_text = string_cat(email_text, n$email_body_sections[i], "\n");
}
}
-
+
email_text = string_cat(email_text, "\n\n--\n[Automatically generated]\n\n");
piped_exec(fmt("%s -t -oi", sendmail), email_text);
}
@@ -396,10 +429,10 @@ event notice(n: Notice::Info) &priority=-5
Log::write(Notice::LOG, n);
if ( ACTION_ALARM in n$actions )
Log::write(Notice::ALARM_LOG, n);
-
+
# Normally suppress further notices like this one unless directed not to.
# n$identifier *must* be specified for suppression to function at all.
- if ( n?$identifier &&
+ if ( n?$identifier &&
ACTION_NO_SUPPRESS !in n$actions &&
[n$note, n$identifier] !in suppressing &&
n$suppress_for != 0secs )
@@ -408,8 +441,8 @@ event notice(n: Notice::Info) &priority=-5
event Notice::begin_suppression(n);
}
}
-
-## This determines if a notice is being suppressed. It is only used
+
+## This determines if a notice is being suppressed. It is only used
## internally as part of the mechanics for the global NOTICE function.
function is_being_suppressed(n: Notice::Info): bool
{
@@ -421,7 +454,7 @@ function is_being_suppressed(n: Notice::Info): bool
else
return F;
}
-
+
# Executes a script with all of the notice fields put into the
# new process' environment as "BRO_ARG_" variables.
function execute_with_notice(cmd: string, n: Notice::Info)
@@ -430,9 +463,9 @@ function execute_with_notice(cmd: string, n: Notice::Info)
#local tgs = tags(n);
#system_env(cmd, tags);
}
-
-# This is run synchronously as a function before all of the other
-# notice related functions and events. It also modifies the
+
+# This is run synchronously as a function before all of the other
+# notice related functions and events. It also modifies the
# :bro:type:`Notice::Info` record in place.
function apply_policy(n: Notice::Info)
{
@@ -447,7 +480,7 @@ function apply_policy(n: Notice::Info)
if ( ! n?$uid )
n$uid = n$conn$uid;
}
-
+
if ( n?$id )
{
if ( ! n?$src )
@@ -469,15 +502,20 @@ function apply_policy(n: Notice::Info)
if ( ! n?$src_peer )
n$src_peer = get_event_peer();
if ( ! n?$peer_descr )
- n$peer_descr = n$src_peer?$descr ?
+ n$peer_descr = n$src_peer?$descr ?
n$src_peer$descr : fmt("%s", n$src_peer$host);
-
+
if ( ! n?$actions )
n$actions = set();
-
+
+ if ( ! n?$email_body_sections )
+ n$email_body_sections = vector();
+ if ( ! n?$email_delay_tokens )
+ n$email_delay_tokens = set();
+
if ( ! n?$policy_items )
n$policy_items = set();
-
+
for ( i in ordered_policy )
{
# If there's no predicate or the predicate returns F.
@@ -485,51 +523,51 @@ function apply_policy(n: Notice::Info)
{
add n$actions[ordered_policy[i]$action];
add n$policy_items[int_to_count(i)];
-
- # If the predicate matched and there was a suppression interval,
+
+ # If the predicate matched and there was a suppression interval,
# apply it to the notice now.
if ( ordered_policy[i]?$suppress_for )
n$suppress_for = ordered_policy[i]$suppress_for;
-
+
# If the policy item wants to halt policy processing, do it now!
if ( ordered_policy[i]$halt )
break;
}
}
-
+
# Apply the suppression time after applying the policy so that policy
- # items can give custom suppression intervals. If there is no
+ # items can give custom suppression intervals. If there is no
# suppression interval given yet, the default is applied.
if ( ! n?$suppress_for )
n$suppress_for = default_suppression_interval;
-
+
# Delete the connection record if it's there so we aren't sending that
- # to remote machines. It can cause problems due to the size of the
+ # to remote machines. It can cause problems due to the size of the
# connection record.
if ( n?$conn )
delete n$conn;
if ( n?$iconn )
delete n$iconn;
}
-
-# Create the ordered notice policy automatically which will be used at runtime
+
+# Create the ordered notice policy automatically which will be used at runtime
# for prioritized matching of the notice policy.
event bro_init() &priority=10
{
# Create the policy log here because it's only written to in this handler.
Log::create_stream(Notice::POLICY_LOG, [$columns=PolicyItem]);
-
+
local tmp: table[count] of set[PolicyItem] = table();
for ( pi in policy )
{
if ( pi$priority < 0 || pi$priority > 10 )
Reporter::fatal("All Notice::PolicyItem priorities must be within 0 and 10");
-
+
if ( pi$priority !in tmp )
tmp[pi$priority] = set();
add tmp[pi$priority][pi];
}
-
+
local rev_count = vector(10,9,8,7,6,5,4,3,2,1,0);
for ( i in rev_count )
{
@@ -545,7 +583,7 @@ event bro_init() &priority=10
}
}
}
-
+
function internal_NOTICE(n: Notice::Info)
{
# Suppress this notice if necessary.
diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro
index c8c56bdc00..b8cfc7b44e 100644
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@@ -4,50 +4,50 @@ module DNS;
export {
redef enum Log::ID += { LOG };
-
+
type Info: record {
- ts: time &log;
- uid: string &log;
- id: conn_id &log;
- proto: transport_proto &log;
- trans_id: count &log &optional;
- query: string &log &optional;
- qclass: count &log &optional;
- qclass_name: string &log &optional;
- qtype: count &log &optional;
- qtype_name: string &log &optional;
- rcode: count &log &optional;
- rcode_name: string &log &optional;
- QR: bool &log &default=F;
- AA: bool &log &default=F;
- TC: bool &log &default=F;
- RD: bool &log &default=F;
- RA: bool &log &default=F;
- Z: count &log &default=0;
- TTL: interval &log &optional;
- answers: set[string] &log &optional;
-
+ ts: time &log;
+ uid: string &log;
+ id: conn_id &log;
+ proto: transport_proto &log;
+ trans_id: count &log &optional;
+ query: string &log &optional;
+ qclass: count &log &optional;
+ qclass_name: string &log &optional;
+ qtype: count &log &optional;
+ qtype_name: string &log &optional;
+ rcode: count &log &optional;
+ rcode_name: string &log &optional;
+ QR: bool &log &default=F;
+ AA: bool &log &default=F;
+ TC: bool &log &default=F;
+ RD: bool &log &default=F;
+ RA: bool &log &default=F;
+ Z: count &log &default=0;
+ answers: vector of string &log &optional;
+ TTLs: vector of interval &log &optional;
+
## This value indicates if this request/response pair is ready to be logged.
ready: bool &default=F;
total_answers: count &optional;
total_replies: count &optional;
};
-
+
type State: record {
## Indexed by query id, returns Info record corresponding to
## query/response which haven't completed yet.
pending: table[count] of Info &optional;
-
+
## This is the list of DNS responses that have completed based on the
## number of responses declared and the number received. The contents
## of the set are transaction IDs.
finished_answers: set[count] &optional;
};
-
+
global log_dns: event(rec: Info);
-
+
## This is called by the specific dns_*_reply events with a "reply" which
- ## may not represent the full data available from the resource record, but
+ ## may not represent the full data available from the resource record, but
## it's generally considered a summarization of the response(s).
global do_reply: event(c: connection, msg: dns_msg, ans: dns_answer, reply: string);
}
@@ -58,11 +58,11 @@ redef record connection += {
};
# DPD configuration.
-redef capture_filters += {
+redef capture_filters += {
["dns"] = "port 53",
["mdns"] = "udp and port 5353",
["llmns"] = "udp and port 5355",
- ["netbios-ns"] = "udp port 137",
+ ["netbios-ns"] = "udp port 137",
};
const dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
@@ -89,7 +89,7 @@ function new_session(c: connection, trans_id: count): Info
state$finished_answers=set();
c$dns_state = state;
}
-
+
local info: Info;
info$ts = network_time();
info$id = c$id;
@@ -102,23 +102,29 @@ function new_session(c: connection, trans_id: count): Info
function set_session(c: connection, msg: dns_msg, is_query: bool)
{
if ( ! c?$dns_state || msg$id !in c$dns_state$pending )
+ {
c$dns_state$pending[msg$id] = new_session(c, msg$id);
-
+ # Try deleting this transaction id from the set of finished answers.
+ # Sometimes hosts will reuse ports and transaction ids and this should
+ # be considered to be a legit scenario (although bad practice).
+ delete c$dns_state$finished_answers[msg$id];
+ }
+
c$dns = c$dns_state$pending[msg$id];
c$dns$rcode = msg$rcode;
c$dns$rcode_name = base_errors[msg$rcode];
-
+
if ( ! is_query )
{
if ( ! c$dns?$total_answers )
c$dns$total_answers = msg$num_answers;
-
- if ( c$dns?$total_replies &&
+
+ if ( c$dns?$total_replies &&
c$dns$total_replies != msg$num_answers + msg$num_addl + msg$num_auth )
{
- event conn_weird("dns_changed_number_of_responses", c,
- fmt("The declared number of responses changed from %d to %d",
+ event conn_weird("dns_changed_number_of_responses", c,
+ fmt("The declared number of responses changed from %d to %d",
c$dns$total_replies,
msg$num_answers + msg$num_addl + msg$num_auth));
}
@@ -129,27 +135,30 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
}
}
}
-
+
event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
{
set_session(c, msg, F);
- c$dns$AA = msg$AA;
- c$dns$RA = msg$RA;
- c$dns$TTL = ans$TTL;
-
if ( ans$answer_type == DNS_ANS )
{
+ c$dns$AA = msg$AA;
+ c$dns$RA = msg$RA;
+
if ( msg$id in c$dns_state$finished_answers )
event conn_weird("dns_reply_seen_after_done", c, "");
-
+
if ( reply != "" )
{
if ( ! c$dns?$answers )
- c$dns$answers = set();
- add c$dns$answers[reply];
+ c$dns$answers = vector();
+ c$dns$answers[|c$dns$answers|] = reply;
+
+ if ( ! c$dns?$TTLs )
+ c$dns$TTLs = vector();
+ c$dns$TTLs[|c$dns$TTLs|] = ans$TTL;
}
-
+
if ( c$dns?$answers && |c$dns$answers| == c$dns$total_answers )
{
add c$dns_state$finished_answers[c$dns$trans_id];
@@ -158,13 +167,12 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
}
}
}
-
+
event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=-5
{
if ( c$dns$ready )
{
Log::write(DNS::LOG, c$dns);
- add c$dns_state$finished_answers[c$dns$trans_id];
# This record is logged and no longer pending.
delete c$dns_state$pending[c$dns$trans_id];
}
@@ -173,41 +181,41 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
{
set_session(c, msg, T);
-
+
c$dns$RD = msg$RD;
c$dns$TC = msg$TC;
c$dns$qclass = qclass;
c$dns$qclass_name = classes[qclass];
c$dns$qtype = qtype;
c$dns$qtype_name = query_types[qtype];
-
+
# Decode netbios name queries
- # Note: I'm ignoring the name type for now. Not sure if this should be
+ # Note: I'm ignoring the name type for now. Not sure if this should be
# worked into the query/response in some fashion.
if ( c$id$resp_p == 137/udp )
query = decode_netbios_name(query);
c$dns$query = query;
-
+
c$dns$Z = msg$Z;
}
-
+
event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
{
event DNS::do_reply(c, msg, ans, fmt("%s", a));
}
-
+
event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, str: string) &priority=5
{
event DNS::do_reply(c, msg, ans, str);
}
-
-event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr,
+
+event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr,
astr: string) &priority=5
{
# TODO: What should we do with astr?
event DNS::do_reply(c, msg, ans, fmt("%s", a));
}
-
+
event dns_NS_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
{
event DNS::do_reply(c, msg, ans, name);
@@ -223,12 +231,12 @@ event dns_MX_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string,
{
event DNS::do_reply(c, msg, ans, name);
}
-
+
event dns_PTR_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
{
event DNS::do_reply(c, msg, ans, name);
}
-
+
event dns_SOA_reply(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa) &priority=5
{
event DNS::do_reply(c, msg, ans, soa$mname);
@@ -238,7 +246,7 @@ event dns_WKS_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
{
event DNS::do_reply(c, msg, ans, "");
}
-
+
event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
{
event DNS::do_reply(c, msg, ans, "");
@@ -247,17 +255,17 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
# TODO: figure out how to handle these
#event dns_EDNS(c: connection, msg: dns_msg, ans: dns_answer)
# {
-#
+#
# }
#
#event dns_EDNS_addl(c: connection, msg: dns_msg, ans: dns_edns_additional)
# {
-#
+#
# }
#
#event dns_TSIG_addl(c: connection, msg: dns_msg, ans: dns_tsig_additional)
# {
-#
+#
# }
@@ -271,10 +279,10 @@ event connection_state_remove(c: connection) &priority=-5
{
if ( ! c?$dns_state )
return;
-
- # If Bro is expiring state, we should go ahead and log all unlogged
+
+ # If Bro is expiring state, we should go ahead and log all unlogged
# request/response pairs now.
for ( trans_id in c$dns_state$pending )
Log::write(DNS::LOG, c$dns_state$pending[trans_id]);
}
-
+
diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro
index 2026f9bfa2..9d8bc68fd5 100644
--- a/scripts/base/protocols/ssl/consts.bro
+++ b/scripts/base/protocols/ssl/consts.bro
@@ -13,6 +13,44 @@ export {
[TLSv11] = "TLSv11",
} &default="UNKNOWN";
+ const alert_levels: table[count] of string = {
+ [1] = "warning",
+ [2] = "fatal",
+ } &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+ const alert_descriptions: table[count] of string = {
+ [0] = "close_notify",
+ [10] = "unexpected_message",
+ [20] = "bad_record_mac",
+ [21] = "decryption_failed",
+ [22] = "record_overflow",
+ [30] = "decompression_failure",
+ [40] = "handshake_failure",
+ [41] = "no_certificate",
+ [42] = "bad_certificate",
+ [43] = "unsupported_certificate",
+ [44] = "certificate_revoked",
+ [45] = "certificate_expired",
+ [46] = "certificate_unknown",
+ [47] = "illegal_parameter",
+ [48] = "unknown_ca",
+ [49] = "access_denied",
+ [50] = "decode_error",
+ [51] = "decrypt_error",
+ [60] = "export_restriction",
+ [70] = "protocol_version",
+ [71] = "insufficient_security",
+ [80] = "internal_error",
+ [90] = "user_canceled",
+ [100] = "no_renegotiation",
+ [110] = "unsupported_extension",
+ [111] = "certificate_unobtainable",
+ [112] = "unrecognized_name",
+ [113] = "bad_certificate_status_response",
+ [114] = "bad_certificate_hash_value",
+ [115] = "unknown_psk_identity",
+ } &default=function(i: count):string { return fmt("unknown-%d", i); };
+
# http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml
const extensions: table[count] of string = {
[0] = "server_name",
@@ -526,8 +564,7 @@ export {
[30] = "akid issuer serial mismatch",
[31] = "keyusage no certsign",
[32] = "unable to get crl issuer",
- [33] = "unhandled critical extension"
-
+ [33] = "unhandled critical extension",
};
}
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index c3c04d3c93..4b2fa39696 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -16,32 +16,33 @@ export {
subject: string &log &optional;
not_valid_before: time &log &optional;
not_valid_after: time &log &optional;
-
+ last_alert: string &log &optional;
+
cert: string &optional;
cert_chain: vector of string &optional;
-
+
## This stores the analyzer id used for the analyzer instance attached
- ## to each connection. It is not used for logging since it's a
+ ## to each connection. It is not used for logging since it's a
## meaningless arbitrary number.
analyzer_id: count &optional;
};
-
+
## This is where the default root CA bundle is defined. By loading the
## mozilla-ca-list.bro script it will be set to Mozilla's root CA list.
const root_certs: table[string] of string = {} &redef;
-
- ## If true, detach the SSL analyzer from the connection to prevent
+
+ ## If true, detach the SSL analyzer from the connection to prevent
## continuing to process encrypted traffic. Helps with performance
## (especially with large file transfers).
const disable_analyzer_after_detection = T &redef;
-
+
## The openssl command line utility. If it's in the path the default
## value will work, otherwise a full path string can be supplied for the
## utility.
const openssl_util = "openssl" &redef;
-
+
global log_ssl: event(rec: Info);
-
+
const ports = {
443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
@@ -86,7 +87,7 @@ function set_session(c: connection)
if ( ! c?$ssl )
c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
}
-
+
function finish(c: connection)
{
Log::write(SSL::LOG, c$ssl);
@@ -98,29 +99,33 @@ function finish(c: connection)
event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5
{
set_session(c);
-
+
# Save the session_id if there is one set.
if ( session_id != /^\x00{32}$/ )
c$ssl$session_id = bytestring_to_hexstr(session_id);
}
-
+
event ssl_server_hello(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count) &priority=5
{
set_session(c);
-
+
c$ssl$version = version_strings[version];
c$ssl$cipher = cipher_desc[cipher];
}
-event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=5
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=5
{
set_session(c);
-
+
+ # We aren't doing anything with client certificates yet.
+ if ( is_orig )
+ return;
+
if ( chain_idx == 0 )
{
# Save the primary cert.
c$ssl$cert = der_cert;
-
+
# Also save other certificate information about the primary cert.
c$ssl$subject = cert$subject;
c$ssl$not_valid_before = cert$not_valid_before;
@@ -132,20 +137,27 @@ event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: co
c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
}
}
-
-event ssl_extension(c: connection, code: count, val: string) &priority=5
+
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
{
set_session(c);
-
- if ( extensions[code] == "server_name" )
+
+ if ( is_orig && extensions[code] == "server_name" )
c$ssl$server_name = sub_bytes(val, 6, |val|);
}
-
+
+event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5
+ {
+ set_session(c);
+
+ c$ssl$last_alert = alert_descriptions[desc];
+ }
+
event ssl_established(c: connection) &priority=5
{
set_session(c);
}
-
+
event ssl_established(c: connection) &priority=-5
{
finish(c);
@@ -163,4 +175,4 @@ event protocol_violation(c: connection, atype: count, aid: count,
{
if ( c?$ssl )
finish(c);
- }
\ No newline at end of file
+ }
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index 837baee117..3927ce811e 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -60,9 +60,9 @@ event bro_init() &priority=3
$notice_threshold=sqli_requests_threshold,
$break_interval=sqli_requests_interval,
$note=SQL_Injection_Attacker]);
- Metrics::add_filter(SQLI_VICTIM, [$log=F,
+ Metrics::add_filter(SQLI_VICTIM, [$log=F,
$notice_threshold=sqli_requests_threshold,
- $break_interval=sqli_requests_interval,
+ $break_interval=sqli_requests_interval,
$note=SQL_Injection_Victim]);
}
diff --git a/scripts/policy/protocols/ssl/cert-hash.bro b/scripts/policy/protocols/ssl/cert-hash.bro
index 80a937f670..1e47ccac2e 100644
--- a/scripts/policy/protocols/ssl/cert-hash.bro
+++ b/scripts/policy/protocols/ssl/cert-hash.bro
@@ -10,11 +10,11 @@ export {
};
}
-event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=4
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=4
{
# We aren't tracking client certificates yet and we are also only tracking
# the primary cert. Watch that this came from an SSL analyzed session too.
- if ( ! is_server || chain_idx != 0 || ! c?$ssl )
+ if ( is_orig || chain_idx != 0 || ! c?$ssl )
return;
c$ssl$cert_hash = md5_hash(der_cert);
diff --git a/scripts/policy/protocols/ssl/expiring-certs.bro b/scripts/policy/protocols/ssl/expiring-certs.bro
index 50480b3a09..0e4db56bc3 100644
--- a/scripts/policy/protocols/ssl/expiring-certs.bro
+++ b/scripts/policy/protocols/ssl/expiring-certs.bro
@@ -33,10 +33,11 @@ export {
const notify_when_cert_expiring_in = 30days &redef;
}
-event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=3
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3
{
# If this isn't the host cert or we aren't interested in the server, just return.
- if ( chain_idx != 0 ||
+ if ( is_orig ||
+ chain_idx != 0 ||
! c$ssl?$cert_hash ||
! addr_matches_host(c$id$resp_h, notify_certs_expiration) )
return;
diff --git a/scripts/policy/protocols/ssl/known-certs.bro b/scripts/policy/protocols/ssl/known-certs.bro
index 90f6ee6186..669432e4d9 100644
--- a/scripts/policy/protocols/ssl/known-certs.bro
+++ b/scripts/policy/protocols/ssl/known-certs.bro
@@ -44,10 +44,10 @@ event bro_init() &priority=5
Log::create_stream(Known::CERTS_LOG, [$columns=CertsInfo, $ev=log_known_certs]);
}
-event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=3
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3
{
# Make sure this is the server cert and we have a hash for it.
- if ( chain_idx != 0 || ! c$ssl?$cert_hash )
+ if ( is_orig || chain_idx != 0 || ! c$ssl?$cert_hash )
return;
local host = c$id$resp_h;
diff --git a/src/Desc.cc b/src/Desc.cc
index c70878de34..12b4a524eb 100644
--- a/src/Desc.cc
+++ b/src/Desc.cc
@@ -41,8 +41,7 @@ ODesc::ODesc(desc_type t, BroFile* arg_f)
do_flush = 1;
include_stats = 0;
indent_with_spaces = 0;
- escape = 0;
- escape_len = 0;
+ escape = false;
}
ODesc::~ODesc()
@@ -56,10 +55,9 @@ ODesc::~ODesc()
free(base);
}
-void ODesc::SetEscape(const char* arg_escape, int len)
+void ODesc::EnableEscaping()
{
- escape = arg_escape;
- escape_len = len;
+ escape = true;
}
void ODesc::PushIndent()
@@ -228,6 +226,25 @@ static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned
return 0;
}
+pair ODesc::FirstEscapeLoc(const char* bytes, size_t n)
+ {
+ pair p(find_first_unprintable(this, bytes, n), 1);
+
+ string str(bytes, n);
+ list::const_iterator it;
+ for ( it = escape_sequences.begin(); it != escape_sequences.end(); ++it )
+ {
+ size_t pos = str.find(*it);
+ if ( pos != string::npos && (p.first == 0 || bytes + pos < p.first) )
+ {
+ p.first = bytes + pos;
+ p.second = it->size();
+ }
+ }
+
+ return p;
+ }
+
void ODesc::AddBytes(const void* bytes, unsigned int n)
{
if ( ! escape )
@@ -241,45 +258,30 @@ void ODesc::AddBytes(const void* bytes, unsigned int n)
while ( s < e )
{
- const char* t1 = (const char*) memchr(s, escape[0], e - s);
-
- if ( ! t1 )
- t1 = e;
-
- const char* t2 = find_first_unprintable(this, s, t1 - s);
-
- if ( t2 && t2 < t1 )
+ pair p = FirstEscapeLoc(s, e - s);
+ if ( p.first )
{
- AddBytesRaw(s, t2 - s);
-
- char hex[6] = "\\x00";
- hex[2] = hex_chars[((*t2) & 0xf0) >> 4];
- hex[3] = hex_chars[(*t2) & 0x0f];
- AddBytesRaw(hex, 4);
-
- s = t2 + 1;
- continue;
+ AddBytesRaw(s, p.first - s);
+ if ( p.second == 1 )
+ {
+ char hex[6] = "\\x00";
+ hex[2] = hex_chars[((*p.first) & 0xf0) >> 4];
+ hex[3] = hex_chars[(*p.first) & 0x0f];
+ AddBytesRaw(hex, 4);
+ }
+ else
+ {
+ string esc_str = get_escaped_string(string(p.first, p.second), true);
+ AddBytesRaw(esc_str.c_str(), esc_str.size());
+ }
+ s = p.first + p.second;
}
-
- if ( memcmp(t1, escape, escape_len) != 0 )
- break;
-
- AddBytesRaw(s, t1 - s);
-
- for ( int i = 0; i < escape_len; ++i )
+ else
{
- char hex[5] = "\\x00";
- hex[2] = hex_chars[((*t1) & 0xf0) >> 4];
- hex[3] = hex_chars[(*t1) & 0x0f];
- AddBytesRaw(hex, 4);
- ++t1;
+ AddBytesRaw(s, e - s);
+ break;
}
-
- s = t1;
}
-
- if ( s < e )
- AddBytesRaw(s, e - s);
}
void ODesc::AddBytesRaw(const void* bytes, unsigned int n)
diff --git a/src/Desc.h b/src/Desc.h
index 4ed05c1763..27cbd4fa01 100644
--- a/src/Desc.h
+++ b/src/Desc.h
@@ -4,6 +4,8 @@
#define descriptor_h
#include
+#include
+#include
#include "BroString.h"
typedef enum {
@@ -48,8 +50,13 @@ public:
void SetFlush(int arg_do_flush) { do_flush = arg_do_flush; }
- // The string passed in must remain valid as long as this object lives.
- void SetEscape(const char* escape, int len);
+ void EnableEscaping();
+ void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); }
+ void AddEscapeSequence(const char* s, size_t n)
+ { escape_sequences.push_back(string(s, n)); }
+ void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); }
+ void RemoveEscapeSequence(const char* s, size_t n)
+ { escape_sequences.remove(string(s, n)); }
void PushIndent();
void PopIndent();
@@ -133,6 +140,19 @@ protected:
void OutOfMemory();
+ /**
+ * Returns the location of the first place in the bytes to be hex-escaped.
+ *
+ * @param bytes the starting memory address to start searching for
+ * escapable character.
+ * @param n the maximum number of bytes to search.
+ * @return a pair whose first element represents a starting memory address
+ * to be escaped up to the number of characters indicated by the
+ * second element. The first element may be 0 if nothing is
+ * to be escaped.
+ */
+ pair FirstEscapeLoc(const char* bytes, size_t n);
+
desc_type type;
desc_style style;
@@ -140,8 +160,8 @@ protected:
unsigned int offset; // where we are in the buffer
unsigned int size; // size of buffer in bytes
- int escape_len; // number of bytes in to escape sequence
- const char* escape; // bytes to escape on output
+ bool escape; // escape unprintable characters in output?
+ list escape_sequences; // additional sequences of chars to escape
BroFile* f; // or the file we're using.
diff --git a/src/Expr.cc b/src/Expr.cc
index 5eadaad631..c34c44a7d1 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -359,7 +359,7 @@ bool NameExpr::DoUnserialize(UnserialInfo* info)
if ( id )
::Ref(id);
else
- reporter->Warning("persistent state config changed: unserialized unknown global name");
+ reporter->Warning("configuration changed: unserialized unknown global name from persistent state");
delete [] name;
}
@@ -4053,7 +4053,15 @@ Val* RecordCoerceExpr::Fold(Val* v) const
val->Assign(i, rhs);
}
else
- val->Assign(i, 0);
+ {
+ const Attr* def =
+ Type()->AsRecordType()->FieldDecl(i)->FindAttr(ATTR_DEFAULT);
+
+ if ( def )
+ val->Assign(i, def->AttrExpr()->Eval(0));
+ else
+ val->Assign(i, 0);
+ }
}
return val;
diff --git a/src/LogMgr.cc b/src/LogMgr.cc
index 0b706f6417..28e9a2ac1f 100644
--- a/src/LogMgr.cc
+++ b/src/LogMgr.cc
@@ -81,16 +81,18 @@ struct LogMgr::Stream {
bool LogField::Read(SerializationFormat* fmt)
{
int t;
+ int st;
- bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type"));
+ bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") );
type = (TypeTag) t;
+ subtype = (TypeTag) st;
return success;
}
bool LogField::Write(SerializationFormat* fmt) const
{
- return (fmt->Write(name, "name") && fmt->Write((int)type, "type"));
+ return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype"));
}
LogVal::~LogVal()
@@ -707,6 +709,14 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
LogField* field = new LogField();
field->name = new_path;
field->type = t->Tag();
+ if ( field->type == TYPE_TABLE )
+ {
+ field->subtype = t->AsSetType()->Indices()->PureType()->Tag();
+ }
+ else if ( field->type == TYPE_VECTOR )
+ {
+ field->subtype = t->AsVectorType()->YieldType()->Tag();
+ }
filter->fields[filter->num_fields - 1] = field;
}
diff --git a/src/LogMgr.h b/src/LogMgr.h
index 10530960cb..3eaba360d5 100644
--- a/src/LogMgr.h
+++ b/src/LogMgr.h
@@ -15,10 +15,12 @@ class SerializationFormat;
struct LogField {
string name;
TypeTag type;
+ // inner type of sets
+ TypeTag subtype;
- LogField() { }
+ LogField() { subtype = TYPE_VOID; }
LogField(const LogField& other)
- : name(other.name), type(other.type) { }
+ : name(other.name), type(other.type), subtype(other.subtype) { }
// (Un-)serialize.
bool Read(SerializationFormat* fmt);
diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc
index 9b1fda3b62..d2c1d91370 100644
--- a/src/LogWriterAscii.cc
+++ b/src/LogWriterAscii.cc
@@ -6,27 +6,6 @@
#include "LogWriterAscii.h"
#include "NetVar.h"
-/**
- * Takes a string, escapes each character into its equivalent hex code (\x##), and
- * returns a string containing all escaped values.
- *
- * @param str string to escape
- * @return A std::string containing a list of escaped hex values of the form \x##
- */
-static string get_escaped_string(const std::string& str)
-{
- char tbuf[16];
- string esc = "";
-
- for ( size_t i = 0; i < str.length(); ++i )
- {
- snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]);
- esc += tbuf;
- }
-
- return esc;
-}
-
LogWriterAscii::LogWriterAscii()
{
file = 0;
@@ -59,7 +38,8 @@ LogWriterAscii::LogWriterAscii()
memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(),
header_prefix_len);
- desc.SetEscape(separator, separator_len);
+ desc.EnableEscaping();
+ desc.AddEscapeSequence(separator, separator_len);
}
LogWriterAscii::~LogWriterAscii()
@@ -102,13 +82,19 @@ bool LogWriterAscii::DoInit(string path, int num_fields,
{
string str = string(header_prefix, header_prefix_len)
+ "separator " // Always use space as separator here.
- + get_escaped_string(string(separator, separator_len))
+ + get_escaped_string(string(separator, separator_len), false)
+ "\n";
if( fwrite(str.c_str(), str.length(), 1, file) != 1 )
goto write_error;
- if ( ! WriteHeaderField("path", path) )
+ if ( ! (WriteHeaderField("set_separator", get_escaped_string(
+ string(set_separator, set_separator_len), false)) &&
+ WriteHeaderField("empty_field", get_escaped_string(
+ string(empty_field, empty_field_len), false)) &&
+ WriteHeaderField("unset_field", get_escaped_string(
+ string(unset_field, unset_field_len), false)) &&
+ WriteHeaderField("path", get_escaped_string(path, false))) )
goto write_error;
string names;
@@ -125,6 +111,12 @@ bool LogWriterAscii::DoInit(string path, int num_fields,
const LogField* field = fields[i];
names += field->name;
types += type_name(field->type);
+ if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) )
+ {
+ types += "[";
+ types += type_name(field->subtype);
+ types += "]";
+ }
}
if ( ! (WriteHeaderField("fields", names)
@@ -238,14 +230,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
break;
}
+ desc->AddEscapeSequence(set_separator, set_separator_len);
for ( int j = 0; j < val->val.set_val.size; j++ )
{
if ( j > 0 )
- desc->AddN(set_separator, set_separator_len);
+ desc->AddRaw(set_separator, set_separator_len);
if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) )
+ {
+ desc->RemoveEscapeSequence(set_separator, set_separator_len);
return false;
+ }
}
+ desc->RemoveEscapeSequence(set_separator, set_separator_len);
break;
}
@@ -258,14 +255,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
break;
}
+ desc->AddEscapeSequence(set_separator, set_separator_len);
for ( int j = 0; j < val->val.vector_val.size; j++ )
{
if ( j > 0 )
- desc->AddN(set_separator, set_separator_len);
+ desc->AddRaw(set_separator, set_separator_len);
if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) )
+ {
+ desc->RemoveEscapeSequence(set_separator, set_separator_len);
return false;
+ }
}
+ desc->RemoveEscapeSequence(set_separator, set_separator_len);
break;
}
diff --git a/src/event.bif b/src/event.bif
index 0c2f7eb780..3f0e2992ed 100644
--- a/src/event.bif
+++ b/src/event.bif
@@ -279,13 +279,13 @@ event ssh_server_version%(c: connection, version: string%);
event ssl_client_hello%(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set%);
event ssl_server_hello%(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count%);
-event ssl_extension%(c: connection, code: count, val: string%);
+event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%);
+event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%);
event ssl_established%(c: connection%);
-event ssl_alert%(c: connection, level: count, desc: count%);
-event x509_certificate%(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string%);
-event x509_extension%(c: connection, data: string%);
-event x509_error%(c: connection, err: count%);
+event x509_certificate%(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string%);
+event x509_extension%(c: connection, is_orig: bool, data: string%);
+event x509_error%(c: connection, is_orig: bool, err: count%);
event stp_create_endp%(c: connection, e: int, is_orig: bool%);
event stp_resume_endp%(e: int%);
diff --git a/src/ssl-analyzer.pac b/src/ssl-analyzer.pac
index 6471d9c4a4..79e00f2033 100644
--- a/src/ssl-analyzer.pac
+++ b/src/ssl-analyzer.pac
@@ -22,11 +22,17 @@
}
};
+ string orig_label(bool is_orig);
void free_X509(void *);
X509* d2i_X509_binpac(X509** px, const uint8** in, int len);
%}
%code{
+string orig_label(bool is_orig)
+ {
+ return string(is_orig ? "originator" :"responder");
+ }
+
void free_X509(void* cert)
{
X509_free((X509*) cert);
@@ -117,14 +123,14 @@ refine connection SSL_Conn += {
function proc_alert(rec: SSLRecord, level : int, desc : int) : bool
%{
BifEvent::generate_ssl_alert(bro_analyzer(), bro_analyzer()->Conn(),
- level, desc);
+ ${rec.is_orig}, level, desc);
return true;
%}
function proc_client_hello(rec: SSLRecord,
version : uint16, ts : double,
session_id : uint8[],
- cipher_suites16 : uint16[],
+ cipher_suites16 : uint16[],
cipher_suites24 : uint24[]) : bool
%{
if ( state_ == STATE_TRACK_LOST )
@@ -150,15 +156,15 @@ refine connection SSL_Conn += {
cipher_set->Assign(ciph, 0);
Unref(ciph);
}
-
+
BifEvent::generate_ssl_client_hello(bro_analyzer(), bro_analyzer()->Conn(),
version, ts,
to_string_val(session_id),
cipher_set);
-
+
delete cipher_suites;
}
-
+
return true;
%}
@@ -187,24 +193,24 @@ refine connection SSL_Conn += {
std::copy(cipher_suites16->begin(), cipher_suites16->end(), std::back_inserter(*ciphers));
else
std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*ciphers), to_int());
-
+
BifEvent::generate_ssl_server_hello(bro_analyzer(),
bro_analyzer()->Conn(),
version, ts,
to_string_val(session_id),
ciphers->size()==0 ? 0 : ciphers->at(0), comp_method);
-
+
delete ciphers;
}
-
+
return true;
%}
- function proc_ssl_extension(type: int, data: bytestring) : bool
+ function proc_ssl_extension(rec: SSLRecord, type: int, data: bytestring) : bool
%{
if ( ssl_extension )
BifEvent::generate_ssl_extension(bro_analyzer(),
- bro_analyzer()->Conn(), type,
+ bro_analyzer()->Conn(), ${rec.is_orig}, type,
new StringVal(data.length(), (const char*) data.data()));
return true;
%}
@@ -222,7 +228,7 @@ refine connection SSL_Conn += {
if ( x509_certificate )
{
STACK_OF(X509)* untrusted_certs = 0;
-
+
for ( unsigned int i = 0; i < certificates->size(); ++i )
{
const bytestring& cert = (*certificates)[i];
@@ -231,7 +237,7 @@ refine connection SSL_Conn += {
if ( ! pTemp )
{
BifEvent::generate_x509_error(bro_analyzer(), bro_analyzer()->Conn(),
- ERR_get_error());
+ ${rec.is_orig}, ERR_get_error());
return false;
}
@@ -257,8 +263,8 @@ refine connection SSL_Conn += {
StringVal* der_cert = new StringVal(cert.length(), (const char*) cert.data());
BifEvent::generate_x509_certificate(bro_analyzer(), bro_analyzer()->Conn(),
+ ${rec.is_orig},
pX509Cert,
- ! ${rec.is_orig},
i, certificates->size(),
der_cert);
@@ -284,7 +290,7 @@ refine connection SSL_Conn += {
StringVal* value = new StringVal(length, (char*)pBuffer);
BifEvent::generate_x509_extension(bro_analyzer(),
- bro_analyzer()->Conn(), value);
+ bro_analyzer()->Conn(), ${rec.is_orig}, value);
OPENSSL_free(pBuffer);
}
}
@@ -445,5 +451,5 @@ refine typeattr CiphertextRecord += &let {
}
refine typeattr SSLExtension += &let {
- proc : bool = $context.connection.proc_ssl_extension(type, data);
+ proc : bool = $context.connection.proc_ssl_extension(rec, type, data);
};
diff --git a/src/ssl-protocol.pac b/src/ssl-protocol.pac
index f60d73b27e..24207ac78b 100644
--- a/src/ssl-protocol.pac
+++ b/src/ssl-protocol.pac
@@ -22,7 +22,6 @@ type uint24 = record {
};
string state_label(int state_nr);
- string orig_label(bool is_orig);
double get_time_from_asn1(const ASN1_TIME * atime);
string handshake_type_label(int type);
%}
@@ -35,7 +34,7 @@ type SSLRecord(is_orig: bool) = record {
head2 : uint8;
head3 : uint8;
head4 : uint8;
- rec : RecordText(this, is_orig)[] &length=length, &requires(content_type);
+ rec : RecordText(this)[] &length=length, &requires(content_type);
} &length = length+5, &byteorder=bigendian,
&let {
version : int =
@@ -54,25 +53,25 @@ type SSLRecord(is_orig: bool) = record {
};
};
-type RecordText(rec: SSLRecord, is_orig: bool) = case $context.connection.state() of {
+type RecordText(rec: SSLRecord) = case $context.connection.state() of {
STATE_ABBREV_SERVER_ENCRYPTED, STATE_CLIENT_ENCRYPTED,
STATE_COMM_ENCRYPTED, STATE_CONN_ESTABLISHED
- -> ciphertext : CiphertextRecord(rec, is_orig);
+ -> ciphertext : CiphertextRecord(rec);
default
- -> plaintext : PlaintextRecord(rec, is_orig);
+ -> plaintext : PlaintextRecord(rec);
};
-type PossibleEncryptedHandshake(rec: SSLRecord, is_orig: bool) = case $context.connection.state() of {
+type PossibleEncryptedHandshake(rec: SSLRecord) = case $context.connection.state() of {
# Deal with encrypted handshakes before the server cipher spec change.
STATE_CLIENT_FINISHED, STATE_CLIENT_ENCRYPTED
- -> ct : CiphertextRecord(rec, is_orig);
+ -> ct : CiphertextRecord(rec);
default -> hs : Handshake(rec);
};
-type PlaintextRecord(rec: SSLRecord, is_orig: bool) = case rec.content_type of {
+type PlaintextRecord(rec: SSLRecord) = case rec.content_type of {
CHANGE_CIPHER_SPEC -> ch_cipher : ChangeCipherSpec(rec);
ALERT -> alert : Alert(rec);
- HANDSHAKE -> handshake : PossibleEncryptedHandshake(rec, is_orig);
+ HANDSHAKE -> handshake : PossibleEncryptedHandshake(rec);
APPLICATION_DATA -> app_data : ApplicationData(rec);
V2_ERROR -> v2_error : V2Error(rec);
V2_CLIENT_HELLO -> v2_client_hello : V2ClientHello(rec);
@@ -81,7 +80,7 @@ type PlaintextRecord(rec: SSLRecord, is_orig: bool) = case rec.content_type of {
default -> unknown_record : UnknownRecord(rec);
};
-type SSLExtension = record {
+type SSLExtension(rec: SSLRecord) = record {
type: uint16;
data_len: uint16;
data: bytestring &length=data_len;
@@ -156,10 +155,6 @@ enum AnalyzerState {
}
}
- string orig_label(bool is_orig)
- {
- return string(is_orig ? "originator" :"responder");
- }
double get_time_from_asn1(const ASN1_TIME * atime)
{
@@ -389,7 +384,7 @@ type ClientHello(rec: SSLRecord) = record {
# This weirdness is to deal with the possible existence or absence
# of the following fields.
ext_len: uint16[] &until($element == 0 || $element != 0);
- extensions : SSLExtension[] &until($input.length() == 0);
+ extensions : SSLExtension(rec)[] &until($input.length() == 0);
} &let {
state_changed : bool =
$context.connection.transition(STATE_INITIAL,
@@ -663,7 +658,7 @@ type UnknownRecord(rec: SSLRecord) = record {
state_changed : bool = $context.connection.lost_track();
};
-type CiphertextRecord(rec: SSLRecord, is_orig: bool) = record {
+type CiphertextRecord(rec: SSLRecord) = record {
cont : bytestring &restofdata &transient;
} &let {
state_changed : bool =
diff --git a/src/util.cc b/src/util.cc
index f81eff8f22..171756fc1c 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -41,6 +41,37 @@
#include "Net.h"
#include "Reporter.h"
+/**
+ * Takes a string, escapes characters into equivalent hex codes (\x##), and
+ * returns a string containing all escaped values.
+ *
+ * @param str string to escape
+ * @param escape_all If true, all characters are escaped. If false, only
+ * characters are escaped that are either whitespace or not printable in
+ * ASCII.
+ * @return A std::string containing a list of escaped hex values of the form
+ * \x## */
+std::string get_escaped_string(const std::string& str, bool escape_all)
+{
+ char tbuf[16];
+ string esc = "";
+
+ for ( size_t i = 0; i < str.length(); ++i )
+ {
+ char c = str[i];
+
+ if ( escape_all || isspace(c) || ! isascii(c) || ! isprint(c) )
+ {
+ snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]);
+ esc += tbuf;
+ }
+ else
+ esc += c;
+ }
+
+ return esc;
+}
+
char* copy_string(const char* s)
{
char* c = new char[strlen(s)+1];
diff --git a/src/util.h b/src/util.h
index 6e76b0f61f..498bdf00e4 100644
--- a/src/util.h
+++ b/src/util.h
@@ -89,6 +89,8 @@ void delete_each(T* t)
delete *it;
}
+std::string get_escaped_string(const std::string& str, bool escape_all);
+
extern char* copy_string(const char* s);
extern int streq(const char* s1, const char* s2);
diff --git a/testing/btest/Baseline/bifs.records_fields/out b/testing/btest/Baseline/bifs.records_fields/out
index b221230fc0..0d52e64255 100644
--- a/testing/btest/Baseline/bifs.records_fields/out
+++ b/testing/btest/Baseline/bifs.records_fields/out
@@ -1,6 +1,6 @@
-[a=42, b=, c=, d=Bar]
+[a=42, b=Foo, c=, d=Bar]
{
-[b] = [type_name=record, log=F, value=, default_val=Foo],
+[b] = [type_name=record, log=F, value=Foo, default_val=Foo],
[d] = [type_name=record, log=T, value=Bar, default_val=],
[c] = [type_name=record, log=F, value=, default_val=],
[a] = [type_name=record, log=F, value=42, default_val=]
diff --git a/testing/btest/Baseline/core.expr-exception/reporter.log b/testing/btest/Baseline/core.expr-exception/reporter.log
index 2dfe6b7b8e..3767de37d8 100644
--- a/testing/btest/Baseline/core.expr-exception/reporter.log
+++ b/testing/btest/Baseline/core.expr-exception/reporter.log
@@ -1,13 +1,16 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path reporter
#fields ts level message location
#types time enum string string
-1300475168.783842 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.915940 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.916118 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.918295 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.952193 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.952228 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.954761 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.962628 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475169.780331 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.783842 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.915940 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.916118 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.918295 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.952193 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.952228 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.954761 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.962628 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475169.780331 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log
index 3736748484..5ce968d5e6 100644
--- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log
+++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path conn
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
#types time string addr port addr port enum string interval count count string bool count string count count count count
diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output
index 4f6230b768..d7ff523927 100644
--- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output
+++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output
@@ -1,20 +1,32 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path packet_filter
#fields ts node filter init success
#types time string string bool bool
-1320367155.152502 - not ip6 T T
+1324314285.981347 - not ip6 T T
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path packet_filter
#fields ts node filter init success
#types time string string bool bool
-1320367155.379066 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T
+1324314286.168294 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path packet_filter
#fields ts node filter init success
#types time string string bool bool
-1320367155.601980 - port 42 T T
+1324314286.350780 - port 42 T T
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path packet_filter
#fields ts node filter init success
#types time string string bool bool
-1320367155.826539 - port 56730 T T
+1324314286.530768 - port 56730 T T
diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output
index bfb2880ed4..3d8aa6ff54 100644
--- a/testing/btest/Baseline/core.reporter-error-in-handler/output
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/output
@@ -1,2 +1,2 @@
-error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
1st error printed on script level
diff --git a/testing/btest/Baseline/core.reporter-fmt-strings/output b/testing/btest/Baseline/core.reporter-fmt-strings/output
index 10a883cb5d..4842dd9fc5 100644
--- a/testing/btest/Baseline/core.reporter-fmt-strings/output
+++ b/testing/btest/Baseline/core.reporter-fmt-strings/output
@@ -1 +1 @@
-error in /Users/jsiwek/tmp/bro/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))
+error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))
diff --git a/testing/btest/Baseline/core.reporter-parse-error/output b/testing/btest/Baseline/core.reporter-parse-error/output
index ca0bc9304b..7606fe5667 100644
--- a/testing/btest/Baseline/core.reporter-parse-error/output
+++ b/testing/btest/Baseline/core.reporter-parse-error/output
@@ -1 +1 @@
-error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"
+error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"
diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output
index 5c0feedf42..3a96954101 100644
--- a/testing/btest/Baseline/core.reporter-runtime-error/output
+++ b/testing/btest/Baseline/core.reporter-runtime-error/output
@@ -1 +1 @@
-error in /Users/seth/bro.git9/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
diff --git a/testing/btest/Baseline/core.reporter-type-mismatch/output b/testing/btest/Baseline/core.reporter-type-mismatch/output
index 6211752225..4c038ea8c5 100644
--- a/testing/btest/Baseline/core.reporter-type-mismatch/output
+++ b/testing/btest/Baseline/core.reporter-type-mismatch/output
@@ -1,3 +1,3 @@
-error in string and /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
-error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
-error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))
+error in string and /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
+error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
+error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))
diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log
index 6f7ba1d8c7..bc2abd142a 100644
--- a/testing/btest/Baseline/core.reporter/logger-test.log
+++ b/testing/btest/Baseline/core.reporter/logger-test.log
@@ -1,6 +1,6 @@
-reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
-reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
-reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
-reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
-reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
-reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
+reporter_info|init test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
+reporter_warning|init test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
+reporter_error|init test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
+reporter_info|done test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
+reporter_warning|done test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
+reporter_error|done test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output
index 2735adc931..185cabb1eb 100644
--- a/testing/btest/Baseline/core.reporter/output
+++ b/testing/btest/Baseline/core.reporter/output
@@ -1,3 +1,3 @@
-/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
-warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
+warning in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
+error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
diff --git a/testing/btest/Baseline/core.vlan-mpls/conn.log b/testing/btest/Baseline/core.vlan-mpls/conn.log
index 69e23f3875..f3c958ea99 100644
--- a/testing/btest/Baseline/core.vlan-mpls/conn.log
+++ b/testing/btest/Baseline/core.vlan-mpls/conn.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path conn
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
#types time string addr port addr port enum string interval count count string bool count string count count count count
diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index 6819dc0813..8fab67304e 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path loaded_scripts
#fields name
#types string
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 7a461a3903..3f77797df8 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path loaded_scripts
#fields name
#types string
diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst
index 1902e2754b..46cc4a8227 100644
--- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst
+++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst
@@ -219,8 +219,7 @@ Events
Summarize "an_event" here.
Give more details about "an_event" here.
- Example::an_event should not be confused as a parameter
-
+ Example::an_event should not be confused as a parameter.
:param name: describe the argument here
diff --git a/testing/btest/Baseline/istate.broccoli/bro.log b/testing/btest/Baseline/istate.broccoli/bro.log
index eeebe944ef..4fbbfc81ae 100644
--- a/testing/btest/Baseline/istate.broccoli/bro.log
+++ b/testing/btest/Baseline/istate.broccoli/bro.log
@@ -1,3 +1,3 @@
-ping received, seq 0, 1303093042.542125 at src, 1303093042.583423 at dest,
-ping received, seq 1, 1303093043.543167 at src, 1303093043.544026 at dest,
-ping received, seq 2, 1303093044.544115 at src, 1303093044.545008 at dest,
+ping received, seq 0, 1324314397.698781 at src, 1324314397.699240 at dest,
+ping received, seq 1, 1324314398.698905 at src, 1324314398.699094 at dest,
+ping received, seq 2, 1324314399.699012 at src, 1324314399.699231 at dest,
diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log
index 06d453c241..1601f8ad3c 100644
--- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log
+++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1319568535.914761 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log
index 06d453c241..1601f8ad3c 100644
--- a/testing/btest/Baseline/istate.events-ssl/sender.http.log
+++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1319568535.914761 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log
index d85d560b6d..25a7f289c0 100644
--- a/testing/btest/Baseline/istate.events/receiver.http.log
+++ b/testing/btest/Baseline/istate.events/receiver.http.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1319568558.542142 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log
index d85d560b6d..25a7f289c0 100644
--- a/testing/btest/Baseline/istate.events/sender.http.log
+++ b/testing/btest/Baseline/istate.events/sender.http.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1319568558.542142 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
diff --git a/testing/btest/Baseline/language.record-default-coercion/out b/testing/btest/Baseline/language.record-default-coercion/out
new file mode 100644
index 0000000000..2f0e6cd17d
--- /dev/null
+++ b/testing/btest/Baseline/language.record-default-coercion/out
@@ -0,0 +1,4 @@
+[a=13, c=13, v=[]]
+0
+[a=13, c=13, v=[test]]
+1
diff --git a/testing/btest/Baseline/language.wrong-delete-field/output b/testing/btest/Baseline/language.wrong-delete-field/output
index f8271e43c2..c2aae8aae3 100644
--- a/testing/btest/Baseline/language.wrong-delete-field/output
+++ b/testing/btest/Baseline/language.wrong-delete-field/output
@@ -1 +1 @@
-error in /da/home/robin/bro/seth/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)
+error in /Users/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
index 7f71757ca0..e5dfb59592 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
@@ -1,16 +1,19 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path communication
#fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message
#types time string string string addr port string string
-1322788789.351248 bro parent - - - info [#1/127.0.0.1:47757] added peer
-1322788789.354851 bro child - - - info [#1/127.0.0.1:47757] connected
-1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] peer connected
-1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] phase: version
-1322788789.355429 bro script - - - info connection established
-1322788789.355429 bro script - - - info requesting events matching /^?(NOTHING)$?/
-1322788789.355429 bro script - - - info accepting state
-1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake
-1322788789.355967 bro parent - - - info warning: no events to request
-1322788789.355967 bro parent - - - info terminating...
-1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro
-1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] closing connection
+1324314302.411344 bro parent - - - info [#1/127.0.0.1:47757] added peer
+1324314302.414978 bro child - - - info [#1/127.0.0.1:47757] connected
+1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] peer connected
+1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] phase: version
+1324314302.417446 bro script - - - info connection established
+1324314302.417446 bro script - - - info requesting events matching /^?(NOTHING)$?/
+1324314302.417446 bro script - - - info accepting state
+1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake
+1324314302.418003 bro parent - - - info warning: no events to request
+1324314302.418003 bro parent - - - info terminating...
+1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro
+1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] closing connection
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
index fc2c133dc6..485bfe3eba 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
@@ -1,6 +1,9 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh-new-default
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167052.603186 1.2.3.4 1234 2.3.4.5 80 success unknown
-1315167052.603186 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314313.140603 1.2.3.4 1234 2.3.4.5 80 success unknown
+1324314313.140603 1.2.3.4 1234 2.3.4.5 80 failure US
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
index b236cb818b..144a7a6426 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
@@ -1,4 +1,7 @@
-#separator \x7c
+#separator |
+#set_separator|,
+#empty_field|(empty)
+#unset_field|-
#path|ssh
#fields|data|data2
#types|string|string
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log
index e1ba48cf8e..10275205a5 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log
@@ -1,9 +1,12 @@
-PREFIX<>separator \x7c
+PREFIX<>separator |
+PREFIX<>set_separator|,
+PREFIX<>empty_field|EMPTY
+PREFIX<>unset_field|NOT-SET
PREFIX<>path|ssh
PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
PREFIX<>types|time|addr|port|addr|port|string|string|bool
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T
+1324314313.345323|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
+1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
+1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
+1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
+1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
index 683fed60f2..c9e69994fc 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields x y z
#types string string string
-\x2d - -
+\x2d - (empty)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
index db9ce497ed..97744b7df8 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - - - - - text/html - -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html - -
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
new file mode 100644
index 0000000000..b88627c806
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
@@ -0,0 +1,8 @@
+#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
+#path test
+#fields ss
+#types table[string]
+CC,AA,\x2c,\x2c\x2c
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
index 3100fa0cb2..0ef81128d3 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
@@ -1,9 +1,12 @@
-#separator \x7c\x7c
+#separator ||
+#set_separator||,
+#empty_field||(empty)
+#unset_field||-
#path||ssh
#fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country
#types||time||addr||port||addr||port||string||string
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||success||unknown
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||US
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||MX
+1324314313.899736||1.2.3.4||1234||2.3.4.5||80||success||unknown
+1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||US
+1324314313.899736||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
+1324314313.899736||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
+1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log
index 33a922cc2b..f66dec7160 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log
@@ -1,5 +1,5 @@
-1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|unknown
-1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|US
-1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|UK
-1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|BR
-1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|MX
+1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|unknown
+1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|US
+1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|UK
+1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|BR
+1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
index 7f512c15d9..00ab6c8ca0 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields data
#types time
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
index c2c32c5c6a..5acaa7b2fc 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields status country a1 b1 b2
#types string string count count count
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
index 18e4d5cbad..086a4836fe 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields status country
#types string string
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
index 49272bfd53..16ba17c62c 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
@@ -1,9 +1,12 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.369918 1.2.3.4 1234 2.3.4.5 80 success unknown
-1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure US
-1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure UK
-1315167053.369918 1.2.3.4 1234 2.3.4.5 80 success BR
-1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure MX
+1324314314.443785 1.2.3.4 1234 2.3.4.5 80 success unknown
+1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314314.443785 1.2.3.4 1234 2.3.4.5 80 success BR
+1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output
index c3dbf607a6..5da27764a5 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output
@@ -1,2 +1,2 @@
-[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=]
-[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US]
+[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=unknown]
+[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
index b078b4746a..4ccf4c836a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields id.orig_p id.resp_h id.resp_p status country
#types port addr port string string
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
index 0a988ff9b9..4aa3d8f0a7 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields t f
#types time file
-1315167053.585834 Foo.log
+1324314314.940195 Foo.log
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
index 5675ef6632..00242d65c1 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
@@ -1,9 +1,12 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields t id.orig_h
#types time addr
-1315167053.694473 1.2.3.4
-1315167053.694473 1.2.3.4
-1315167053.694473 1.2.3.4
-1315167053.694473 1.2.3.4
-1315167053.694473 1.2.3.4
+1324314315.040480 1.2.3.4
+1324314315.040480 1.2.3.4
+1324314315.040480 1.2.3.4
+1324314315.040480 1.2.3.4
+1324314315.040480 1.2.3.4
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
index d8d90cf1fa..e2b3da6efd 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path local
#fields ts id.orig_h
#types time addr
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
index a17c2821f5..1ac18ff5f7 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path remote
#fields ts id.orig_h
#types time addr
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
index 2c196340cc..a6b8a4e090 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
@@ -6,37 +6,58 @@ static-prefix-1-US.log
static-prefix-2-MX2.log
static-prefix-2-UK.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path static-prefix-0-BR
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.803346 1.2.3.4 1234 2.3.4.5 80 success BR
+1324314315.385189 1.2.3.4 1234 2.3.4.5 80 success BR
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path static-prefix-0-MX3
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX3
+1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX3
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path static-prefix-0-unknown
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.803346 1.2.3.4 1234 2.3.4.5 80 success unknown
+1324314315.385189 1.2.3.4 1234 2.3.4.5 80 success unknown
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path static-prefix-1-MX
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX
+1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path static-prefix-1-US
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure US
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path static-prefix-2-MX2
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX2
+1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX2
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path static-prefix-2-UK
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure UK
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
index ba688d7843..733bb02847 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test.failure
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.923545 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314315.498365 1.2.3.4 1234 2.3.4.5 80 failure US
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
index 7a91b1a2d9..0261caeb06 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test.success
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167053.923545 1.2.3.4 1234 2.3.4.5 80 success -
+1324314315.498365 1.2.3.4 1234 2.3.4.5 80 success unknown
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
index c00e7765d5..d9bd34309a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field EMPTY
+#unset_field -
#path test
#fields b i e c p sn a d t iv s sc ss se vc ve
-#types bool int enum count port subnet addr double time interval string table table table vector vector
-T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315167054.320958 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY
+#types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string]
+T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324314315.880694 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
index aba9fdddd9..6cb58bf4ac 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test.failure
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
index b928c37685..f5b79ee2c4 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
@@ -1,9 +1,12 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success -
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success unknown
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success BR
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
index a951c6ed1a..c40e56af93 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
@@ -1,6 +1,9 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test.success
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success -
-1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success unknown
+1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success BR
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
index 6185e86028..cb3d4aafb8 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
@@ -1,6 +1,9 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh.failure
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure US
-1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure UK
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
index a4ec2dc7de..38a5bb660c 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure US
-1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure UK
-1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure BR
+1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure BR
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
index 337ed3ca32..915915f43e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
@@ -18,11 +18,14 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.00.05.log, pat
custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, path=test2, open=1299499195.0, close=1299499205.0, terminating=F]
custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F]
custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T]
+#empty_field (empty)
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#path test
#path test2
#separator \x09
+#set_separator ,
#types time addr port addr port
+#unset_field -
1299466805.000000 10.0.0.1 20 10.0.0.2 1024
1299470395.000000 10.0.0.2 20 10.0.0.3 0
1299470405.000000 10.0.0.1 20 10.0.0.2 1025
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
index 74ce45023a..d31783edc4 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
@@ -10,6 +10,9 @@ test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
> test.2011-03-07-03-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -17,6 +20,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299470395.000000 10.0.0.2 20 10.0.0.3 0
> test.2011-03-07-04-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -24,6 +30,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299473995.000000 10.0.0.2 20 10.0.0.3 1
> test.2011-03-07-05-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -31,6 +40,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299477595.000000 10.0.0.2 20 10.0.0.3 2
> test.2011-03-07-06-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -38,6 +50,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299481195.000000 10.0.0.2 20 10.0.0.3 3
> test.2011-03-07-07-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -45,6 +60,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299484795.000000 10.0.0.2 20 10.0.0.3 4
> test.2011-03-07-08-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -52,6 +70,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299488395.000000 10.0.0.2 20 10.0.0.3 5
> test.2011-03-07-09-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -59,6 +80,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299491995.000000 10.0.0.2 20 10.0.0.3 6
> test.2011-03-07-10-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -66,6 +90,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299495595.000000 10.0.0.2 20 10.0.0.3 7
> test.2011-03-07-11-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
@@ -73,6 +100,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
1299499195.000000 10.0.0.2 20 10.0.0.3 8
> test.2011-03-07-12-00-05.log
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p
#types time addr port addr port
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
index 84521cb645..09afe2031c 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
@@ -1,9 +1,12 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path /dev/stdout
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167067.393739 1.2.3.4 1234 2.3.4.5 80 success unknown
-1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure US
-1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure UK
-1315167067.393739 1.2.3.4 1234 2.3.4.5 80 success BR
-1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure MX
+1324314328.844271 1.2.3.4 1234 2.3.4.5 80 success unknown
+1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314328.844271 1.2.3.4 1234 2.3.4.5 80 success BR
+1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
index 5b93b6e23b..53292324af 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
@@ -1,9 +1,12 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
-1315167067.507542 1.2.3.4 1234 2.3.4.5 80 success unknown
-1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure US
-1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure UK
-1315167067.507542 1.2.3.4 1234 2.3.4.5 80 success BR
-1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure MX
+1324314328.950525 1.2.3.4 1234 2.3.4.5 80 success unknown
+1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure US
+1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure UK
+1324314328.950525 1.2.3.4 1234 2.3.4.5 80 success BR
+1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
index ffd579c224..74aa0312a1 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field EMPTY
+#unset_field -
#path ssh
#fields b i e c p sn a d t iv s sc ss se vc ve f
-#types bool int enum count port subnet addr double time interval string table table table vector vector func
-T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+#types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] func
+T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324314329.051618 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
index 12bb1d1704..7956ad11a0 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path testing
#fields a.val1 a.val2 b
#types count count count
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
index b9a54404ed..65ab5592bf 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path ssh
#fields vec
-#types vector
+#types vector[string]
-,2,-,-,5
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
index 1677297ecc..a278bdc56a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path metrics
#fields ts metric_id filter_name index.host index.str index.network value
#types time enum string addr string subnet count
-1317950616.401733 TEST_METRIC foo-bar 6.5.4.3 - - 4
-1317950616.401733 TEST_METRIC foo-bar 1.2.3.4 - - 6
-1317950616.401733 TEST_METRIC foo-bar 7.2.1.5 - - 2
+1324314335.570789 TEST_METRIC foo-bar 6.5.4.3 - - 4
+1324314335.570789 TEST_METRIC foo-bar 1.2.3.4 - - 6
+1324314335.570789 TEST_METRIC foo-bar 7.2.1.5 - - 2
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
index 45334cf3d7..8ee19c255b 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path metrics
#fields ts metric_id filter_name index.host index.str index.network value
#types time enum string addr string subnet count
-1315167083.455574 TEST_METRIC foo-bar 6.5.4.3 - - 2
-1315167083.455574 TEST_METRIC foo-bar 1.2.3.4 - - 3
-1315167083.455574 TEST_METRIC foo-bar 7.2.1.5 - - 1
+1324314344.807073 TEST_METRIC foo-bar 6.5.4.3 - - 2
+1324314344.807073 TEST_METRIC foo-bar 1.2.3.4 - - 3
+1324314344.807073 TEST_METRIC foo-bar 7.2.1.5 - - 1
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
index f5df2e96f3..33f55ce608 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path notice
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network
-#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet
-1316952194.679491 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - -
+#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet
+1324314350.184962 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - -
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
index 33745500e0..437b1465a1 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
@@ -1,6 +1,9 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path notice
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network
-#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet
-1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - -
-1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 6.5.4.3 - -
+#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet
+1324314359.357148 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - -
+1324314359.357148 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - -
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
index 0662c13294..fb1e1b3d47 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path notice
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network
-#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet
-1316952264.931290 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - -
+#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet
+1324314363.721823 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - -
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
index 6e0214b7d3..9e6e1b1916 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path notice
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network
-#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet
-1316950574.408256 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - -
+#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet
+1324314378.560010 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - -
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
index 6b4c925e0f..d134c97049 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path notice
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude
-#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double
-1316950497.513136 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 - - - - - -
+#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double
+1324314387.663586 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - -
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
index 812b4bc151..ddcea2e9c7 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - - - - - text/html - -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - -
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
index 386eaf8901..cec098a50b 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
index 9515eb8168..d4e5679da1 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
@@ -1,9 +1,12 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file
-1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - FAKE_MIME - -
-1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - FAKE_MIME - -
-1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - FAKE_MIME - -
-1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - - - - - image/png e0029eea80812e9a8e57b8d05d52938a -
-1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - - - - - image/png 30aa926344f58019d047e85ba049ca1e -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
+1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - FAKE_MIME - -
+1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - FAKE_MIME - -
+1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - FAKE_MIME - -
+1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a -
+1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e -
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
index 01d62b3981..dfaf34acbf 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
@@ -1,9 +1,12 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path http
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file
-#types time string addr port addr port count string string string string string count count count string count string string table string string table string file
-1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - - -
-1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - - -
-1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - - -
-1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - - - - - - -
-1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - - - - - - -
+#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string file
+1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - -
+1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - -
+1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - -
+1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - -
+1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - -
diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
index d224556632..39ff897fae 100644
--- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
+++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
@@ -1,8 +1,11 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path irc
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size extraction_file
-#types time string addr port addr port string string table string string string table string count file
-1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - -
-1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - -
-1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - -
-1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies - - ladyvampress-default(2011-07-07)-OS.zip 42208 -
+#types time string addr port addr port string string table[string] string string string table[enum] string count file
+1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - -
+1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - -
+1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - -
+1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 -
diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
index a692d2dd4d..342923ba7b 100644
--- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
+++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
@@ -1,8 +1,11 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path irc
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size dcc_mime_type extraction_file
-#types time string addr port addr port string string table string string string table string count string file
-1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - - -
-1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - -
-1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - - -
-1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies - IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
+#types time string addr port addr port string string table[string] string string string table[enum] string count string file
+1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - -
+1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - -
+1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - -
+1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
index b93720cfe6..2c1380cb44 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path smtp
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent
-#types time string addr port addr port count string string table string string table string string string string addr string string string vector string
+#types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string
1254722768.219663 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
index 63b287a791..453b55932e 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path smtp_entities
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt
#types time string addr port addr port count string count string string file string
-1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat -
-1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - -
-1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat -
+1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat (empty)
+1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty)
+1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat (empty)
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
index e45d8dc757..2b471782d5 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path smtp_entities
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt
#types time string addr port addr port count string count string string file string
-1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - -
-1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - -
-1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - -
+1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - (empty)
+1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty)
+1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - (empty)
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
index cde5156594..0799292857 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path known_hosts
#fields ts host
#types time addr
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
index 008eb364ed..6fdba24d39 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path known_hosts
#fields ts host
#types time addr
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
index 43b28ded8a..9ef6ee47b7 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
@@ -1,4 +1,7 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path known_hosts
#fields ts host
#types time addr
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
index ad9fa52e1c..d53da6f693 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path known_services
#fields ts host port_num port_proto service
-#types time addr port enum table
+#types time addr port enum table[string]
1308930691.049431 172.16.238.131 22 tcp SSH
1308930694.550308 172.16.238.131 80 tcp HTTP
1308930716.462556 74.125.225.81 80 tcp HTTP
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
index 1607d69f24..ef1722d6a1 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
@@ -1,7 +1,10 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path known_services
#fields ts host port_num port_proto service
-#types time addr port enum table
+#types time addr port enum table[string]
1308930691.049431 172.16.238.131 22 tcp SSH
1308930694.550308 172.16.238.131 80 tcp HTTP
1308930718.361665 172.16.238.131 21 tcp FTP
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
index 0d1210c941..3fc68cdb91 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
@@ -1,6 +1,9 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path known_services
#fields ts host port_num port_proto service
-#types time addr port enum table
+#types time addr port enum table[string]
1308930716.462556 74.125.225.81 80 tcp HTTP
1308930726.872485 141.142.192.39 22 tcp SSH
diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
index 945960e03e..9d80898e0f 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
@@ -1,5 +1,8 @@
#separator \x09
+#set_separator ,
+#empty_field (empty)
+#unset_field -
#path dns
-#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z TTL answers auth addl
-#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count interval table table table
-930613226.529070 UWkUyAuUGXf 212.180.42.100 25000 131.243.64.3 53 tcp 34798 - - - - - 0 NOERROR F F F F T 0 31337.000000 4.3.2.1 - -
+#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z answers TTLs auth addl
+#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count vector[string] vector[interval] table[string] table[string]
+930613226.529070 UWkUyAuUGXf 212.180.42.100 25000 131.243.64.3 53 tcp 34798 - - - - - 0 NOERROR F F F F T 0 4.3.2.1 31337.000000 - -
diff --git a/testing/btest/language/record-default-coercion.bro b/testing/btest/language/record-default-coercion.bro
new file mode 100644
index 0000000000..7e717c39e2
--- /dev/null
+++ b/testing/btest/language/record-default-coercion.bro
@@ -0,0 +1,18 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+type MyRecord: record {
+ a: count &default=13;
+ c: count;
+ v: vector of string &default=vector();
+};
+
+event bro_init()
+ {
+ local r: MyRecord = [$c=13];
+ print r;
+ print |r$v|;
+ r$v[|r$v|] = "test";
+ print r;
+ print |r$v|;
+ }
diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro
new file mode 100644
index 0000000000..f5fb7a6259
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro
@@ -0,0 +1,21 @@
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff test.log
+
+module Test;
+
+export {
+ redef enum Log::ID += { LOG };
+
+ type Log: record {
+ ss: set[string];
+ } &log;
+}
+
+event bro_init()
+{
+ Log::create_stream(Test::LOG, [$columns=Log]);
+
+
+ Log::write(Test::LOG, [$ss=set("AA", ",", ",,", "CC")]);
+}
+
|