Merge remote-tracking branch 'origin/topic/jsiwek/empty-lines'

* origin/topic/jsiwek/empty-lines:
  Add 'smtp_excessive_pending_cmds' weird
  Fix SMTP command string comparisons
  Improve handling of empty lines in several text protocol analyzers
  Add rate-limiting sampling mechanism for weird events
  Teach timestamp canonifier about timestamps before ~2001
This commit is contained in:
Jon Siwek 2018-08-20 15:35:16 -05:00
commit bcf97f70ea
31 changed files with 1078 additions and 15 deletions

View file

@ -0,0 +1,351 @@
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird
net_weird, my_net_weird
flow_weird, my_flow_weird
conn_weird, my_conn_weird
net_weird, whitelisted_net_weird
flow_weird, whitelisted_flow_weird
conn_weird, whitelisted_conn_weird

View file

@ -55,4 +55,5 @@ traceroute
tunnel
unified2
weird
weird_stats
x509

View file

@ -0,0 +1,13 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird_stats
#open 2018-07-26-23-11-27
#fields ts name num_seen
#types time string count
1532646687.827249 weird3 1
1532646687.827249 weird2 1000
1532646687.827249 weird1 2000
1532646692.877464 weird1 2
#close 2018-07-26-23-11-34

View file

@ -0,0 +1,12 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird_stats
#open 2018-07-27-00-20-35
#fields ts name num_seen
#types time string count
1532650834.978616 my_weird 1000
1532650840.011592 my_weird 2000
1532650845.043367 my_weird 10
#close 2018-07-27-00-20-47

View file

@ -0,0 +1,55 @@
# @TEST-EXEC: bro -b -r $TRACES/http/bro.org.pcap %INPUT >output
# @TEST-EXEC: btest-diff output
redef Weird::sampling_duration = 5sec;
redef Weird::sampling_threshold = 10;
redef Weird::sampling_rate = 10;
redef Weird::sampling_whitelist = set("whitelisted_net_weird",
"whitelisted_flow_weird",
"whitelisted_conn_weird");
event conn_weird(name: string, c: connection, addl: string)
{
print "conn_weird", name;
}
event flow_weird(name: string, src: addr, dst: addr)
{
print "flow_weird", name;
}
event net_weird(name: string)
{
print "net_weird", name;
}
event gen_weirds(c: connection)
{
local num = 30;
while ( num != 0 )
{
Reporter::net_weird("my_net_weird");
Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h);
Reporter::conn_weird("my_conn_weird", c);
Reporter::net_weird("whitelisted_net_weird");
Reporter::flow_weird("whitelisted_flow_weird", c$id$orig_h, c$id$resp_h);
Reporter::conn_weird("whitelisted_conn_weird", c);
--num;
}
}
global did_one_connection = F;
event new_connection(c: connection)
{
if ( did_one_connection )
return;
did_one_connection = T;
event gen_weirds(c); # should permit 10 + 2 of each "my" weird
schedule 2sec { gen_weirds(c) }; # should permit 3 of each "my" weird
schedule 7sec { gen_weirds(c) }; # should permit 10 + 2 of each "my" weird
# Total of 27 "my" weirds of each type and 90 of each "whitelisted" type
}

View file

@ -0,0 +1,93 @@
# @TEST-SERIALIZE: comm
#
# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
# @TEST-EXEC: btest-bg-wait 20
# @TEST-EXEC: btest-diff manager-1/weird_stats.log
@TEST-START-FILE cluster-layout.bro
redef Cluster::nodes = {
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp],
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
};
@TEST-END-FILE
@load misc/weird-stats
redef Cluster::retry_interval = 1sec;
redef Broker::default_listen_retry = 1sec;
redef Broker::default_connect_retry = 1sec;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
redef WeirdStats::weird_stat_interval = 5secs;
event terminate_me()
{
terminate();
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
terminate();
}
event ready_again()
{
Reporter::net_weird("weird1");
if ( Cluster::node == "worker-2" )
{
schedule 5secs { terminate_me() };
}
}
event ready_for_data()
{
local n = 0;
if ( Cluster::node == "worker-1" )
{
while ( n < 1000 )
{
Reporter::net_weird("weird1");
++n;
}
Reporter::net_weird("weird3");
}
else if ( Cluster::node == "worker-2" )
{
while ( n < 1000 )
{
Reporter::net_weird("weird1");
Reporter::net_weird("weird2");
++n;
}
}
schedule 5secs { ready_again() };
}
@if ( Cluster::local_node_type() == Cluster::MANAGER )
event bro_init()
{
Broker::auto_publish(Cluster::worker_topic, ready_for_data);
}
global peer_count = 0;
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
++peer_count;
if ( peer_count == 2 )
event ready_for_data();
}
@endif

View file

@ -0,0 +1,32 @@
# @TEST-EXEC: btest-bg-run bro bro %INPUT
# @TEST-EXEC: btest-bg-wait 20
# @TEST-EXEC: btest-diff bro/weird_stats.log
@load misc/weird-stats.bro
redef exit_only_after_terminate = T;
redef WeirdStats::weird_stat_interval = 5sec;
event die()
{
terminate();
}
event gen_weirds(n: count, done: bool &default = F)
{
while ( n != 0 )
{
Reporter::net_weird("my_weird");
--n;
}
if ( done )
schedule 5sec { die() };
}
event bro_init()
{
event gen_weirds(1000);
schedule 7.5sec { gen_weirds(2000) } ;
schedule 12.5sec { gen_weirds(10, T) } ;
}

View file

@ -9,5 +9,5 @@ else
sed="sed -E"
fi
$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
$sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'