From 33c85895b84ba23d234072cf248e128366eb37e0 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 12 Aug 2016 11:32:46 -0700 Subject: [PATCH] Make netcontrol cluster test stable. It now consistently works for me. --- .../manager-1.netcontrol.log | 32 +++++++++---------- .../manager-1.netcontrol_catch_release.log | 22 ++++++------- .../worker-2..stdout | 9 +++--- .../netcontrol/catch-and-release-cluster.bro | 22 +++++++------ 4 files changed, 44 insertions(+), 41 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol.log b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol.log index f89ae7d4d9..974349e229 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol.log +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol.log @@ -3,21 +3,21 @@ #empty_field (empty) #unset_field - #path netcontrol -#open 2016-08-05-17-46-57 +#open 2016-08-12-17-38-49 #fields ts rule_id category cmd state action target entity_type entity mod msg priority expire location plugin #types time string enum string enum string enum string string string string int interval string string -1470419217.355712 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All -1470419217.355712 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All -1470419217.355712 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - -1470419220.470685 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 - Debug-All -1470419220.470685 worker-1:2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 - Debug-All -1470419220.470685 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 - Debug-All -1470419220.470685 worker-1:2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 - Debug-All -1470419220.570873 worker-1:2 NetControl::RULE EXPIRE NetControl::TIMEOUT NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 - Debug-All -1470419220.570873 worker-1:2 NetControl::RULE REMOVE NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 - Debug-All -1470419220.572465 worker-1:2 NetControl::RULE REMOVE NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 - Debug-All -1470419221.963109 2 NetControl::RULE REMOVE NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - reason here 0 600.000000 - Debug-All -1470419221.963109 2 NetControl::RULE REMOVE NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 - Debug-All -1470419221.963109 4 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 3600.000000 Re-drop by catch-and-release: Debug-All -1470419221.963109 4 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 3600.000000 Re-drop by catch-and-release: Debug-All -#close 2016-08-05-17-47-02 +1471023529.752740 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All +1471023529.752740 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All +1471023529.752740 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - +1471023532.819263 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 connection drop worker-1 Debug-All +1471023532.819263 worker-1:2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All +1471023532.819263 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 connection drop worker-1 Debug-All +1471023532.819263 worker-1:2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All +1471023532.920126 worker-1:2 NetControl::RULE EXPIRE NetControl::TIMEOUT NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All +1471023532.920126 worker-1:2 NetControl::RULE REMOVE NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All +1471023532.921768 worker-1:2 NetControl::RULE REMOVE NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All +1471023534.308087 2 NetControl::RULE REMOVE NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - worker-2 0 600.000000 connection drop worker-1 Debug-All +1471023534.308087 2 NetControl::RULE REMOVE NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 connection drop worker-1 Debug-All +1471023534.308087 4 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 3600.000000 Re-drop by catch-and-release: direct cr worker-1 Debug-All +1471023534.308087 4 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 3600.000000 Re-drop by catch-and-release: direct cr worker-1 Debug-All +#close 2016-08-12-17-38-54 diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol_catch_release.log b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol_catch_release.log index a3e4751027..54202fffbe 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol_catch_release.log +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol_catch_release.log @@ -3,16 +3,16 @@ #empty_field (empty) #unset_field - #path netcontrol_catch_release -#open 2016-08-05-17-47-19 +#open 2016-08-12-17-38-52 #fields ts rule_id ip action block_interval watch_interval blocked_until watched_until num_blocked location message #types time string addr enum interval interval time time count string string -1470419239.093089 2 192.168.18.50 NetControl::DROP 600.000000 3600.000000 1470419839.093089 1470422839.093089 1 - - -1470419239.093089 2 192.168.18.50 NetControl::DROPPED 600.000000 3600.000000 1470419839.093089 1470422839.093089 1 - - -1470419239.093089 worker-1:2 8.8.8.8 NetControl::ADDED 600.000000 3600.000000 - 1470422839.093089 1 - Address already blocked outside of catch-and-release. Catch and release will monitor and only actively block if it appears in network traffic. -1470419239.193930 worker-1:2 8.8.8.8 NetControl::UNBLOCK 600.000000 3600.000000 - 1470422839.093089 1 - - -1470419240.599721 2 192.168.18.50 NetControl::INFO 600.000000 3600.000000 1470419839.093089 1470422839.093089 1 - Block seen while in rule_entities. No action taken. -1470419240.599721 2 192.168.18.50 NetControl::UNBLOCK 600.000000 3600.000000 1470419839.093089 1470422839.093089 1 - reason here -1470419240.599721 4 8.8.8.8 NetControl::SEEN_AGAIN 3600.000000 86400.000000 1470422840.599721 1470505640.599721 2 - - -1470419240.599721 4 8.8.8.8 NetControl::DROPPED 3600.000000 86400.000000 1470422840.599721 1470505640.599721 2 - - -1470419238.504810 2 192.168.18.50 NetControl::INFO 600.000000 3600.000000 1470419839.093089 1470422839.093089 1 - Already blocked using catch-and-release - ignoring duplicate -#close 2016-08-05-17-47-20 +1471023532.819263 2 192.168.18.50 NetControl::DROP 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 - +1471023532.819263 2 192.168.18.50 NetControl::DROPPED 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 - +1471023532.819263 worker-1:2 8.8.8.8 NetControl::ADDED 600.000000 3600.000000 - 1471027132.819263 1 direct cr worker-1 Address already blocked outside of catch-and-release. Catch and release will monitor and only actively block if it appears in network traffic. +1471023532.920126 worker-1:2 8.8.8.8 NetControl::UNBLOCK 600.000000 3600.000000 - 1471027132.819263 1 direct cr worker-1 - +1471023534.308087 2 192.168.18.50 NetControl::INFO 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 Block seen while in rule_entities. No action taken. +1471023534.308087 2 192.168.18.50 NetControl::UNBLOCK 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 worker-2 +1471023534.308087 4 8.8.8.8 NetControl::SEEN_AGAIN 3600.000000 86400.000000 1471027134.308087 1471109934.308087 2 direct cr worker-1 - +1471023534.308087 4 8.8.8.8 NetControl::DROPPED 3600.000000 86400.000000 1471027134.308087 1471109934.308087 2 direct cr worker-1 - +1471023532.239980 2 192.168.18.50 NetControl::INFO 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 Already blocked using catch-and-release - ignoring duplicate +#close 2016-08-12-17-38-54 diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/worker-2..stdout index 0e455cf6d4..f61fe92474 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/worker-2..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/worker-2..stdout @@ -1,8 +1,9 @@ Suspend, worker-2 -New block, 192.168.18.50, [block_until=1468427134.768038, watch_until=1468430134.768038, num_reblocked=0, current_interval=0, current_block_id=2, location=] -New block, 8.8.8.8, [block_until=, watch_until=1468430134.768038, num_reblocked=0, current_interval=0, current_block_id=worker-1:2, location=] +New block, 192.168.18.50, [block_until=1471027194.791177, watch_until=1471030194.791177, num_reblocked=0, current_interval=0, current_block_id=2, location=connection drop worker-1] +New block, 8.8.8.8, [block_until=, watch_until=1471030194.791177, num_reblocked=0, current_interval=0, current_block_id=worker-1:2, location=direct cr worker-1] Resume, worker-2 Connection established -Info, [block_until=1468427134.768038, watch_until=1468430134.768038, num_reblocked=0, current_interval=0, current_block_id=2, location=] +Info, [block_until=1471027194.791177, watch_until=1471030194.791177, num_reblocked=0, current_interval=0, current_block_id=2, location=connection drop worker-1] Delete block, 192.168.18.50 -New block, 8.8.8.8, [block_until=1468430136.256898, watch_until=1468512936.256898, num_reblocked=1, current_interval=1, current_block_id=4, location=] +New block, 8.8.8.8, [block_until=1471030196.295249, watch_until=1471112996.295249, num_reblocked=1, current_interval=1, current_block_id=4, location=direct cr worker-1] +remote connection closed diff --git a/testing/btest/scripts/base/frameworks/netcontrol/catch-and-release-cluster.bro b/testing/btest/scripts/base/frameworks/netcontrol/catch-and-release-cluster.bro index be1391a248..fd7de7e442 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/catch-and-release-cluster.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/catch-and-release-cluster.bro @@ -4,7 +4,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT" # @TEST-EXEC: btest-bg-run worker-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-2 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT" -# @TEST-EXEC: btest-bg-wait 15 +# @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v ^# | $SCRIPTS/diff-remove-timestamps' btest-diff manager-1/netcontrol.log # @TEST-EXEC: btest-diff manager-1/netcontrol_catch_release.log # @TEST-EXEC: btest-diff worker-2/.stdout @@ -17,6 +17,8 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef exit_only_after_terminate = T; + redef Log::default_rotation_interval = 0secs; @load base/frameworks/netcontrol @@ -32,6 +34,7 @@ global peer_count = 0; event remote_connection_handshake_done(p: event_peer) &priority=-5 { ++peer_count; + print "remote_connection_handshake_done", peer_count; if ( peer_count == 2 ) { event ready_for_data_1(); @@ -48,6 +51,10 @@ event bro_init() suspend_processing(); } +event remote_connection_closed(p: event_peer) { + print "remote connection closed"; + terminate(); +} @endif @if ( Cluster::node == "worker-1" ) @@ -80,18 +87,18 @@ event connection_established(c: connection) local id = c$id; local info = NetControl::get_catch_release_info(id$orig_h); print "Info", info; - NetControl::drop_address_catch_release(id$orig_h); + NetControl::drop_address_catch_release(id$orig_h, cat("connection drop ", Cluster::node)); if ( info$current_block_id != "" ) { - NetControl::unblock_address_catch_release(id$orig_h, "reason here"); + NetControl::unblock_address_catch_release(id$orig_h, Cluster::node); } } @if ( Cluster::node == "worker-1" ) event connection_established(c: connection) { - NetControl::drop_address(8.8.8.8, 0.1secs); - NetControl::drop_address_catch_release(8.8.8.8); + NetControl::drop_address(8.8.8.8, 0.1secs, cat("direct drop ", Cluster::node)); + NetControl::drop_address_catch_release(8.8.8.8, cat("direct cr ", Cluster::node)); } @endif @@ -113,14 +120,9 @@ event NetControl::catch_release_block_delete(a: addr) } event terminate_me() { - print "Terminate"; terminate(); } -event remote_connection_closed(p: event_peer) { - schedule 1sec { terminate_me() }; -} - @if ( Cluster::local_node_type() == Cluster::MANAGER ) event NetControl::rule_added(r: NetControl::Rule, p: NetControl::PluginState, msg: string) {