From 18d968adcd300425c2c540677d6e5a8c621467a3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 1 Dec 2011 11:52:01 -0800 Subject: [PATCH 1/6] Adapting attribute serialization when talking to Broccoli. Broccoli doesn't support expressions, and we now no longer send them when serializing attributes. This is the Bro change mentioned in #606. It's needs a correspondinly modified Broccoli identifying itself as such, and it isn't tested yet ... Addresses #606. --- src/Attr.cc | 6 +++++- src/RemoteSerializer.cc | 11 ++++++++++- src/RemoteSerializer.h | 1 + src/SerialInfo.h | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/Attr.cc b/src/Attr.cc index a5a350f452..12e1c888af 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -481,7 +481,11 @@ bool Attributes::DoSerialize(SerialInfo* info) const loop_over_list((*attrs), i) { Attr* a = (*attrs)[i]; - SERIALIZE_OPTIONAL(a->AttrExpr()) + + // Broccoli doesn't support expressions. + Expr* e = (! info->broccoli_peer) ? a->AttrExpr() : 0; + SERIALIZE_OPTIONAL(e); + if ( ! SERIALIZE(char(a->Tag())) ) return false; } diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index a21a7abc60..600cdd5bca 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -385,6 +385,9 @@ inline void RemoteSerializer::SetupSerialInfo(SerialInfo* info, Peer* peer) peer->phase == Peer::RUNNING ) info->new_cache_strategy = true; + if ( (peer->caps & Peer::BROCCOLI_PEER) ) + info->broccoli_peer = true; + info->include_locations = false; } @@ -1457,7 +1460,7 @@ void RemoteSerializer::Finish() Poll(true); while ( io->CanWrite() ); - loop_over_list(peers, i) + loop_over_list(peers, i) { CloseConnection(peers[i]); } @@ -2113,6 +2116,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer) if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) ) Log(LogInfo, "peer supports keep-in-cache; using that", peer); + if ( (peer->caps & Peer::BROCCOLI_PEER) ) + Log(LogInfo, "peer is a Broccoli", peer); + if ( peer->logs_requested ) log_mgr->SendAllWritersTo(peer->id); @@ -2365,6 +2371,9 @@ bool RemoteSerializer::ProcessSerialization() current_peer->phase == Peer::RUNNING ) info.new_cache_strategy = true; + if ( current_peer->caps & Peer::BROCCOLI_PEER ) + info.broccoli_peer = true; + if ( ! forward_remote_state_changes ) ignore_accesses = true; diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index f849a6a2b5..b64fdcbe66 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -198,6 +198,7 @@ protected: static const int NO_CACHING = 2; static const int PID_64BIT = 4; static const int NEW_CACHE_STRATEGY = 8; + static const int BROCCOLI_PEER = 16; // Constants to remember to who did something. static const int NONE = 0; diff --git a/src/SerialInfo.h b/src/SerialInfo.h index d322aa4b37..aa4c382349 100644 --- a/src/SerialInfo.h +++ b/src/SerialInfo.h @@ -15,6 +15,7 @@ public: pid_32bit = false; include_locations = true; new_cache_strategy = false; + broccoli_peer = false; } SerialInfo(const SerialInfo& info) @@ -28,6 +29,7 @@ public: pid_32bit = info.pid_32bit; include_locations = info.include_locations; new_cache_strategy = info.new_cache_strategy; + broccoli_peer = info.broccoli_peer; } // Parameters that control serialization. @@ -46,6 +48,11 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; + // If true, we're connecting to a Broccoli. If so, serialization + // specifics may be adapted for functionality Broccoli does not + // support. + bool broccoli_peer; + ChunkedIO::Chunk* chunk; // chunk written right before the serialization // Attributes set during serialization. @@ -70,6 +77,7 @@ public: print = 0; pid_32bit = false; new_cache_strategy = false; + broccoli_peer = false; } UnserialInfo(const UnserialInfo& info) @@ -86,6 +94,7 @@ public: print = info.print; pid_32bit = info.pid_32bit; new_cache_strategy = info.new_cache_strategy; + broccoli_peer = info.broccoli_peer; } // Parameters that control unserialization. @@ -106,6 +115,11 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; + // If true, we're connecting to a Broccoli. If so, serialization + // specifics may be adapted for functionality Broccoli does not + // support. + bool broccoli_peer; + // If a global ID already exits, of these policies is used. enum { Keep, // keep the old ID and ignore the new From a43ed633a493e5666d15ec6ca74b14ab65d642ae Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 2 Dec 2011 13:59:33 -0600 Subject: [PATCH 2/6] Changes to Broxygen master script package index - Now only lists packages as those directories in the script hierarchy that contain an __load__.bro file. - Script packages (dirs with a __load__.bro file), can now include a README (in reST format) that will automatically be appended under the link to a specific package in the master package index. --- doc/bin/group_index_generator.py | 1 + doc/scripts/CMakeLists.txt | 24 +++++++++++++++--------- doc/scripts/packages.rst | 5 ----- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/doc/bin/group_index_generator.py b/doc/bin/group_index_generator.py index e9e17f3ca9..720081e8b5 100755 --- a/doc/bin/group_index_generator.py +++ b/doc/bin/group_index_generator.py @@ -49,6 +49,7 @@ with open(group_list, 'r') as f_group_list: if not os.path.exists(os.path.dirname(group_file)): os.makedirs(os.path.dirname(group_file)) with open(group_file, 'w') as f_group_file: + f_group_file.write(":orphan:\n\n") title = "Package Index: %s\n" % os.path.dirname(group) f_group_file.write(title); for n in range(len(title)): diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index c14eab6d25..33d473b005 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -73,12 +73,14 @@ macro(REST_TARGET srcDir broInput) elseif (${extension} STREQUAL ".bif.bro") set(group bifs) elseif (relDstDir) - set(pkgIndex ${relDstDir}/index) - set(group ${pkgIndex}) + set(group ${relDstDir}/index) # add package index to master package list if not already in it - list(FIND MASTER_PKG_LIST ${pkgIndex} _found) + # and if a __load__.bro exists in the original script directory + list(FIND MASTER_PKG_LIST ${relDstDir} _found) if (_found EQUAL -1) - list(APPEND MASTER_PKG_LIST ${pkgIndex}) + if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro) + list(APPEND MASTER_PKG_LIST ${relDstDir}) + endif () endif () else () set(group "") @@ -137,11 +139,15 @@ file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") # policy/packages.rst file set(MASTER_PKG_INDEX_TEXT "") foreach (pkg ${MASTER_PKG_LIST}) - # strip of the trailing /index for the link name - get_filename_component(lnktxt ${pkg} PATH) - # pretty-up the link name by removing common scripts/ prefix - string(REPLACE "scripts/" "" lnktxt "${lnktxt}") - set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${lnktxt} <${pkg}>") + set(MASTER_PKG_INDEX_TEXT + "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n") + if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README) + file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme) + foreach (line ${pkgreadme}) + set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${line}") + endforeach () + set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n") + endif () endforeach () file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}") diff --git a/doc/scripts/packages.rst b/doc/scripts/packages.rst index 47e974b0c8..56909ffbc6 100644 --- a/doc/scripts/packages.rst +++ b/doc/scripts/packages.rst @@ -10,8 +10,3 @@ script, it supports being loaded in mass as a whole directory for convenience. Packages/scripts in the ``base/`` directory are all loaded by default, while ones in ``policy/`` provide functionality and customization options that are more appropriate for users to decide whether they'd like to load it or not. - -.. toctree:: - :maxdepth: 1 - - From 506a42638ae53f6bc86ed77a23090d567b45f54d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 5 Dec 2011 13:02:39 -0600 Subject: [PATCH 3/6] Omit loading local-.bro scripts from base cluster framework. The loading of these is better handled by BroControl and it seems odd to load them from a base/ script anyway since they'll contain site/policy specific code. Addresses #663 --- scripts/base/frameworks/cluster/__load__.bro | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/base/frameworks/cluster/__load__.bro b/scripts/base/frameworks/cluster/__load__.bro index 3334164866..bccb35dbb1 100644 --- a/scripts/base/frameworks/cluster/__load__.bro +++ b/scripts/base/frameworks/cluster/__load__.bro @@ -28,17 +28,14 @@ redef Communication::listen_port = Cluster::nodes[Cluster::node]$p; @if ( Cluster::local_node_type() == Cluster::MANAGER ) @load ./nodes/manager -@load site/local-manager @endif @if ( Cluster::local_node_type() == Cluster::PROXY ) @load ./nodes/proxy -@load site/local-proxy @endif @if ( Cluster::local_node_type() == Cluster::WORKER ) @load ./nodes/worker -@load site/local-worker @endif @endif From 9d5f79a0fa3c34998385149662297a718aec2ed6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 6 Dec 2011 11:49:29 -0800 Subject: [PATCH 4/6] Updating submodule(s). [nomail] --- CHANGES | 5 +++++ VERSION | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index b7e69d4997..dd0b98bf72 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.0-beta-97 | 2011-12-06 11:49:29 -0800 + + * Omit loading local-.bro scripts from base cluster framework. + Addresses #663 (Jon Siwek) + 2.0-beta-94 | 2011-12-03 15:57:19 -0800 * Adapting attribute serialization when talking to Broccoli. (Robin diff --git a/VERSION b/VERSION index 11e9b6983f..d8c774d329 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-94 +2.0-beta-97 diff --git a/aux/broccoli b/aux/broccoli index d8f9d4698e..05d51b55dd 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d8f9d4698e8e02f493a669c5adcf897506671b5d +Subproject commit 05d51b55dd5a8430151cff74f7876bd73385d5cd diff --git a/aux/broctl b/aux/broctl index be772bbada..406c689c88 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit be772bbada79b106db33fb9de5f56fa71226adc5 +Subproject commit 406c689c88997c4459eaf36bd2c65bce2e847c2d From 3c2fa085d43c176e4ae75874964eb810cd278cef Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 7 Dec 2011 07:45:48 -0800 Subject: [PATCH 5/6] Adapting diff-all so that it expands globs in both current and baseline directory. This way, it now spots if a Baseline file isn't produced anymore. Closes #677. --- CHANGES | 5 +++++ VERSION | 2 +- testing/external/scripts/diff-all | 11 +++++++++-- testing/external/scripts/skel/test.skeleton | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index dd0b98bf72..000c7ae6e8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.0-beta-98 | 2011-12-07 08:12:08 -0800 + + * Adapting test-suite's diff-all so that it expands globs in both + current and baseline directory. Closes #677. (Robin Sommer) + 2.0-beta-97 | 2011-12-06 11:49:29 -0800 * Omit loading local-.bro scripts from base cluster framework. diff --git a/VERSION b/VERSION index d8c774d329..6f30af8ad8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-97 +2.0-beta-98 diff --git a/testing/external/scripts/diff-all b/testing/external/scripts/diff-all index 597c769687..329bbb7f00 100755 --- a/testing/external/scripts/diff-all +++ b/testing/external/scripts/diff-all @@ -1,6 +1,10 @@ #! /usr/bin/env bash # -# Runs btest-diff on $@ and fails if any fails. +# Runs btest-diff on $@ and fails if any fails. If $@ contains globs, we expand +# them relative to *both* the current directory and the test's baseline +# directory so that we spot missing files. Note that you will need to quote +# the globals in the TEST-EXEC line as otherwise they will have been expanded relative +# to the current directory already when this scripts runs. diag=$TEST_DIAGNOSTICS @@ -14,7 +18,10 @@ fi rc=0; -for i in $@; do +files_cwd=`ls $@` +files_baseline=`cd $TEST_BASELINE && ls $@` + +for i in `echo $files_cwd $files_baseline | sort | uniq`; do if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" ]]; then if ! btest-diff $i; then echo "" >>$diag diff --git a/testing/external/scripts/skel/test.skeleton b/testing/external/scripts/skel/test.skeleton index becd970d78..a76f3d4d09 100644 --- a/testing/external/scripts/skel/test.skeleton +++ b/testing/external/scripts/skel/test.skeleton @@ -1,5 +1,5 @@ # @TEST-EXEC: zcat $TRACES/trace.gz | bro -r - %INPUT -# @TEST-EXEC: $SCRIPTS/diff-all *.log +# @TEST-EXEC: $SCRIPTS/diff-all '*.log' @load testing-setup @load test-all-policy From f1e132cd1acb2ba37c20f90b634ad0e6c74e8cd3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 7 Dec 2011 10:28:56 -0800 Subject: [PATCH 6/6] Adding missing script. --- testing/external/scripts/perftools-adapt-paths | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 testing/external/scripts/perftools-adapt-paths diff --git a/testing/external/scripts/perftools-adapt-paths b/testing/external/scripts/perftools-adapt-paths new file mode 100755 index 0000000000..2eda2477c7 --- /dev/null +++ b/testing/external/scripts/perftools-adapt-paths @@ -0,0 +1,10 @@ +#! /usr/bin/env bash +# +# Adapts relative paths in perftools stderr output to work +# directly from the top-level test directory. +# +# Returns an exit code > 0 if there's a leak. + +cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1 + +grep -q "No leaks found" $1