Merge branch 'topic/jswaro/feature/HookAddToAnalyzer-tcprs-support' of https://github.com/jswaro/bro

Making two changes here:

    - Renaming the hook to SetupAnalyzerTree.

    - Reverting the reversal of the script load order. Instead, I'm
      adding an additional script that Bro looks for to load first,
      "__preload__.bro". Also extending the plugin docs to cover this.

    - Increasing plugin API version, as I suppose adding a new virtual
      function may invalidate binary compatibility.

* 'topic/jswaro/feature/HookAddToAnalyzer-tcprs-support' of https://github.com/jswaro/bro:
  Add hook 'HookAddToAnalyzerTree' to support TCPRS plugin

CHANGES

@@ -1,4 +1,16 @@
 
+2.4-84 | 2015-08-10 14:44:39 -0700
+
+  * Add hook 'HookSetupAnalyzerTree' to allow plugins access to a
+    connection's initial analyzer tree for customization. (James
+    Swaro)
+
+  * Plugins now look for a file "__preload__.bro" in the top-level
+    script directory. If found, they load it first, before any scripts
+    defining BiF elements. This can be used to define types that the
+    BiFs already depend on (like a custom type for an event argument).
+    (Robin Sommer)
+
 2.4-81 | 2015-08-08 07:38:42 -0700
 
   * Fix a test that is failing very frequently. (Daniel Thayer)

VERSION

@@ -1 +1 @@
-2.4-81
+2.4-84

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 1a525eef9132855c2dfaf5ba62fcc572d97873d5
+Subproject commit 2470f64b58d875f9491e251b866a15a2ec4c05da

aux/plugins

@@ -1 +1 @@
-Subproject commit bb86ad945c823c94ea8385ec4ebb9546ba5198af
+Subproject commit 09cd828ba80a0df69a78e64743aedf23a29e6bdc

doc/devel/plugins.rst

@@ -209,8 +209,15 @@ directory. With the skeleton, ``<base>`` corresponds to ``build/``.
     "@load"ed.
 
 ``scripts``/__load__.bro
-    A Bro script that will be loaded immediately when the plugin gets
+    A Bro script that will be loaded when the plugin gets activated.
-    activated. See below for more information on activating plugins.
+    When this script executes, any BiF elements that the plugin
+    defines will already be available. See below for more information
+    on activating plugins.
+
+``scripts``/__preload__.bro
+    A Bro script that will be loaded when the plugin gets activated,
+    but before any BiF elements become available. See below for more
+    information on activating plugins.
 
 ``lib/bif/``
     Directory with auto-generated Bro scripts that declare the plugin's
@@ -279,7 +286,9 @@ Activating a plugin will:
     1. Load the dynamic module
     2. Make any bif items available
     3. Add the ``scripts/`` directory to ``BROPATH``
-    4. Load ``scripts/__load__.bro``
+    5. Load ``scripts/__preload__.bro``
+    6. Make BiF elements available to scripts.
+    7. Load ``scripts/__load__.bro``
 
 By default, Bro will automatically activate all dynamic plugins found
 in its search path ``BRO_PLUGIN_PATH``. However, in bare mode (``bro

src/analyzer/Manager.cc

@@ -505,6 +505,8 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn)
 	if ( ! analyzed )
 		conn->SetLifetime(non_analyzed_lifetime);
 
+	PLUGIN_HOOK_VOID(HOOK_SETUP_ANALYZER_TREE, HookSetupAnalyzerTree(conn));
+
 	return true;
 	}
 

src/plugin/Manager.cc

@@ -182,9 +182,17 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 		add_to_bro_path(scripts);
 		}
 
-	// Load {bif,scripts}/__load__.bro automatically.
+	// First load {scripts}/__preload__.bro automatically.
+	string init = dir + "scripts/__preload__.bro";
 
-	string init = dir + "lib/bif/__load__.bro";
+	if ( is_file(init) )
+		{
+		DBG_LOG(DBG_PLUGINS, "  Loading %s", init.c_str());
+		scripts_to_load.push_back(init);
+		}
+
+	// Load {bif,scripts}/__load__.bro automatically.
+	init = dir + "lib/bif/__load__.bro";
 
 	if ( is_file(init) )
 		{
@@ -660,6 +668,33 @@ void Manager::HookDrainEvents() const
 
 	}
 
+void Manager::HookSetupAnalyzerTree(Connection *conn) const
+	{
+	HookArgumentList args;
+
+	if ( HavePluginForHook(META_HOOK_PRE) )
+		{
+		args.push_back(conn);
+		MetaHookPre(HOOK_SETUP_ANALYZER_TREE, args);
+		}
+
+	hook_list *l = hooks[HOOK_SETUP_ANALYZER_TREE];
+
+	if ( l )
+		{
+		for (hook_list::iterator i = l->begin() ; i != l->end(); ++i)
+			{
+			Plugin *p = (*i).second;
+			p->HookSetupAnalyzerTree(conn);
+			}
+		}
+
+	if ( HavePluginForHook(META_HOOK_POST) )
+		{
+		MetaHookPost(HOOK_SETUP_ANALYZER_TREE, args, HookArgument());
+		}
+	}
+
 void Manager::HookUpdateNetworkTime(double network_time) const
 	{
 	HookArgumentList args;

src/plugin/Manager.h

@@ -264,6 +264,15 @@ public:
 	 */
 	void HookUpdateNetworkTime(double network_time) const;
 
+	/**
+	 * Hook that executes when a connection's initial analyzer tree
+	 * has been fully set up. The hook can manipulate the tree at this time,
+	 * for example by adding further analyzers.
+	 *
+	 * @param conn The connection.
+	 */
+	void HookSetupAnalyzerTree(Connection *conn) const;
+
 	/**
 	 * Hook that informs plugins that the event queue is being drained.
 	 */

src/plugin/Plugin.cc

@@ -23,6 +23,7 @@ const char* plugin::hook_name(HookType h)
 		"DrainEvents",
 		"UpdateNetworkTime",
 		"BroObjDtor",
+		"SetupAnalyzerTree",
 		// MetaHooks
 		"MetaHookPre",
 		"MetaHookPost",
@@ -310,6 +311,10 @@ void Plugin::HookUpdateNetworkTime(double network_time)
 	{
 	}
 
+void Plugin::HookSetupAnalyzerTree(Connection *conn)
+	{
+	}
+
 void Plugin::HookBroObjDtor(void* obj)
 	{
 	}

src/plugin/Plugin.h

@@ -14,7 +14,7 @@
 
 // We allow to override this externally for testing purposes.
 #ifndef BRO_PLUGIN_API_VERSION
-#define BRO_PLUGIN_API_VERSION 3
+#define BRO_PLUGIN_API_VERSION 4
 #endif
 
 class ODesc;
@@ -39,6 +39,7 @@ enum HookType {
 	HOOK_DRAIN_EVENTS,		//< Activates Plugin::HookDrainEvents()
 	HOOK_UPDATE_NETWORK_TIME,	//< Activates Plugin::HookUpdateNetworkTime.
 	HOOK_BRO_OBJ_DTOR,		//< Activates Plugin::HookBroObjDtor.
+	HOOK_SETUP_ANALYZER_TREE,	//< Activates Plugin::HookAddToAnalyzerTree
 
 	// Meta hooks.
 	META_HOOK_PRE,			//< Activates Plugin::MetaHookPre().
@@ -636,6 +637,8 @@ protected:
 	 */
 	virtual void HookUpdateNetworkTime(double network_time);
 
+	virtual void HookSetupAnalyzerTree(Connection *conn);
+
 	/**
 	 * Hook for destruction of objects registered with
 	 * RequestBroObjDtor(). When Bro's reference counting triggers the

testing/btest/Baseline/plugins.hooks/output

@@ -220,7 +220,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1429655378.868621, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1439244305.210087, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@@ -326,7 +326,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1429655378.868621, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1439244305.210087, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::build, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, )) -> <no result>
@@ -490,6 +490,7 @@
 0.000000   MetaHookPost  LoadFile(./top-k.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./topk) -> -1
 0.000000   MetaHookPost  LoadFile(./types.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./types.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./unique) -> -1
 0.000000   MetaHookPost  LoadFile(./utils) -> -1
 0.000000   MetaHookPost  LoadFile(./utils-commands) -> -1
@@ -509,6 +510,7 @@
 0.000000   MetaHookPost  LoadFile(.<...>/raw) -> -1
 0.000000   MetaHookPost  LoadFile(.<...>/sqlite) -> -1
 0.000000   MetaHookPost  LoadFile(<...>/__load__.bro) -> -1
+0.000000   MetaHookPost  LoadFile(<...>/__preload__.bro) -> -1
 0.000000   MetaHookPost  LoadFile(<...>/hooks.bro) -> -1
 0.000000   MetaHookPost  LoadFile(base/bif) -> -1
 0.000000   MetaHookPost  LoadFile(base/init-default.bro) -> -1
@@ -810,7 +812,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1429655378.868621, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1439244305.210087, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@@ -916,7 +918,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1429655378.868621, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1439244305.210087, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::build, <frame>, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, ))
@@ -1080,6 +1082,7 @@
 0.000000   MetaHookPre   LoadFile(./top-k.bif.bro)
 0.000000   MetaHookPre   LoadFile(./topk)
 0.000000   MetaHookPre   LoadFile(./types.bif.bro)
+0.000000   MetaHookPre   LoadFile(./types.bro)
 0.000000   MetaHookPre   LoadFile(./unique)
 0.000000   MetaHookPre   LoadFile(./utils)
 0.000000   MetaHookPre   LoadFile(./utils-commands)
@@ -1099,6 +1102,7 @@
 0.000000   MetaHookPre   LoadFile(.<...>/raw)
 0.000000   MetaHookPre   LoadFile(.<...>/sqlite)
 0.000000   MetaHookPre   LoadFile(<...>/__load__.bro)
+0.000000   MetaHookPre   LoadFile(<...>/__preload__.bro)
 0.000000   MetaHookPre   LoadFile(<...>/hooks.bro)
 0.000000   MetaHookPre   LoadFile(base/bif)
 0.000000   MetaHookPre   LoadFile(base/init-default.bro)
@@ -1399,7 +1403,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1429655378.868621, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1439244305.210087, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1505,7 +1509,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1429655378.868621, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1439244305.210087, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Notice::want_pp()
 0.000000 | HookCallFunction PacketFilter::build()
 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, )