Merge remote branch 'origin/topic/jsiwek/misc-doc-fixes'

* origin/topic/jsiwek/misc-doc-fixes:
  More tweaks to generated script docs.
  Various changes to documentation framework.

Closes #598.

CHANGES

@@ -1,4 +1,16 @@
 
+1.6-dev-1221 | 2011-09-08 08:41:17 -0700
+
+  * Updates for documentation framework and script docs. (Jon Siwek)
+
+  * The script level PF_RING support isn't working so removing it.
+    (Seth Hall)
+
+  * Delete SSL certificates from memory after ssl_established event.
+    (Seth Hall)
+
+  * Small fixes for SSL analysis. (Seth Hall)
+
 1.6-dev-1212 | 2011-09-07 16:15:28 -0700
 
   * Internally, the UID generation can now return values from

Makefile

@@ -13,15 +13,14 @@ all: configured
 install: configured
 	( cd $(BUILD) && make install )
 
-clean: configured
+clean: configured docclean
 	( cd $(BUILD) && make clean )
-	( cd $(BUILD) && make docclean && make restclean )
 
 doc: configured
 	( cd $(BUILD) && make doc )
 
 docclean: configured
-	( cd $(BUILD) && make docclean && make restclean )
+	( cd $(BUILD) && make docclean )
 
 dist:
 	@./pkg/make-src-packages

VERSION

@@ -1 +1 @@
-1.6-dev-1212
+1.6-dev-1221

aux/broctl

@@ -1 +1 @@
-Subproject commit 3679dbce2e1a7fc31065b174670fa54e35c3ae75
+Subproject commit 2b9053d40d7ef497c8cef6357b59f43129976d65

doc/CMakeLists.txt

@@ -1 +1,4 @@
+add_custom_target(doc)
+add_custom_target(docclean)
+
 add_subdirectory(scripts)

doc/scripts/CMakeLists.txt

@@ -212,9 +212,9 @@ add_custom_target(restclean
                           ${RST_OUTPUT_DIR}
                   VERBATIM)
 
-# The "doc" target generates reST documentation for any outdated bro scripts
+# The "sphinxdoc" target generates reST documentation for any outdated bro
-# and then uses Sphinx to generate HTML documentation from the reST
+# scripts and then uses Sphinx to generate HTML documentation from the reST
-add_custom_target(doc
+add_custom_target(sphinxdoc
                   # copy the template documentation to the build directory
                   # to give as input for sphinx
                   COMMAND "${CMAKE_COMMAND}" -E copy_directory
@@ -252,13 +252,16 @@ add_custom_target(doc
                   # SOURCES just adds stuff to IDE projects as a convenience
                   SOURCES ${DOC_SOURCES})
 
-# The "docclean" target removes just the Sphinx input/output directories
+# The "sphinxclean" target removes just the Sphinx input/output directories
 # from the build directory.
-add_custom_target(docclean
+add_custom_target(sphinxclean
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${DOC_SOURCE_WORKDIR}
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${DOC_OUTPUT_DIR}
                   VERBATIM)
 
-add_dependencies(doc docclean restdoc)
+add_dependencies(sphinxdoc sphinxclean restdoc)
+
+add_dependencies(doc sphinxdoc)
+add_dependencies(docclean sphinxclean restclean)

doc/scripts/DocSourcesList.cmake

@@ -16,13 +16,13 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
 rest_target(${psd} base/init-default.bro internal)
 rest_target(${psd} base/init-bare.bro internal)
 
-rest_target(${CMAKE_BINARY_DIR}/src/base bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base types.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@@ -33,6 +33,7 @@ rest_target(${psd} base/frameworks/control/main.bro)
 rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
+rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/metrics/cluster.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
@@ -101,6 +102,7 @@ rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
+rest_target(${psd} policy/misc/pf-ring-load-balancing.bro)
 rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)

doc/scripts/example.bro

@@ -5,20 +5,6 @@
 ##! (reST) document's summary section.
 ##!
 ##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
-##!
-##! A script's logging information has to be documented manually as minimally
-##! shown below.  Note that references may not always be possible (e.g.
-##! anonymous filter functions) and a script may not need to document
-##! each of "columns", "event", "filter" depending on exactly what it's doing.
-##!
-##! **Logging Stream ID:** :bro:enum:`Example::EXAMPLE`
-##!     :Columns:    :bro:type:`Example::Info`
-##!     :Event:      :bro:id:`Example::log_example`
-##!     :Filter:     ``example-filter``
-##!         uses :bro:id:`Example::filter_func` to determine whether to
-##!         exclude the ``ts`` field
-##!
-##! :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
 
 # Comments that use a single pound sign (#) are not significant to
 # a script's auto-generated documentation, but ones that use a

doc/scripts/genDocSourcesList.sh

@@ -73,7 +73,7 @@ bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
 for file in $bifs
 do
     f=${file:2}.bro
-    echo "rest_target(\${CMAKE_BINARY_DIR}/src/base $f)" >> $outfile
+    echo "rest_target(\${CMAKE_BINARY_DIR}/src base/$f)" >> $outfile
 done
 
 scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`

scripts/base/frameworks/logging/main.bro

@@ -52,7 +52,7 @@ export {
 		## If not given, all entries are recorded.
 		##
 		## rec: An instance of the streams's ``columns`` type with its
-		## fields set to the values to logged.
+		##      fields set to the values to logged.
 		##
 		## Returns: True if the entry is to be recorded.
 		pred: function(rec: any): bool &optional;

scripts/base/init-bare.bro

@@ -1382,8 +1382,9 @@ const enable_syslog = F &redef;
 const peer_description = "bro" &redef;
 
 ## If true, broadcast events/state received from one peer to other peers.
-## NOTE: These options are only temporary. They will disappear when we get a
+##
-## more sophisticated script-level communication framework.
+## .. note:: These options are only temporary. They will disappear when we get
+##    a more sophisticated script-level communication framework.
 const forward_remote_events = F &redef;
 ## See :bro:id:`forward_remote_events`
 const forward_remote_state_changes = F &redef;
@@ -1513,6 +1514,6 @@ const skip_http_data = F &redef;
 ## UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro.
 const parse_udp_tunnels = F &redef;
 
-## Load the logging framework here because it uses fairly deep integration with 
+# Load the logging framework here because it uses fairly deep integration with 
-## BiFs and script-land defined types.
+# BiFs and script-land defined types.
 @load base/frameworks/logging

scripts/base/protocols/conn/main.bro

@@ -74,11 +74,11 @@ export {
 		## This history is not meant to encode how much data that happened to be.
 		history:      string          &log &optional;
 		## Number of packets the originator sent.
-		## Only set if :bro:id:`use_conn_size_analyzer`=T
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_pkts:     count      &log &optional;
 		## Number IP level bytes the originator sent (as seen on the wire,
 		## taken from IP total_length header field).
-		## Only set if :bro:id:`use_conn_size_analyzer`=T
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_ip_bytes: count      &log &optional;
 		## Number of packets the responder sent. See ``orig_pkts``.
 		resp_pkts:     count      &log &optional;

src/BroDoc.cc

@@ -227,6 +227,7 @@ void BroDoc::WriteDocFile() const
 		WriteToDoc("%s\n", packet_filter.c_str());
 		}
 
+#if 0   // Disabled for now.
 	BroDocObjList::const_iterator it;
 	bool hasPrivateIdentifiers = false;
 
@@ -241,6 +242,7 @@ void BroDoc::WriteDocFile() const
 
 	if ( hasPrivateIdentifiers )
 		WriteInterface("Private Interface", '-', '~', false, false);
+#endif
 	}
 
 void BroDoc::WriteInterface(const char* heading, char underline,

src/BroDoc.h

@@ -167,6 +167,18 @@ public:
 		all.push_back(o);
 		}
 
+	/**
+	 * Schedules documentation of an event handler declared by the script.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script event handler and
+	 *        also any associated comments about it.
+	 */
+	void AddEventHandler(const BroDocObj* o)
+		{
+		event_handlers.push_back(o);
+		all.push_back(o);
+		}
+
 	/**
 	 * Schedules documentation of a function declared by the script.
 	 * @param o A pointer to a BroDocObj which contains the internal
@@ -228,6 +240,7 @@ protected:
 	BroDocObjList types;
 	BroDocObjList notices;
 	BroDocObjList events;
+	BroDocObjList event_handlers;
 	BroDocObjMap functions;
 	BroDocObjList redefs;
 

src/CMakeLists.txt

@@ -418,6 +418,7 @@ collect_headers(bro_HEADERS ${bro_SRCS})
 
 add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}")
 add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}")
+add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}")
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 

src/parse.y

@@ -1179,7 +1179,7 @@ func_hdr:
 				   FUNC_FLAVOR_EVENT, 0, $3);
 			$$ = $3;
 			if ( generate_documentation )
-				current_reST_doc->AddEvent(
+				current_reST_doc->AddEventHandler(
 					new BroDocObj($2, reST_doc_comments));
 			}
 	|	TOK_REDEF TOK_EVENT event_id func_params

src/scan.l

@@ -80,6 +80,19 @@ static const char* canon_doc_comment(const char* comment)
 	return ( comment[0] == ' ' ) ? comment + 1 : comment;
 	}
 
+static std::string canon_doc_func_param(const char* id_start)
+	{
+	std::string id_name(id_start, strcspn(id_start, ":"));
+	const char* comment = id_start + id_name.size() + 1;
+	std::string doc;
+
+	if ( id_name == "Returns" )
+		doc.append(":returns:").append(comment);
+	else
+		doc.append(":param ").append(id_name).append(":").append(comment);
+	return doc;
+	}
+
 static ino_t get_inode_num(FILE* f, const char* filename)
 	{
 	struct stat b;
@@ -155,6 +168,12 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 	return TOK_POST_DOC;
 }
 
+<DOC>##{OWS}{ID}:.* {
+	const char* id_start = skip_whitespace(yytext + 2);
+	yylval.str = copy_string(canon_doc_func_param(id_start).c_str());
+	return TOK_DOC;
+}
+
 <DOC>##.* {
 	if ( yytext[2] != '#' )
 		{
@@ -169,20 +188,6 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 		// Comment is documenting either a function parameter or return type,
 		// so appropriate reST markup substitutions are automatically made
 		// in order to distinguish them from other comments.
-		const char* id_start = skip_whitespace(yytext + 2);
-		size_t id_len = strcspn(id_start, ":");
-		char* id_name = new char[id_len + 1];
-		strncpy(id_name, id_start, id_len);
-		id_name[id_len] = '\0';
-		const char* comment = id_start + id_len + 1;
-
-		std::string doc;
-
-		if ( streq(id_name, "Returns") )
-			doc.append(":returns:").append(comment);
-		else
-			doc.append(":param ").append(id_name).append(":").append(comment);
-
 		if ( ! reST_doc_comments )
 			reST_doc_comments = new std::list<std::string>();
 
@@ -192,9 +197,8 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 		// 2) has a blank line between it and non-field-list reST markup,
 		//    which is required for correct HTML rendering by Sphinx
 		reST_doc_comments->push_back("");
-		reST_doc_comments->push_back(doc);
+		const char* id_start = skip_whitespace(yytext + 2);
-
+		reST_doc_comments->push_back(canon_doc_func_param(id_start));
-		delete [] id_name;
 		}
 }
 

src/util.cc

@@ -891,7 +891,7 @@ const char* normalize_path(const char* path)
 	return copy_string(new_path.c_str());
 	}
 
-// Returns the subpath of the root Bro script install/source directory in
+// Returns the subpath of the root Bro script install/source/build directory in
 // which the loaded file is located.  If it's not under a subpath of that
 // directory (e.g. cwd or custom path) then the full path is returned.
 void get_script_subpath(const std::string& full_filename, const char** subpath)
@@ -909,11 +909,15 @@ void get_script_subpath(const std::string& full_filename, const char** subpath)
 
 	// first check if this is some subpath of the installed scripts root path,
 	// if not check if it's a subpath of the script source root path,
-	// if neither, will just use the given directory
+	// then check if it's a subpath of the build directory (where BIF scripts
-	if ( (p=my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
+	// will get generated).
+	// If none of those, will just use the given directory.
+	if ( (p = my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
 		my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
-	else if ( (p=my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
+	else if ( (p = my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
 		my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
+	else if ( (p = my_subpath.find(BRO_BUILD_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_BUILD_PATH));
 
 	// if root path found, remove path separators until next path component
 	if ( p != std::string::npos )

testing/btest/Baseline/doc.autogen-reST-example/example.rst

@@ -15,20 +15,6 @@ these comments are transferred directly into the auto-generated
 
 .. tip:: You can embed directives and roles within ``##``-stylized comments.
 
-A script's logging information has to be documented manually as minimally
-shown below.  Note that references may not always be possible (e.g.
-anonymous filter functions) and a script may not need to document
-each of "columns", "event", "filter" depending on exactly what it's doing.
-
-**Logging Stream ID:** :bro:enum:`Example::EXAMPLE`
-    :Columns:    :bro:type:`Example::Info`
-    :Event:      :bro:id:`Example::log_example`
-    :Filter:     ``example-filter``
-        uses :bro:id:`Example::filter_func` to determine whether to
-        exclude the ``ts`` field
-
-:Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
-
 :Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
 
 Summary
@@ -72,8 +58,6 @@ Events
 
 :bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in
                                                   logging streams and is raised once for each log entry.
-
-:bro:id:`bro_init`: :bro:type:`event`
 ================================================= =============================================================
 
 Functions
@@ -233,10 +217,6 @@ Events
    This is a declaration of an example event that can be used in
    logging streams and is raised once for each log entry.
 
-.. bro:id:: bro_init
-
-   :Type: :bro:type:`event` ()
-
 Functions
 ~~~~~~~~~
 .. bro:id:: Example::a_function
@@ -309,40 +289,3 @@ Filters added::
     [ssl] = tcp port 443,
     [nntps] = tcp port 562
 
-Private Interface
------------------
-State Variables
-~~~~~~~~~~~~~~~
-.. bro:id:: Example::example_ports
-
-   :Type: :bro:type:`set` [:bro:type:`port`]
-   :Attributes: :bro:attr:`&redef`
-   :Default:
-
-   ::
-
-      {
-         443/tcp,
-         562/tcp
-      }
-
-Types
-~~~~~
-.. bro:type:: Example::PrivateRecord
-
-   :Type: :bro:type:`record`
-
-      field1: :bro:type:`bool`
-
-      field2: :bro:type:`count`
-
-Functions
-~~~~~~~~~
-.. bro:id:: Example::filter_func
-
-   :Type: :bro:type:`function` (rec: :bro:type:`Example::Info`) : :bro:type:`bool`
-
-.. bro:id:: Example::function_without_proto
-
-   :Type: :bro:type:`function` (tag: :bro:type:`string`) : :bro:type:`string`
-

testing/btest/Baseline/doc.autogen-reST-func-params/autogen-reST-func-params.rst

@@ -0,0 +1,57 @@
+.. Automatically generated.  Do not edit.
+
+autogen-reST-func-params.bro
+============================
+
+:download:`Original Source File <autogen-reST-func-params.bro>`
+
+Overview
+--------
+
+
+Summary
+~~~~~~~
+Types
+#####
+======================================== =
+:bro:type:`test_rec`: :bro:type:`record`
+======================================== =
+
+Functions
+#########
+===================================== ======================================
+:bro:id:`test_func`: :bro:type:`func` This is a global function declaration.
+===================================== ======================================
+
+Public Interface
+----------------
+Types
+~~~~~
+.. bro:type:: test_rec
+
+   :Type: :bro:type:`record`
+
+      field_func: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
+         This is a record field function.
+         
+         :param i: First param.
+         :param j: Second param.
+         
+         :returns: A string.
+
+Functions
+~~~~~~~~~
+.. bro:id:: test_func
+
+   :Type: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
+
+   This is a global function declaration.
+   
+   
+   :param i: First param.
+   
+   :param j: Second param.
+   
+   
+   :returns: A string.
+

testing/btest/doc/autogen-reST-func-params.bro

@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro --doc-scripts %INPUT
+# @TEST-EXEC: btest-diff autogen-reST-func-params.rst
+
+## This is a global function declaration.
+##
+## i: First param.
+## j: Second param.
+##
+## Returns: A string.
+global test_func: function(i: int, j: int): string;
+
+type test_rec: record {
+	## This is a record field function.
+	##
+	## i: First param.
+	## j: Second param.
+	##
+	## Returns: A string.
+	field_func: function(i: int, j: int): string;
+};