Various changes to documentation framework.

- Reorganize top-level 'doc' Makefile target so submodules can easily
  add their own doc-generating routines to it.  e.g. the Bro project
  makes a placeholder 'doc' target, then adds 'restdoc', 'sphinxdoc';
  later Broccoli can add it's own target as a dependency for generating
  API docs.

- Fixed generated docs for BIFs not being organized under a base/
  subdirectory like the original source files.

- Fixed documentation style for function parameters not applying to
  functions declared as record fields.

- Misc. script documentation tweaks to address warnings given by Sphinx.

Makefile

@@ -13,15 +13,14 @@ all: configured
 install: configured
 	( cd $(BUILD) && make install )
 
-clean: configured
+clean: configured docclean
 	( cd $(BUILD) && make clean )
-	( cd $(BUILD) && make docclean && make restclean )
 
 doc: configured
 	( cd $(BUILD) && make doc )
 
 docclean: configured
-	( cd $(BUILD) && make docclean && make restclean )
+	( cd $(BUILD) && make docclean )
 
 dist:
 	@./pkg/make-src-packages

doc/CMakeLists.txt

@@ -1 +1,4 @@
+add_custom_target(doc)
+add_custom_target(docclean)
+
 add_subdirectory(scripts)

doc/scripts/CMakeLists.txt

@@ -212,9 +212,9 @@ add_custom_target(restclean
                           ${RST_OUTPUT_DIR}
                   VERBATIM)
 
-# The "doc" target generates reST documentation for any outdated bro scripts
+# The "sphinxdoc" target generates reST documentation for any outdated bro
-# and then uses Sphinx to generate HTML documentation from the reST
+# scripts and then uses Sphinx to generate HTML documentation from the reST
-add_custom_target(doc
+add_custom_target(sphinxdoc
                   # copy the template documentation to the build directory
                   # to give as input for sphinx
                   COMMAND "${CMAKE_COMMAND}" -E copy_directory
@@ -252,13 +252,16 @@ add_custom_target(doc
                   # SOURCES just adds stuff to IDE projects as a convenience
                   SOURCES ${DOC_SOURCES})
 
-# The "docclean" target removes just the Sphinx input/output directories
+# The "sphinxclean" target removes just the Sphinx input/output directories
 # from the build directory.
-add_custom_target(docclean
+add_custom_target(sphinxclean
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${DOC_SOURCE_WORKDIR}
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${DOC_OUTPUT_DIR}
                   VERBATIM)
 
-add_dependencies(doc docclean restdoc)
+add_dependencies(sphinxdoc sphinxclean restdoc)
+
+add_dependencies(doc sphinxdoc)
+add_dependencies(docclean sphinxclean restclean)

doc/scripts/DocSourcesList.cmake

@@ -16,13 +16,13 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
 rest_target(${psd} base/init-default.bro internal)
 rest_target(${psd} base/init-bare.bro internal)
 
-rest_target(${CMAKE_BINARY_DIR}/src/base bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src/base types.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@@ -33,6 +33,7 @@ rest_target(${psd} base/frameworks/control/main.bro)
 rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
+rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/metrics/cluster.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
@@ -101,6 +102,7 @@ rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
+rest_target(${psd} policy/misc/pf-ring-load-balancing.bro)
 rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)

doc/scripts/genDocSourcesList.sh

@@ -73,7 +73,7 @@ bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
 for file in $bifs
 do
     f=${file:2}.bro
-    echo "rest_target(\${CMAKE_BINARY_DIR}/src/base $f)" >> $outfile
+    echo "rest_target(\${CMAKE_BINARY_DIR}/src base/$f)" >> $outfile
 done
 
 scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`

scripts/base/init-bare.bro

@@ -1382,8 +1382,9 @@ const enable_syslog = F &redef;
 const peer_description = "bro" &redef;
 
 ## If true, broadcast events/state received from one peer to other peers.
-## NOTE: These options are only temporary. They will disappear when we get a
+##
-## more sophisticated script-level communication framework.
+## .. note:: These options are only temporary. They will disappear when we get
+##    a more sophisticated script-level communication framework.
 const forward_remote_events = F &redef;
 ## See :bro:id:`forward_remote_events`
 const forward_remote_state_changes = F &redef;
@@ -1513,6 +1514,6 @@ const skip_http_data = F &redef;
 ## UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro.
 const parse_udp_tunnels = F &redef;
 
-## Load the logging framework here because it uses fairly deep integration with 
+# Load the logging framework here because it uses fairly deep integration with 
-## BiFs and script-land defined types.
+# BiFs and script-land defined types.
 @load base/frameworks/logging

scripts/base/protocols/conn/main.bro

@@ -74,11 +74,11 @@ export {
 		## This history is not meant to encode how much data that happened to be.
 		history:      string          &log &optional;
 		## Number of packets the originator sent.
-		## Only set if :bro:id:`use_conn_size_analyzer`=T
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_pkts:     count      &log &optional;
 		## Number IP level bytes the originator sent (as seen on the wire,
 		## taken from IP total_length header field).
-		## Only set if :bro:id:`use_conn_size_analyzer`=T
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_ip_bytes: count      &log &optional;
 		## Number of packets the responder sent. See ``orig_pkts``.
 		resp_pkts:     count      &log &optional;

src/CMakeLists.txt

@@ -418,6 +418,7 @@ collect_headers(bro_HEADERS ${bro_SRCS})
 
 add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}")
 add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}")
+add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}")
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 

src/scan.l

@@ -80,6 +80,19 @@ static const char* canon_doc_comment(const char* comment)
 	return ( comment[0] == ' ' ) ? comment + 1 : comment;
 	}
 
+static std::string canon_doc_func_param(const char* id_start)
+	{
+	std::string id_name(id_start, strcspn(id_start, ":"));
+	const char* comment = id_start + id_name.size() + 1;
+	std::string doc;
+
+	if ( id_name == "Returns" )
+		doc.append(":returns:").append(comment);
+	else
+		doc.append(":param ").append(id_name).append(":").append(comment);
+	return doc;
+	}
+
 static ino_t get_inode_num(FILE* f, const char* filename)
 	{
 	struct stat b;
@@ -155,6 +168,12 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 	return TOK_POST_DOC;
 }
 
+<DOC>##{OWS}{ID}:.* {
+	const char* id_start = skip_whitespace(yytext + 2);
+	yylval.str = copy_string(canon_doc_func_param(id_start).c_str());
+	return TOK_DOC;
+}
+
 <DOC>##.* {
 	if ( yytext[2] != '#' )
 		{
@@ -169,20 +188,6 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 		// Comment is documenting either a function parameter or return type,
 		// so appropriate reST markup substitutions are automatically made
 		// in order to distinguish them from other comments.
-		const char* id_start = skip_whitespace(yytext + 2);
-		size_t id_len = strcspn(id_start, ":");
-		char* id_name = new char[id_len + 1];
-		strncpy(id_name, id_start, id_len);
-		id_name[id_len] = '\0';
-		const char* comment = id_start + id_len + 1;
-
-		std::string doc;
-
-		if ( streq(id_name, "Returns") )
-			doc.append(":returns:").append(comment);
-		else
-			doc.append(":param ").append(id_name).append(":").append(comment);
-
 		if ( ! reST_doc_comments )
 			reST_doc_comments = new std::list<std::string>();
 
@@ -192,9 +197,8 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 		// 2) has a blank line between it and non-field-list reST markup,
 		//    which is required for correct HTML rendering by Sphinx
 		reST_doc_comments->push_back("");
-		reST_doc_comments->push_back(doc);
+		const char* id_start = skip_whitespace(yytext + 2);
-
+		reST_doc_comments->push_back(canon_doc_func_param(id_start));
-		delete [] id_name;
 		}
 }
 

src/util.cc

@@ -891,7 +891,7 @@ const char* normalize_path(const char* path)
 	return copy_string(new_path.c_str());
 	}
 
-// Returns the subpath of the root Bro script install/source directory in
+// Returns the subpath of the root Bro script install/source/build directory in
 // which the loaded file is located.  If it's not under a subpath of that
 // directory (e.g. cwd or custom path) then the full path is returned.
 void get_script_subpath(const std::string& full_filename, const char** subpath)
@@ -909,11 +909,15 @@ void get_script_subpath(const std::string& full_filename, const char** subpath)
 
 	// first check if this is some subpath of the installed scripts root path,
 	// if not check if it's a subpath of the script source root path,
-	// if neither, will just use the given directory
+	// then check if it's a subpath of the build directory (where BIF scripts
+	// will get generated).
+	// If none of those, will just use the given directory.
 	if ( (p=my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
 		my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
 	else if ( (p=my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
 		my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
+	else if ( (p=my_subpath.find(BRO_BUILD_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_BUILD_PATH));
 
 	// if root path found, remove path separators until next path component
 	if ( p != std::string::npos )

testing/btest/Baseline/doc.autogen-reST-func-params/autogen-reST-func-params.rst

@@ -0,0 +1,57 @@
+.. Automatically generated.  Do not edit.
+
+autogen-reST-func-params.bro
+============================
+
+:download:`Original Source File <autogen-reST-func-params.bro>`
+
+Overview
+--------
+
+
+Summary
+~~~~~~~
+Types
+#####
+======================================== =
+:bro:type:`test_rec`: :bro:type:`record`
+======================================== =
+
+Functions
+#########
+===================================== ======================================
+:bro:id:`test_func`: :bro:type:`func` This is a global function declaration.
+===================================== ======================================
+
+Public Interface
+----------------
+Types
+~~~~~
+.. bro:type:: test_rec
+
+   :Type: :bro:type:`record`
+
+      field_func: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
+         This is a record field function.
+         
+         :param i: First param.
+         :param j: Second param.
+         
+         :returns: A string.
+
+Functions
+~~~~~~~~~
+.. bro:id:: test_func
+
+   :Type: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
+
+   This is a global function declaration.
+   
+   
+   :param i: First param.
+   
+   :param j: Second param.
+   
+   
+   :returns: A string.
+

testing/btest/doc/autogen-reST-func-params.bro

@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro --doc-scripts %INPUT
+# @TEST-EXEC: btest-diff autogen-reST-func-params.rst
+
+## This is a global function declaration.
+##
+## i: First param.
+## j: Second param.
+##
+## Returns: A string.
+global test_func: function(i: int, j: int): string;
+
+type test_rec: record {
+	## This is a record field function.
+	##
+	## i: First param.
+	## j: Second param.
+	##
+	## Returns: A string.
+	field_func: function(i: int, j: int): string;
+};