More changes to how Bro generates docs for scripts in subdirs of BROPATH

The output reST filename now reflects the subdir information (by flattening
'/' path separators into the '^' character).  This is to prevent file name
conflicts when generated reST docs, but during the CMake 'doc' target to
build HTML docs, everything gets unflattened.

doc/scripts/CMakeLists.txt

@@ -59,8 +59,6 @@ macro(REST_TARGET srcDir broInput)
         set(basename "${basename}.init")
     endif ()
 
-    set (restFile "${basename}.rst")
-
     if (NOT relDstDir)
         set(docName "${basename}")
         set(dstDir "${RST_OUTPUT_DIR}")
@@ -69,7 +67,9 @@ macro(REST_TARGET srcDir broInput)
         set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}")
     endif ()
 
-    set(restOutput "${dstDir}/${restFile}")
+    set(restFile "${docName}.rst")
+    string(REPLACE "/" "^" restFile ${restFile})
+    set(restOutput "${dstDir}/${basename}.rst")
 
     set(indexEntry "   ${docName} <${docName}>")
     set(MASTER_POLICY_INDEX_TEXT "${MASTER_POLICY_INDEX_TEXT}\n${indexEntry}")
@@ -97,7 +97,7 @@ macro(REST_TARGET srcDir broInput)
     if (${group} STREQUAL "default" OR ${group} STREQUAL "bifs")
         set(BRO_ARGS --doc-scripts --exec '')
     else ()
-        set(BRO_ARGS --doc-scripts ${srcDir}/${broInput})
+        set(BRO_ARGS --doc-scripts ${broInput})
     endif ()
 
     add_custom_command(OUTPUT ${restOutput}
@@ -105,7 +105,7 @@ macro(REST_TARGET srcDir broInput)
         COMMAND "${CMAKE_COMMAND}"
         ARGS -E remove_directory .state
         # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH} ${CMAKE_BINARY_DIR}/src/bro
+        COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
         ARGS ${BRO_ARGS} || (rm -rf .state *.log *.rst && exit 1)
         # move generated doc into a new directory tree that
         # defines the final structure of documents

src/BroDoc.cc

@@ -7,47 +7,86 @@
 #include "BroDoc.h"
 #include "BroDocObj.h"
 
-BroDoc::BroDoc(const std::string& sourcename)
+BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 	{
-#ifdef DEBUG
-	fprintf(stdout, "Documenting source: %s\n", sourcename.c_str());
-#endif
-	source_filename = sourcename.substr(sourcename.find_last_of('/') + 1);
+	size_t f_pos = abs.find_last_of('/');
+	if ( std::string::npos == f_pos )
+		source_filename = abs;
+	else
+		source_filename = abs.substr(f_pos + 1);
 
-	size_t ext_pos = source_filename.find_last_of('.');
-	std::string ext = source_filename.substr(ext_pos + 1);
-
-	if ( ext_pos == std::string::npos || ext != "bro" )
+	if ( rel == abs )
 		{
-		if ( source_filename != "bro.init" && source_filename != "<stdin>" )
-			{
-			fprintf(stderr,
-				"Warning: documenting file without .bro extension: %s\n",
-				sourcename.c_str());
+		// The Bro script must have been loaded from an explicit path,
+		// so just use the basename as the document title
+		doc_title = source_filename;
 		}
 	else
 		{
-			// Force the reST documentation file to be "bro.init.rst".
-			ext_pos = std::string::npos;
-			}
+		// Must have relied on BROPATH to load the script, keep the relative
+		// directory as part of the source file name
+		size_t ext_pos = rel.find_last_of('.');
+		std::string rel_ext = rel.substr(ext_pos + 1);
+		ext_pos = abs.find_last_of('.');
+		std::string abs_ext = abs.substr(ext_pos + 1);
+
+		if ( rel_ext == abs_ext || std::string::npos == ext_pos )
+			doc_title = rel;
+		else
+			doc_title = rel + "." + abs_ext;
 		}
 
-	reST_filename = source_filename.substr(0, ext_pos);
+	reST_filename = doc_title;
+	size_t ext_pos = reST_filename.find(".bro");
+	if ( std::string::npos == ext_pos )
 		reST_filename += ".rst";
+	else
+		reST_filename.replace(ext_pos, 4, ".rst");
+
+	reST_filename = doc_title.substr(0, ext_pos);
+	reST_filename += ".rst";
+
+/*
+	// if the bro source file is being loaded from a relative path,
+	// re-create that directory tree to store the output
+	size_t f_pos = reST_filename.find_last_of('/');
+	if ( std::string::npos != f_pos )
+		{
+		std::string outdir = reST_filename.substr(0, f_pos);
+		std::string subdir;
+		while ( ! outdir.empty() )
+			{
+			size_t pos = outdir.find_first_of('/');
+			if ( pos != std::string::npos ) pos++;
+			subdir += outdir.substr(0, pos);
+			outdir.erase(0, pos);
+			ensure_dir(subdir.c_str());
+			}
+		}
+*/
+	// Instead of re-creating the directory hierarchy based on related
+	// loads, just replace the directory separatories such that the reST
+	// output will all be placed in a flat directory (the working dir).
+	std::for_each(reST_filename.begin(), reST_filename.end(), replace_slash());
+
 	reST_file = fopen(reST_filename.c_str(), "w");
 
 	if ( ! reST_file )
-		fprintf(stderr, "Failed to open %s", reST_filename.c_str());
+		fprintf(stderr, "Failed to open %s\n", reST_filename.c_str());
+
 #ifdef DEBUG
-	else
-		fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str());
+	fprintf(stdout, "Documenting absolute source: %s\n", abs.c_str());
+	fprintf(stdout, "\trelative load: %s\n", rel.c_str());
+	fprintf(stdout, "\tdoc title: %s\n", doc_title.c_str());
+	fprintf(stdout, "\tbro file: %s\n", source_filename.c_str());
+	fprintf(stdout, "\trst file: %s\n", reST_filename.c_str());
 #endif
 	}
 
 BroDoc::~BroDoc()
 	{
 	if ( reST_file && fclose( reST_file ) )
-		fprintf(stderr, "Failed to close %s", reST_filename.c_str());
+		fprintf(stderr, "Failed to close %s\n", reST_filename.c_str());
 
 	FreeBroDocObjPtrList(all);
 	}
@@ -98,7 +137,7 @@ void BroDoc::WriteDocFile() const
 	{
 	WriteToDoc(".. Automatically generated.  Do not edit.\n\n");
 
-	WriteSectionHeading(source_filename.c_str(), '=');
+	WriteSectionHeading(doc_title.c_str(), '=');
 
 	WriteToDoc("\n:download:`Original Source File <%s>`\n\n",
 		source_filename.c_str());

src/BroDoc.h

@@ -22,10 +22,15 @@ public:
 	 * the filename of the Bro script that generates it, except any
 	 * ".bro" file extension is stripped and ".rst" takes it place.
 	 * If the filename doesn't end in ".bro", then ".rst" is just appended.
-	 * @param sourcename The name of the Bro script for which to generate
-	 *        documentation.  May contain a path.
+	 * Any '/' characters in the reST file name that result from choice of
+	 * the 'rel' parameter are replaced with '^'.
+	 * @param rel A string representing the path relative to BROPATH off of
+	 *        which the source file is loaded or generally any filesystem
+	 *        path to a Bro script.  May or may not have .bro file extension.
+	 * @param abs The absolute path to the Bro script for which to generate
+	 *        documentation.
 	 */
-	BroDoc(const std::string& sourcename);
+	BroDoc(const std::string& rel, const std::string& abs);
 
 	/**
 	 * BroDoc destructor
@@ -203,7 +208,8 @@ public:
 protected:
 	FILE* reST_file;
 	std::string reST_filename;
-	std::string source_filename;
+	std::string source_filename; // points to the basename of source file
+	std::string doc_title;
 	std::string packet_filter;
 
 	std::list<std::string> modules;
@@ -357,6 +363,13 @@ private:
 		{
 		return ! o->IsPublicAPI();
 		}
+
+    struct replace_slash {
+        void operator()(char& c)
+            {
+            if ( c == '/' ) c = '^';
+            }
+    };
 };
 
 #endif

src/scan.l

@@ -610,7 +610,7 @@ static int load_files_with_prefix(const char* orig_file)
 
 			if ( generate_documentation )
 				{
-				current_reST_doc = new BroDoc(full_filename);
+				current_reST_doc = new BroDoc(file, full_filename);
 				docs_generated.push_back(current_reST_doc);
 				}
 			}