Merge remote-tracking branch 'origin/topic/jsiwek/zeekygen-code-links'

- Applied minor spelling/grammar suggestions from Johanna during merge

* origin/topic/jsiwek/zeekygen-code-links:
  Change update-zeekygen-docs.sh to set release branch in Sphinx config
  Teach Zeekygen to produce source-code-range information
  Add normalize_script_path() zeek::zeekygen::detail namespace
  Add starts_with()/ends_with() to zeek::util namespace

src/ID.cc

@@ -18,6 +18,7 @@
 #include "zeek/zeekygen/Manager.h"
 #include "zeek/zeekygen/IdentifierInfo.h"
 #include "zeek/zeekygen/ScriptInfo.h"
+#include "zeek/zeekygen/utils.h"
 #include "zeek/module_util.h"
 
 namespace zeek {
@@ -492,7 +493,15 @@ void ID::DescribeReST(ODesc* d, bool roles_only) const
 			d->Add(".. zeek:type:: ");
 		else
 			d->Add(".. zeek:id:: ");
+
 		d->Add(name);
+
+		if ( auto sc = zeek::zeekygen::detail::source_code_range(this) )
+			{
+			d->PushIndent();
+			d->Add(util::fmt(":source-code: %s", sc->data()));
+			d->PopIndentNoNL();
+			}
 		}
 
 	d->PushIndent();

src/util.cc

@@ -78,36 +78,6 @@ static bool can_read(const string& path)
 	return access(path.c_str(), R_OK) == 0;
 	}
 
-static bool starts_with(std::string_view s, std::string_view beginning)
-	{
-	if ( beginning.size() > s.size() )
-		return false;
-
-	return std::equal(beginning.begin(), beginning.end(), s.begin());
-	}
-
-TEST_CASE("util starts_with")
-	{
-	CHECK(starts_with("abcde", "ab") == true);
-	CHECK(starts_with("abcde", "de") == false);
-	CHECK(starts_with("abcde", "abcedf") == false);
-	}
-
-static bool ends_with(std::string_view s, std::string_view ending)
-	{
-	if ( ending.size() > s.size() )
-		return false;
-
-	return std::equal(ending.rbegin(), ending.rend(), s.rbegin());
-	}
-
-TEST_CASE("util ends_with")
-	{
-	CHECK(ends_with("abcde", "de") == true);
-	CHECK(ends_with("abcde", "fg") == false);
-	CHECK(ends_with("abcde", "abcedf") == false);
-	}
-
 static string zeek_path_value;
 
 namespace zeek::util {
@@ -1157,6 +1127,36 @@ int streq(const char* s1, const char* s2)
 	return ! strcmp(s1, s2);
 	}
 
+bool starts_with(std::string_view s, std::string_view beginning)
+	{
+	if ( beginning.size() > s.size() )
+		return false;
+
+	return std::equal(beginning.begin(), beginning.end(), s.begin());
+	}
+
+TEST_CASE("util starts_with")
+	{
+	CHECK(starts_with("abcde", "ab") == true);
+	CHECK(starts_with("abcde", "de") == false);
+	CHECK(starts_with("abcde", "abcedf") == false);
+	}
+
+bool ends_with(std::string_view s, std::string_view ending)
+	{
+	if ( ending.size() > s.size() )
+		return false;
+
+	return std::equal(ending.rbegin(), ending.rend(), s.rbegin());
+	}
+
+TEST_CASE("util ends_with")
+	{
+	CHECK(ends_with("abcde", "de") == true);
+	CHECK(ends_with("abcde", "fg") == false);
+	CHECK(ends_with("abcde", "abcedf") == false);
+	}
+
 char* skip_whitespace(char* s)
 	{
 	while ( *s == ' ' || *s == '\t' )

src/util.h

@@ -311,6 +311,8 @@ std::vector<std::string_view> tokenize_string(std::string_view input, const char
 
 extern char* copy_string(const char* s);
 extern int streq(const char* s1, const char* s2);
+extern bool starts_with(std::string_view s, std::string_view beginning);
+extern bool ends_with(std::string_view s, std::string_view ending);
 
 extern char* skip_whitespace(char* s);
 extern const char* skip_whitespace(const char* s);

src/zeekygen/Manager.cc

@@ -5,8 +5,9 @@
 #include <utility>
 #include <cstdlib>
 
-#include "zeek/plugin/Manager.h"
+#include "zeek/DebugLogger.h"
 #include "zeek/util.h"
+#include "zeek/zeekygen/utils.h"
 #include "zeek/zeekygen/Info.h"
 #include "zeek/zeekygen/PackageInfo.h"
 #include "zeek/zeekygen/ScriptInfo.h"
@@ -52,20 +53,6 @@ static string RemoveLeadingSpace(const string& s)
 	return rval;
 	}
 
-// Turns a script's full path into a shortened, normalized version that we
-// use for indexing.
-static string NormalizeScriptPath(const string& path)
-	{
-	if ( auto p = plugin_mgr->LookupPluginByPath(path) )
-		{
-		auto rval = util::detail::normalize_path(path);
-		auto prefix = util::SafeBasename(p->PluginDirectory()).result;
-		return prefix + "/" + rval.substr(p->PluginDirectory().size() + 1);
-		}
-
-	return util::detail::without_zeekpath_component(path);
-	}
-
 Manager::Manager(const string& arg_config, const string& bro_command)
 	: disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(),
 	  identifiers(), all_info(), last_identifier_seen(), incomplete_type(),
@@ -135,7 +122,7 @@ void Manager::Script(const string& path)
 	if ( disabled )
 		return;
 
-	string name = NormalizeScriptPath(path);
+	string name = normalize_script_path(path);
 
 	if ( scripts.GetInfo(name) )
 		{
@@ -179,8 +166,8 @@ void Manager::ScriptDependency(const string& path, const string& dep)
 		return;
 		}
 
-	string name = NormalizeScriptPath(path);
-	string depname = NormalizeScriptPath(dep);
+	string name = normalize_script_path(path);
+	string depname = normalize_script_path(dep);
 	ScriptInfo* script_info = scripts.GetInfo(name);
 
 	if ( ! script_info )
@@ -204,7 +191,7 @@ void Manager::ModuleUsage(const string& path, const string& module)
 	if ( disabled )
 		return;
 
-	string name = NormalizeScriptPath(path);
+	string name = normalize_script_path(path);
 	ScriptInfo* script_info = scripts.GetInfo(name);
 
 	if ( ! script_info )
@@ -263,7 +250,7 @@ void Manager::StartType(zeek::detail::IDPtr id)
 		return;
 		}
 
-	string script = NormalizeScriptPath(id->GetLocationInfo()->filename);
+	string script = normalize_script_path(id->GetLocationInfo()->filename);
 	ScriptInfo* script_info = scripts.GetInfo(script);
 
 	if ( ! script_info )
@@ -327,7 +314,7 @@ void Manager::Identifier(zeek::detail::IDPtr id, bool from_redef)
 		return;
 		}
 
-	string script = NormalizeScriptPath(id->GetLocationInfo()->filename);
+	string script = normalize_script_path(id->GetLocationInfo()->filename);
 	ScriptInfo* script_info = scripts.GetInfo(script);
 
 	if ( ! script_info )
@@ -357,7 +344,7 @@ void Manager::RecordField(const zeek::detail::ID* id, const TypeDecl* field,
 		return;
 		}
 
-	string script = NormalizeScriptPath(path);
+	string script = normalize_script_path(path);
 	idd->AddRecordField(field, script, comment_buffer, from_redef);
 	comment_buffer.clear();
 	DBG_LOG(DBG_ZEEKYGEN, "Document record field %s, identifier %s, script %s",
@@ -385,7 +372,7 @@ void Manager::Redef(const zeek::detail::ID* id, const string& path,
 		return;
 		}
 
-	string from_script = NormalizeScriptPath(path);
+	string from_script = normalize_script_path(path);
 	ScriptInfo* script_info = scripts.GetInfo(from_script);
 
 	if ( ! script_info )
@@ -413,7 +400,7 @@ void Manager::SummaryComment(const string& script, const string& comment)
 	if ( disabled )
 		return;
 
-	string name = NormalizeScriptPath(script);
+	string name = normalize_script_path(script);
 	ScriptInfo* info = scripts.GetInfo(name);
 
 	if ( info )

src/zeekygen/utils.cc

@@ -6,8 +6,11 @@
 #include <errno.h>
 
 #include "zeek/ID.h"
+#include "zeek/Val.h"
+#include "zeek/Func.h"
 #include "zeek/Scope.h"
 #include "zeek/Reporter.h"
+#include "zeek/plugin/Manager.h"
 
 using namespace std;
 
@@ -137,4 +140,79 @@ string redef_indication(const string& from_script)
 	                 from_script.c_str());
 	}
 
+std::string normalize_script_path(std::string_view path)
+	{
+	if ( auto p = plugin_mgr->LookupPluginByPath(path) )
+		{
+		auto rval = util::detail::normalize_path(path);
+		auto prefix = util::SafeBasename(p->PluginDirectory()).result;
+		return prefix + "/" + rval.substr(p->PluginDirectory().size() + 1);
+		}
+
+	return util::detail::without_zeekpath_component(path);
+	}
+
+std::optional<std::string> source_code_range(const zeek::detail::ID* id)
+	{
+	const auto& type = id->GetType();
+
+	if ( ! type )
+		return {};
+
+	// Some object locations won't end up capturing concrete syntax of closing
+	// braces on subsequent line -- of course that doesn't have to always be
+	// case, but it's true for current code style and the possibility of
+	// capturing an extra line of context is not harmful (human reader shouldn't
+	// be too confused by it).
+	int extra_lines = 0;
+	const zeek::detail::Location* loc = &zeek::detail::no_location;
+
+	switch ( type->Tag() ) {
+	case TYPE_FUNC:
+		{
+		const auto& v = id->GetVal();
+
+		if ( v && v->AsFunc()->GetBodies().size() == 1 )
+			{
+			// Either a function or an event/hook with single body can
+			// report that single, continuous range.
+			loc = v->AsFunc()->GetBodies()[0].stmts->GetLocationInfo();
+			++extra_lines;
+			}
+		else
+			loc = id->GetLocationInfo();
+		}
+		break;
+	case TYPE_ENUM:
+		// Fallthrough
+	case TYPE_RECORD:
+		if ( id->IsType() )
+			{
+			loc = type->GetLocationInfo();
+
+			if ( zeek::util::ends_with(loc->filename, ".bif.zeek") )
+				// Source code won't be available to reference, so fall back
+				// to identifier location which may actually be in a regular
+				// .zeek script.
+				loc = id->GetLocationInfo();
+			else
+				++extra_lines;
+			}
+		else
+			loc = id->GetLocationInfo();
+
+		break;
+	default:
+		loc = id->GetLocationInfo();
+		break;
+	}
+
+	if ( loc == &zeek::detail::no_location )
+		return {};
+
+	return util::fmt("%s %d %d",
+	                 normalize_script_path(loc->filename).data(),
+	                 loc->first_line, loc->last_line + extra_lines);
+	}
+
 } // namespace zeek::zeekygen::detail

src/zeekygen/utils.h

@@ -6,6 +6,7 @@
 
 #include <time.h> // for time_t
 #include <string>
+#include <optional>
 
 namespace zeek::detail { class ID; }
 
@@ -64,4 +65,28 @@ bool is_all_whitespace(const std::string& s);
  */
 std::string redef_indication(const std::string& from_script);
 
+/**
+ * Turns a script's path into a shortened, normalized version that
+ * can be used for indexing and cross-referencing links.
+ *
+ * @param path  the associate path to a Zeek script, which may be absolute.
+ *
+ * @return  a normalized/shortened path (containing no ZEEKPATH components)
+ *
+ */
+std::string normalize_script_path(std::string_view path);
+
+/**
+ * Determines the associated section of source code associated with an
+ * identifier's definition.
+ *
+ * @param id  identifier for which obtain source code location info is obtained
+ *
+ * @return  a nil value if source code location could not be determined, else
+ * a space-separated string with 3 components.  The 1st component is a path
+ * relative to the "scripts/" directory, the 2nd and 3rd components are
+ * line numbers denoting the start and end of the relevant source code.
+ */
+std::optional<std::string> source_code_range(const zeek::detail::ID* id);
+
 } // namespace zeek::zeekygen::detail