Teach Zeekygen to produce source-code-range information

Related to https://github.com/zeek/zeek-docs/issues/56

src/ID.cc

@@ -18,6 +18,7 @@
 #include "zeek/zeekygen/Manager.h"
 #include "zeek/zeekygen/IdentifierInfo.h"
 #include "zeek/zeekygen/ScriptInfo.h"
+#include "zeek/zeekygen/utils.h"
 #include "zeek/module_util.h"
 
 namespace zeek {
@@ -492,7 +493,15 @@ void ID::DescribeReST(ODesc* d, bool roles_only) const
 			d->Add(".. zeek:type:: ");
 		else
 			d->Add(".. zeek:id:: ");
+
 		d->Add(name);
+
+		if ( auto sc = zeek::zeekygen::detail::source_code_range(this) )
+			{
+			d->PushIndent();
+			d->Add(util::fmt(":source-code: %s", sc->data()));
+			d->PopIndentNoNL();
+			}
 		}
 
 	d->PushIndent();

src/zeekygen/utils.cc

@@ -6,6 +6,8 @@
 #include <errno.h>
 
 #include "zeek/ID.h"
+#include "zeek/Val.h"
+#include "zeek/Func.h"
 #include "zeek/Scope.h"
 #include "zeek/Reporter.h"
 #include "zeek/plugin/Manager.h"
@@ -149,4 +151,68 @@ std::string normalize_script_path(std::string_view path)
 
 	return util::detail::without_zeekpath_component(path);
 	}
+
+std::optional<std::string> source_code_range(const zeek::detail::ID* id)
+	{
+	const auto& type = id->GetType();
+
+	if ( ! type )
+		return {};
+
+	// Some object locations won't end up capturing concrete syntax of closing
+	// braces on subsequent line -- of course that doesn't have to always be
+	// case, but it's true for current code style and the possibility of
+	// capturing an extra line of context is not harmful (human reader shouldn't
+	// be too confused by it).
+	int extra_lines = 0;
+	const zeek::detail::Location* loc = &zeek::detail::no_location;
+
+	switch ( type->Tag() ) {
+	case TYPE_FUNC:
+		{
+		const auto& v = id->GetVal();
+
+		if ( v && v->AsFunc()->GetBodies().size() == 1 )
+			{
+			// Either a function or an event/hook with single body can
+			// report that single, contiguous range.
+			loc = v->AsFunc()->GetBodies()[0].stmts->GetLocationInfo();
+			++extra_lines;
+			}
+		else
+			loc = id->GetLocationInfo();
+		}
+		break;
+	case TYPE_ENUM:
+		// Fallthrough
+	case TYPE_RECORD:
+		if ( id->IsType() )
+			{
+			loc = type->GetLocationInfo();
+
+			if ( zeek::util::ends_with(loc->filename, ".bif.zeek") )
+				// Source code won't be availabel to reference, so fall back
+				// to identifier location which may actually be in a regular
+				// .zeek script.
+				loc = id->GetLocationInfo();
+			else
+				++extra_lines;
+			}
+		else
+			loc = id->GetLocationInfo();
+
+		break;
+	default:
+		loc = id->GetLocationInfo();
+		break;
+	}
+
+	if ( loc == &zeek::detail::no_location )
+		return {};
+
+	return util::fmt("%s %d %d",
+	                 normalize_script_path(loc->filename).data(),
+	                 loc->first_line, loc->last_line + extra_lines);
+	}
+
 } // namespace zeek::zeekygen::detail

src/zeekygen/utils.h

@@ -6,6 +6,7 @@
 
 #include <time.h> // for time_t
 #include <string>
+#include <optional>
 
 namespace zeek::detail { class ID; }
 
@@ -74,4 +75,18 @@ std::string redef_indication(const std::string& from_script);
  *
  */
 std::string normalize_script_path(std::string_view path);
+
+/**
+ * Determines the associated section of source code associated with an
+ * identifier's definition.
+ *
+ * @param id  identifier for which obtain source code location info is obtained
+ *
+ * @return  a nil value if source code location could not be determined, else
+ * a space-separated string with 3 components.  The 1st component is a path
+ * relative to the "scripts/" directory, the 2nd and 3rd components are
+ * line numbers denoting the start and end of the relevant source code.
+ */
+std::optional<std::string> source_code_range(const zeek::detail::ID* id);
+
 } // namespace zeek::zeekygen::detail

testing/btest/Baseline/doc.zeekygen.enums/autogen-reST-enums.rst

@@ -1,5 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 .. zeek:type:: TestEnum1
+   :source-code: <...>/enums.zeek 6 15
 
    :Type: :zeek:type:`enum`
 
@@ -31,6 +32,7 @@
    There's tons of ways an enum can look...
 
 .. zeek:type:: TestEnum2
+   :source-code: <...>/enums.zeek 17 26
 
    :Type: :zeek:type:`enum`
 
@@ -52,6 +54,7 @@
    The final comma is optional
 
 .. zeek:id:: TestEnumVal
+   :source-code: <...>/enums.zeek 40 40
 
    :Type: :zeek:type:`TestEnum1`
    :Attributes: :zeek:attr:`&redef`

testing/btest/Baseline/doc.zeekygen.example/example.rst

@@ -111,6 +111,7 @@ Detailed Interface
 Redefinable Options
 ###################
 .. zeek:id:: ZeekygenExample::an_option
+   :source-code: zeekygen/example.zeek 132 132
 
    :Type: :zeek:type:`set` [:zeek:type:`addr`, :zeek:type:`addr`, :zeek:type:`string`]
    :Attributes: :zeek:attr:`&redef`
@@ -120,6 +121,7 @@ Redefinable Options
    The type/attribute information is all generated automatically.
 
 .. zeek:id:: ZeekygenExample::option_with_init
+   :source-code: zeekygen/example.zeek 135 135
 
    :Type: :zeek:type:`interval`
    :Attributes: :zeek:attr:`&redef`
@@ -131,6 +133,7 @@ Redefinable Options
 State Variables
 ###############
 .. zeek:id:: ZeekygenExample::a_var
+   :source-code: zeekygen/example.zeek 140 140
 
    :Type: :zeek:type:`bool`
 
@@ -139,6 +142,7 @@ State Variables
    in the generated docs.
 
 .. zeek:id:: ZeekygenExample::summary_test
+   :source-code: zeekygen/example.zeek 148 148
 
    :Type: :zeek:type:`string`
 
@@ -147,6 +151,7 @@ State Variables
    by the table of all identifiers declared by this script.
 
 .. zeek:id:: ZeekygenExample::var_without_explicit_type
+   :source-code: zeekygen/example.zeek 143 143
 
    :Type: :zeek:type:`string`
    :Default: ``"this works"``
@@ -156,6 +161,7 @@ State Variables
 Types
 #####
 .. zeek:type:: ZeekygenExample::ComplexRecord
+   :source-code: zeekygen/example.zeek 110 117
 
    :Type: :zeek:type:`record`
 
@@ -177,6 +183,7 @@ Types
    General documentation for a type "ComplexRecord" goes here.
 
 .. zeek:type:: ZeekygenExample::Info
+   :source-code: zeekygen/example.zeek 124 128
 
    :Type: :zeek:type:`record`
 
@@ -193,6 +200,7 @@ Types
    (provided they are also @load'ed).
 
 .. zeek:type:: ZeekygenExample::SimpleEnum
+   :source-code: zeekygen/example.zeek 78 85
 
    :Type: :zeek:type:`enum`
 
@@ -219,6 +227,7 @@ Types
    It can span multiple lines.
 
 .. zeek:type:: ZeekygenExample::SimpleRecord
+   :source-code: zeekygen/example.zeek 97 101
 
    :Type: :zeek:type:`record`
 
@@ -239,6 +248,7 @@ Types
 Events
 ######
 .. zeek:id:: ZeekygenExample::an_event
+   :source-code: zeekygen/example.zeek 171 171
 
    :Type: :zeek:type:`event` (name: :zeek:type:`string`)
 
@@ -255,6 +265,7 @@ Events
 Functions
 #########
 .. zeek:id:: ZeekygenExample::a_function
+   :source-code: zeekygen/example.zeek 161 161
 
    :Type: :zeek:type:`function` (tag: :zeek:type:`string`, msg: :zeek:type:`string`) : :zeek:type:`string`
 

testing/btest/Baseline/doc.zeekygen.func-params/autogen-reST-func-params.rst

@@ -1,5 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 .. zeek:id:: test_func_params_func
+   :source-code: <...>/func-params.zeek 11 11
 
    :Type: :zeek:type:`function` (i: :zeek:type:`int`, j: :zeek:type:`int`) : :zeek:type:`string`
 
@@ -14,6 +15,7 @@
    :returns: A string.
 
 .. zeek:type:: test_func_params_rec
+   :source-code: <...>/func-params.zeek 13 21
 
    :Type: :zeek:type:`record`
 

testing/btest/Baseline/doc.zeekygen.identifier/test.rst

@@ -1,5 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 .. zeek:id:: ZeekygenExample::Zeekygen_One
+   :source-code: zeekygen/example.zeek 59 59
 
    :Type: :zeek:type:`Notice::Type`
 
@@ -7,6 +8,7 @@
    will document "Zeekygen_One".
 
 .. zeek:id:: ZeekygenExample::Zeekygen_Two
+   :source-code: zeekygen/example.zeek 60 60
 
    :Type: :zeek:type:`Notice::Type`
 
@@ -14,11 +16,13 @@
    will document "ZEEKYGEN_TWO".
 
 .. zeek:id:: ZeekygenExample::Zeekygen_Three
+   :source-code: zeekygen/example.zeek 62 62
 
    :Type: :zeek:type:`Notice::Type`
 
 
 .. zeek:id:: ZeekygenExample::Zeekygen_Four
+   :source-code: zeekygen/example.zeek 64 64
 
    :Type: :zeek:type:`Notice::Type`
 
@@ -26,11 +30,13 @@
    it's probably best to use only one style consistently.
 
 .. zeek:id:: ZeekygenExample::LOG
+   :source-code: zeekygen/example.zeek 70 70
 
    :Type: :zeek:type:`Log::ID`
 
 
 .. zeek:type:: ZeekygenExample::SimpleEnum
+   :source-code: zeekygen/example.zeek 78 85
 
    :Type: :zeek:type:`enum`
 
@@ -57,6 +63,7 @@
    It can span multiple lines.
 
 .. zeek:id:: ZeekygenExample::ONE
+   :source-code: zeekygen/example.zeek 81 81
 
    :Type: :zeek:type:`ZeekygenExample::SimpleEnum`
 
@@ -64,29 +71,34 @@
    And can also span multiple lines.
 
 .. zeek:id:: ZeekygenExample::TWO
+   :source-code: zeekygen/example.zeek 82 82
 
    :Type: :zeek:type:`ZeekygenExample::SimpleEnum`
 
    Or this style is valid to document the preceding enum value.
 
 .. zeek:id:: ZeekygenExample::THREE
+   :source-code: zeekygen/example.zeek 83 83
 
    :Type: :zeek:type:`ZeekygenExample::SimpleEnum`
 
 
 .. zeek:id:: ZeekygenExample::FOUR
+   :source-code: zeekygen/example.zeek 89 89
 
    :Type: :zeek:type:`ZeekygenExample::SimpleEnum`
 
    And some documentation for "FOUR".
 
 .. zeek:id:: ZeekygenExample::FIVE
+   :source-code: zeekygen/example.zeek 91 91
 
    :Type: :zeek:type:`ZeekygenExample::SimpleEnum`
 
    Also "FIVE".
 
 .. zeek:type:: ZeekygenExample::SimpleRecord
+   :source-code: zeekygen/example.zeek 97 101
 
    :Type: :zeek:type:`record`
 
@@ -105,6 +117,7 @@
    for enums.
 
 .. zeek:type:: ZeekygenExample::ComplexRecord
+   :source-code: zeekygen/example.zeek 110 117
 
    :Type: :zeek:type:`record`
 
@@ -126,6 +139,7 @@
    General documentation for a type "ComplexRecord" goes here.
 
 .. zeek:type:: ZeekygenExample::Info
+   :source-code: zeekygen/example.zeek 124 128
 
    :Type: :zeek:type:`record`
 
@@ -142,6 +156,7 @@
    (provided they are also @load'ed).
 
 .. zeek:id:: ZeekygenExample::an_option
+   :source-code: zeekygen/example.zeek 132 132
 
    :Type: :zeek:type:`set` [:zeek:type:`addr`, :zeek:type:`addr`, :zeek:type:`string`]
    :Attributes: :zeek:attr:`&redef`
@@ -151,6 +166,7 @@
    The type/attribute information is all generated automatically.
 
 .. zeek:id:: ZeekygenExample::option_with_init
+   :source-code: zeekygen/example.zeek 135 135
 
    :Type: :zeek:type:`interval`
    :Attributes: :zeek:attr:`&redef`
@@ -160,14 +176,16 @@
    More docs can be added here.
 
 .. zeek:id:: ZeekygenExample::a_var
+   :source-code: zeekygen/example.zeek 140 140
 
    :Type: :zeek:type:`bool`
 
    Put some documentation for "a_var" here.  Any global/non-const that
-   isn't a function/event/hook is classified as a "state variable"
+   isn't a function<...>/hook is classified as a "state variable"
    in the generated docs.
 
 .. zeek:id:: ZeekygenExample::var_without_explicit_type
+   :source-code: zeekygen/example.zeek 143 143
 
    :Type: :zeek:type:`string`
    :Default: ``"this works"``
@@ -175,6 +193,7 @@
    Types are inferred, that information is self-documenting.
 
 .. zeek:id:: ZeekygenExample::summary_test
+   :source-code: zeekygen/example.zeek 148 148
 
    :Type: :zeek:type:`string`
 
@@ -183,6 +202,7 @@
    by the table of all identifiers declared by this script.
 
 .. zeek:id:: ZeekygenExample::a_function
+   :source-code: zeekygen/example.zeek 161 161
 
    :Type: :zeek:type:`function` (tag: :zeek:type:`string`, msg: :zeek:type:`string`) : :zeek:type:`string`
 
@@ -202,6 +222,7 @@
    :returns: Describe the return type here.
 
 .. zeek:id:: ZeekygenExample::an_event
+   :source-code: zeekygen/example.zeek 171 171
 
    :Type: :zeek:type:`event` (name: :zeek:type:`string`)
 
@@ -216,11 +237,13 @@
    :name: Describe the argument here.
 
 .. zeek:id:: ZeekygenExample::function_without_proto
+   :source-code: zeekygen/example.zeek 176 184
 
    :Type: :zeek:type:`function` (tag: :zeek:type:`string`) : :zeek:type:`string`
 
 
 .. zeek:type:: ZeekygenExample::PrivateRecord
+   :source-code: zeekygen/example.zeek 190 193
 
    :Type: :zeek:type:`record`
 

testing/btest/Baseline/doc.zeekygen.records/autogen-reST-records.rst

@@ -1,5 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 .. zeek:type:: TestRecord1
+   :source-code: <...>/records.zeek 6 9
 
    :Type: :zeek:type:`record`
 
@@ -9,6 +10,7 @@
 
 
 .. zeek:type:: TestRecord2
+   :source-code: <...>/records.zeek 12 23
 
    :Type: :zeek:type:`record`
 

testing/btest/Baseline/doc.zeekygen.type-aliases/autogen-reST-type-aliases.rst

@@ -1,17 +1,20 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 .. zeek:type:: ZeekygenTest::TypeAlias
+   :source-code: <...>/type-aliases.zeek 9 9
 
    :Type: :zeek:type:`bool`
 
    This is just an alias for a builtin type ``bool``.
 
 .. zeek:type:: ZeekygenTest::NotTypeAlias
+   :source-code: <...>/type-aliases.zeek 12 12
 
    :Type: :zeek:type:`bool`
 
    This type should get its own comments, not associated w/ TypeAlias.
 
 .. zeek:type:: ZeekygenTest::OtherTypeAlias
+   :source-code: <...>/type-aliases.zeek 18 18
 
    :Type: :zeek:type:`bool`
 
@@ -21,18 +24,21 @@
    find out what the actual type is...
 
 .. zeek:id:: ZeekygenTest::a
+   :source-code: <...>/type-aliases.zeek 21 21
 
    :Type: :zeek:type:`ZeekygenTest::TypeAlias`
 
    But this should reference a type of ``TypeAlias``.
 
 .. zeek:id:: ZeekygenTest::b
+   :source-code: <...>/type-aliases.zeek 24 24
 
    :Type: :zeek:type:`ZeekygenTest::OtherTypeAlias`
 
    And this should reference a type of ``OtherTypeAlias``.
 
 .. zeek:type:: ZeekygenTest::MyRecord
+   :source-code: <...>/type-aliases.zeek 26 30
 
    :Type: :zeek:type:`record`
 

testing/btest/Baseline/doc.zeekygen.vectors/autogen-reST-vectors.rst

@@ -1,5 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 .. zeek:id:: test_vector0
+   :source-code: <...>/vectors.zeek 11 11
 
    :Type: :zeek:type:`vector` of :zeek:type:`string`
    :Default:
@@ -12,6 +13,7 @@
    Yield type is documented/cross-referenced for primitize types.
 
 .. zeek:id:: test_vector1
+   :source-code: <...>/vectors.zeek 14 14
 
    :Type: :zeek:type:`vector` of :zeek:type:`TestRecord`
    :Default:
@@ -24,6 +26,7 @@
    Yield type is documented/cross-referenced for composite types.
 
 .. zeek:id:: test_vector2
+   :source-code: <...>/vectors.zeek 17 17
 
    :Type: :zeek:type:`vector` of :zeek:type:`vector` of :zeek:type:`TestRecord`
    :Default:

testing/btest/doc/zeekygen/enums.zeek

@@ -1,5 +1,5 @@
 # @TEST-EXEC: unset ZEEK_DISABLE_ZEEKYGEN; unset BRO_DISABLE_BROXYGEN; zeek -b -X zeekygen.config %INPUT
-# @TEST-EXEC: btest-diff autogen-reST-enums.rst
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff autogen-reST-enums.rst
 
 @TEST-START-FILE zeekygen.config
 identifier	TestEnum*	autogen-reST-enums.rst

testing/btest/doc/zeekygen/func-params.zeek

@@ -1,5 +1,5 @@
 # @TEST-EXEC: unset ZEEK_DISABLE_ZEEKYGEN; unset BRO_DISABLE_BROXYGEN; zeek -b -X zeekygen.config %INPUT
-# @TEST-EXEC: btest-diff autogen-reST-func-params.rst
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff autogen-reST-func-params.rst
 
 @TEST-START-FILE zeekygen.config
 identifier	test_func_params*	autogen-reST-func-params.rst

testing/btest/doc/zeekygen/identifier.zeek

@@ -1,6 +1,6 @@
 # @TEST-PORT: BROKER_PORT
 # @TEST-EXEC: unset ZEEK_DISABLE_ZEEKYGEN; unset BRO_DISABLE_BROXYGEN; zeek -b -X zeekygen.config %INPUT Broker::default_port=$BROKER_PORT
-# @TEST-EXEC: btest-diff test.rst
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff test.rst
 
 @TEST-START-FILE zeekygen.config
 identifier	ZeekygenExample::*	test.rst

testing/btest/doc/zeekygen/records.zeek

@@ -1,5 +1,5 @@
 # @TEST-EXEC: unset ZEEK_DISABLE_ZEEKYGEN; unset BRO_DISABLE_BROXYGEN; zeek -b -X zeekygen.config %INPUT
-# @TEST-EXEC: btest-diff autogen-reST-records.rst
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff autogen-reST-records.rst
 
 @TEST-START-FILE zeekygen.config
 identifier	TestRecord*	autogen-reST-records.rst

testing/btest/doc/zeekygen/type-aliases.zeek

@@ -1,5 +1,5 @@
 # @TEST-EXEC: unset ZEEK_DISABLE_ZEEKYGEN; unset BRO_DISABLE_BROXYGEN; zeek -b -X zeekygen.config %INPUT
-# @TEST-EXEC: btest-diff autogen-reST-type-aliases.rst
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff autogen-reST-type-aliases.rst
 
 @TEST-START-FILE zeekygen.config
 identifier	ZeekygenTest::*	autogen-reST-type-aliases.rst

testing/btest/doc/zeekygen/vectors.zeek

@@ -1,5 +1,5 @@
 # @TEST-EXEC: unset ZEEK_DISABLE_ZEEKYGEN; unset BRO_DISABLE_BROXYGEN; zeek -b -X zeekygen.config %INPUT
-# @TEST-EXEC: btest-diff autogen-reST-vectors.rst
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff autogen-reST-vectors.rst
 
 @TEST-START-FILE zeekygen.config
 identifier	test_vector*	autogen-reST-vectors.rst