Changes to scanner and parser to allow record field comments.

The scanner can now be told to start/stop producing new token types that
assist in documenting record field types (and eventually enums also).

TOK_DOC:
    Produced on "##" style comments; documents the field that follows.

TOK_POST_DOC:
    Produced on "##<" style comments; documents the previous field.

src/input.h

@@ -30,6 +30,8 @@ extern void do_atifdef(const char* id);
 extern void do_atifndef(const char* id);
 extern void do_atelse();
 extern void do_atendif();
+extern void do_doc_token_start();
+extern void do_doc_token_stop();
 
 extern int line_number;
 extern const char* filename;

src/parse.y

@@ -3,7 +3,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 %}
 
-%expect 71
+%expect 74
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ALARM TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -28,6 +28,8 @@
 
 %token TOK_DEBUG
 
+%token TOK_DOC TOK_POST_DOC
+
 %left ',' '|'
 %right '=' TOK_ADD_TO TOK_REMOVE_FROM
 %right '?' ':' TOK_USING
@@ -41,7 +43,7 @@
 %right '!'
 %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR
 
-%type <str> TOK_ID TOK_PATTERN_TEXT single_pattern
+%type <str> TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC opt_doc_list opt_post_doc_list
 %type <id> local_id global_id event_id global_or_event_id resolve_id begin_func
 %type <id_l> local_id_list
 %type <ic> init_class
@@ -137,6 +139,31 @@ static void parser_redef_enum (ID *id)
 		}
 	}
 
+static char* concat_opt_docs (const char* pre, const char* post)
+	{
+	if ( ! pre && ! post )
+		return 0;
+
+	size_t len = 0;
+	if ( pre )
+		len += strlen(pre);
+	if ( post )
+		len += strlen(post);
+	char* s = new char[len + 1];
+	s[0] = '\0';
+	if ( pre )
+		{
+		strcat(s, pre);
+		delete [] pre;
+		}
+	if ( post )
+		{
+		strcat(s, post);
+		delete [] post;
+		}
+	return s;
+	}
+
 %}
 
 %union {
@@ -713,10 +740,11 @@ type:
 				$$ = new SetType($3, 0);
 				}
 
-	|	TOK_RECORD '{' type_decl_list '}'
+	|	TOK_RECORD '{' { do_doc_token_start(); } type_decl_list '}'
 				{
-				set_location(@1, @4);
-				$$ = new RecordType($3);
+				do_doc_token_stop();
+				set_location(@1, @5);
+				$$ = new RecordType($4);
 				}
 
 	|	TOK_UNION '{' type_list '}'
@@ -811,10 +839,12 @@ type_decl_list:
 	;
 
 type_decl:
-		TOK_ID ':' type opt_attr ';'
+		opt_doc_list TOK_ID ':' type opt_attr ';' opt_post_doc_list
 			{
-			set_location(@1, @5);
-			$$ = new TypeDecl($3, $1, $4);
+			set_location(@2, @6);
+			$$ = new TypeDecl($4, $2, $5);
+			if ( generate_documentation )
+				$$->comment = concat_opt_docs($1, $7);
 			}
 	;
 
@@ -1366,6 +1396,30 @@ resolve_id:
 			}
 	;
 
+opt_post_doc_list:
+		opt_post_doc_list TOK_POST_DOC
+			{
+			$$ = concat_opt_docs($1, $2);
+			}
+	|
+		TOK_POST_DOC
+			{ $$ = $1; }
+	|
+			{ $$ = 0; }
+	;
+
+opt_doc_list:
+		opt_doc_list TOK_DOC
+			{
+			$$ = concat_opt_docs($1, $2);
+			}
+	|
+		TOK_DOC
+			{ $$ = $1; }
+	|
+			{ $$ = 0; }
+	;
+
 %%
 
 int yyerror(const char msg[])

src/scan.l

@@ -113,6 +113,7 @@ static void report_file();
 
 %x RE
 %x IGNORE
+%s DOC
 
 OWS	[ \t]*
 WS	[ \t]+
@@ -143,6 +144,17 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 		);
 	}
 
+<DOC>##<.* {
+   yylval.str = copy_string(yytext + 3);
+   return TOK_POST_DOC;
+}
+
+<DOC>##[^#\n].* {
+   yylval.str = copy_string(yytext + 2);
+   return TOK_DOC;
+}
+
+
 ##[^#\n].* {
 	if ( generate_documentation )
 		{
@@ -156,7 +168,7 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 
 {WS}	/* eat whitespace */
 
-<INITIAL,IGNORE>\n	{
+<INITIAL,IGNORE,DOC>\n	{
 			report_file();
 			++line_number;
 			++yylloc.first_line;
@@ -664,6 +676,16 @@ void do_atendif()
 	--current_depth;
 	}
 
+void do_doc_token_start()
+   {
+   if ( generate_documentation ) BEGIN(DOC);
+   }
+
+void do_doc_token_stop()
+   {
+   if ( generate_documentation ) BEGIN(INITIAL);
+   }
+
 // Be careful to never delete things from this list, as the strings
 // are referred to (in order to save the locations of tokens and statements,
 // for error reporting and debugging).