# HG changeset patch # User John W. Eaton # Date 1590688117 14400 # Node ID 59dfd9ed72a3bc23306f9a1050c9e31094614327 # Parent a3ea758870dcd6419b00e3b352f99087f0fe1034 capture comments for classdef classes, properties, events, and enumerations * parse.h, oct-parse.yy: Refactor parser rules for collecting comments and docstrings for classdef parse tree elements. * pt-classdef.h, pt-classdef.cc (tree_classdef_property, tree_classdef_event, tree_classdef_enum): Store comments and docstrings and provide access to them. (tree_classdef_body): Store docstring for classdef object. (tree_classdef): Provide access to classdef docstring from classdef body object. diff -r a3ea758870dc -r 59dfd9ed72a3 libinterp/parse-tree/oct-parse.yy --- a/libinterp/parse-tree/oct-parse.yy Fri May 29 11:00:41 2020 -0400 +++ b/libinterp/parse-tree/oct-parse.yy Thu May 28 13:48:37 2020 -0400 @@ -1731,11 +1731,16 @@ { YYUSE ($5); + octave::comment_list *lc = $2; + octave::comment_list *tc = lexer.get_comment (); + lexer.m_parsing_classdef = false; - if (! ($$ = parser.make_classdef ($1, $3, $4, $6, $7, $8, $2))) + if (! ($$ = parser.make_classdef ($1, $3, $4, $6, $7, $8, + lc, tc))) { - // make_classdef deleted $3, $4, $6, and $7. + // make_classdef deleted $3, $4, $6, $7, LC, and + // TC. YYABORT; } } @@ -1868,14 +1873,18 @@ ; properties_block - : PROPERTIES opt_sep stash_comment attr_list property_list END + : PROPERTIES stash_comment opt_sep attr_list property_list END { - YYUSE ($2); + YYUSE ($3); + + octave::comment_list *lc = $2; + octave::comment_list *tc = lexer.get_comment (); if (! ($$ = parser.make_classdef_properties_block - ($1, $4, $5, $6, $3))) + ($1, $4, $5, $6, lc, tc))) { - // make_classdef_properties_block deleted $4 and $5. + // make_classdef_properties_block deleted $4, + // $5, LC, and TC. YYABORT; } } @@ -1898,30 +1907,60 @@ { YYUSE ($2); + // We don't look ahead to grab end-of-line comments. + // Instead, they are grabbed when we see the + // identifier that becomes the next element in the + // list. If the element at the end of the list + // doesn't have a doc string, see whether the + // element we are adding is stroing an end-of-line + // comment for us to use. + + octave::tree_classdef_property *last_elt = $1->back (); + + if (! last_elt->have_doc_string ()) + { + octave::comment_list *cl = $3->comments (); + + if (cl) + { + octave::comment_elt elt = cl->front (); + + if (elt.is_end_of_line ()) + last_elt->doc_string (elt.text ()); + } + } + $1->append ($3); $$ = $1; } ; -class_property : identifier - { $$ = new octave::tree_classdef_property ($1); } - | identifier '=' decl_param_init expression +class_property : stash_comment identifier { - YYUSE ($2); + $$ = new octave::tree_classdef_property ($2, $1); + } + | stash_comment identifier '=' decl_param_init expression + { + YYUSE ($3); lexer.m_looking_at_initializer_expression = false; - $$ = new octave::tree_classdef_property ($1, $4); + + $$ = new octave::tree_classdef_property ($2, $5, $1); } ; -methods_block : METHODS opt_sep stash_comment attr_list methods_list END +methods_block : METHODS stash_comment opt_sep attr_list methods_list END { - YYUSE ($2); + YYUSE ($3); + + octave::comment_list *lc = $2; + octave::comment_list *tc = lexer.get_comment (); if (! ($$ = parser.make_classdef_methods_block - ($1, $4, $5, $6, $3))) + ($1, $4, $5, $6, lc, tc))) { - // make_classdef_methods_block deleted $4 and $5. + // make_classdef_methods_block deleted $4, $5, + // LC, and TC. YYABORT; } } @@ -1952,6 +1991,7 @@ { lexer.m_defining_func--; lexer.m_parsed_function_name.pop (); + $$ = parser.finish_classdef_external_method ($5, $2, $1); } ; @@ -1994,14 +2034,18 @@ } ; -events_block : EVENTS opt_sep stash_comment attr_list events_list END +events_block : EVENTS stash_comment opt_sep attr_list events_list END { - YYUSE ($2); + YYUSE ($3); + + octave::comment_list *lc = $2; + octave::comment_list *tc = lexer.get_comment (); if (! ($$ = parser.make_classdef_events_block - ($1, $4, $5, $6, $3))) + ($1, $4, $5, $6, lc, tc))) { - // make_classdef_events_block deleted $4 and $5. + // make_classdef_events_block deleted $4, $5, + // LC, and TC. YYABORT; } } @@ -2028,18 +2072,22 @@ } ; -class_event : identifier - { $$ = new octave::tree_classdef_event ($1); } +class_event : stash_comment identifier + { $$ = new octave::tree_classdef_event ($2, $1); } ; -enum_block : ENUMERATION opt_sep stash_comment attr_list enum_list END +enum_block : ENUMERATION stash_comment opt_sep attr_list enum_list END { - YYUSE ($2); + YYUSE ($3); + + octave::comment_list *lc = $2; + octave::comment_list *tc = lexer.get_comment (); if (! ($$ = parser.make_classdef_enum_block - ($1, $4, $5, $6, $3))) + ($1, $4, $5, $6, lc, tc))) { - // make_classdef_enum_block deleted $3 and $4. + // make_classdef_enum_block deleted $4, $5, LC, + // and TC. YYABORT; } } @@ -2066,12 +2114,12 @@ } ; -class_enum : identifier '(' expression ')' +class_enum : stash_comment identifier '(' expression ')' { - YYUSE ($2); - YYUSE ($4); - - $$ = new octave::tree_classdef_enum ($1, $3); + YYUSE ($3); + YYUSE ($5); + + $$ = new octave::tree_classdef_enum ($2, $4, $1); } ; @@ -3787,13 +3835,17 @@ // and methods, and adding to the list of known objects) and creates // a parse tree containing meta information about the class. + // LC contains comments appearing before the classdef keyword. + // TC contains comments appearing between the classdef elements + // and the final end token for the classdef block. + tree_classdef * base_parser::make_classdef (token *tok_val, tree_classdef_attribute_list *a, tree_identifier *id, tree_classdef_superclass_list *sc, tree_classdef_body *body, token *end_tok, - comment_list *lc) + comment_list *lc, comment_list *tc) { tree_classdef *retval = nullptr; @@ -3817,6 +3869,8 @@ delete id; delete sc; delete body; + delete lc; + delete tc; bison_error ("invalid classdef definition, the class name must match the filename", l, c); @@ -3825,8 +3879,6 @@ { if (end_token_ok (end_tok, token::classdef_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); - int l = tok_val->line (); int c = tok_val->column (); @@ -3843,6 +3895,8 @@ delete id; delete sc; delete body; + delete lc; + delete tc; end_token_error (end_tok, token::switch_end); } @@ -3851,23 +3905,54 @@ return retval; } + // LC contains comments appearing before the properties keyword. + // If this properties block appears first in the list of classdef + // elements, this comment list will be used for the help text for the + // classdef block. + + // TC contains comments appearing between the list of properties + // and the final end token for the properties block and may be used to + // find the doc string for the final property in the list. + tree_classdef_properties_block * base_parser::make_classdef_properties_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_property_list *plist, token *end_tok, - comment_list *lc) + comment_list *lc, + comment_list *tc) { tree_classdef_properties_block *retval = nullptr; if (end_token_ok (end_tok, token::properties_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); - int l = tok_val->line (); int c = tok_val->column (); - if (! plist) + if (plist) + { + // If the element at the end of the list doesn't have a doc + // string, see whether the first element of TC is an + // end-of-line comment for us to use. + + if (tc) + { + tree_classdef_property *last_elt = plist->back (); + + if (! last_elt->have_doc_string ()) + { + comment_elt first_comment_elt = tc->front (); + + if (first_comment_elt.is_end_of_line ()) + { + std::string eol_comment = first_comment_elt.text (); + + last_elt->doc_string (eol_comment); + } + } + } + } + else plist = new tree_classdef_property_list (); retval = new tree_classdef_properties_block (a, plist, lc, tc, l, c); @@ -3876,6 +3961,8 @@ { delete a; delete plist; + delete lc; + delete tc; end_token_error (end_tok, token::properties_end); } @@ -3883,19 +3970,22 @@ return retval; } + // LC contains comments appearing before the methods keyword. + // If this methods block appears first in the list of classdef + // elements, this comment list will be used for the help text for the + // classdef block. + tree_classdef_methods_block * base_parser::make_classdef_methods_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_methods_list *mlist, - token *end_tok, - comment_list *lc) + token *end_tok, comment_list *lc, + comment_list *tc) { tree_classdef_methods_block *retval = nullptr; if (end_token_ok (end_tok, token::methods_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); - int l = tok_val->line (); int c = tok_val->column (); @@ -3908,6 +3998,8 @@ { delete a; delete mlist; + delete lc; + delete tc; end_token_error (end_tok, token::methods_end); } @@ -3915,19 +4007,27 @@ return retval; } + // LC contains comments appearing before the events keyword. + // If this events block appears first in the list of classdef + // elements, this comment list will be used for the help text for the + // classdef block. + + // TC contains comments appearing between the list of events and + // the final end token for the events block and may be used to find + // the doc string for the final event in the list. + tree_classdef_events_block * base_parser::make_classdef_events_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_events_list *elist, token *end_tok, - comment_list *lc) + comment_list *lc, + comment_list *tc) { tree_classdef_events_block *retval = nullptr; if (end_token_ok (end_tok, token::events_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); - int l = tok_val->line (); int c = tok_val->column (); @@ -3940,6 +4040,8 @@ { delete a; delete elist; + delete lc; + delete tc; end_token_error (end_tok, token::events_end); } @@ -3947,19 +4049,28 @@ return retval; } + // LC contains comments appearing before the enumeration keyword. + // If this enumeration block appears first in the list of classdef + // elements, this comment list will be used for the help text for the + // classdef block. + + // TC contains comments appearing between the list of + // enumerations and the final end token for the enumeration block and + // may be used to find the doc string for the final enumeration in the + // list. + tree_classdef_enum_block * base_parser::make_classdef_enum_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_enum_list *elist, token *end_tok, - comment_list *lc) + comment_list *lc, + comment_list *tc) { tree_classdef_enum_block *retval = nullptr; if (end_token_ok (end_tok, token::enumeration_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); - int l = tok_val->line (); int c = tok_val->column (); @@ -3972,6 +4083,8 @@ { delete a; delete elist; + delete lc; + delete tc; end_token_error (end_tok, token::enumeration_end); } diff -r a3ea758870dc -r 59dfd9ed72a3 libinterp/parse-tree/parse.h --- a/libinterp/parse-tree/parse.h Fri May 29 11:00:41 2020 -0400 +++ b/libinterp/parse-tree/parse.h Thu May 28 13:48:37 2020 -0400 @@ -353,31 +353,35 @@ make_classdef (token *tok_val, tree_classdef_attribute_list *a, tree_identifier *id, tree_classdef_superclass_list *sc, tree_classdef_body *body, token *end_tok, - comment_list *lc); + comment_list *lc, comment_list *tc); tree_classdef_properties_block * make_classdef_properties_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_property_list *plist, - token *end_tok, comment_list *lc); + token *end_tok, comment_list *lc, + comment_list *tc); tree_classdef_methods_block * make_classdef_methods_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_methods_list *mlist, - token *end_tok, comment_list *lc); + token *end_tok, comment_list *lc, + comment_list *tc); tree_classdef_events_block * make_classdef_events_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_events_list *elist, - token *end_tok, comment_list *lc); + token *end_tok, comment_list *lc, + comment_list *tc); tree_classdef_enum_block * make_classdef_enum_block (token *tok_val, tree_classdef_attribute_list *a, tree_classdef_enum_list *elist, - token *end_tok, comment_list *lc); + token *end_tok, comment_list *lc, + comment_list *tc); octave_user_function * start_classdef_external_method (tree_identifier *id, diff -r a3ea758870dc -r 59dfd9ed72a3 libinterp/parse-tree/pt-classdef.cc --- a/libinterp/parse-tree/pt-classdef.cc Fri May 29 11:00:41 2020 -0400 +++ b/libinterp/parse-tree/pt-classdef.cc Thu May 28 13:48:37 2020 -0400 @@ -27,6 +27,8 @@ # include "config.h" #endif +#include + #include "ov.h" #include "ov-classdef.h" #include "pt-classdef.h" @@ -119,6 +121,35 @@ // Classdef property + std::string check_for_doc_string (comment_list *comments) + { + // If the comment list ends in a block comment or full-line comment, + // then it is the doc string for this property. + + if (comments) + { + comment_elt last_elt = comments->back (); + + if (last_elt.is_block () || last_elt.is_full_line ()) + return last_elt.text (); + } + + return ""; + } + + tree_classdef_property::tree_classdef_property (tree_identifier *i, + comment_list *comments) + : m_id (i), m_expr (nullptr), m_comments (comments), + m_doc_string (check_for_doc_string (m_comments)) + { } + + tree_classdef_property::tree_classdef_property (tree_identifier *i, + tree_expression *e, + comment_list *comments) + : m_id (i), m_expr (e), m_comments (comments), + m_doc_string (check_for_doc_string (m_comments)) + { } + // Classdef property_list tree_classdef_property_list::~tree_classdef_property_list (void) @@ -139,6 +170,12 @@ // Classdef event + tree_classdef_event::tree_classdef_event (tree_identifier *i, + comment_list *comments) + : m_id (i), m_comments (comments), + m_doc_string (check_for_doc_string (m_comments)) + { } + // Classdef events_list tree_classdef_events_list::~tree_classdef_events_list (void) @@ -155,6 +192,13 @@ // Classdef enum + tree_classdef_enum::tree_classdef_enum (tree_identifier *i, + tree_expression *e, + comment_list *comments) + : m_id (i), m_expr (e), m_comments (comments), + m_doc_string (check_for_doc_string (m_comments)) + { } + // Classdef enum_list tree_classdef_enum_list::~tree_classdef_enum_list (void) @@ -171,6 +215,38 @@ // Classdef body + tree_classdef_body::tree_classdef_body (void) + : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst () + { } + + tree_classdef_body::tree_classdef_body (tree_classdef_properties_block *pb) + : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst (), + m_doc_string (pb ? get_doc_string (pb->leading_comment ()) : "") + { + append (pb); + } + + tree_classdef_body::tree_classdef_body (tree_classdef_methods_block *mb) + : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst (), + m_doc_string (mb ? get_doc_string (mb->leading_comment ()) : "") + { + append (mb); + } + + tree_classdef_body::tree_classdef_body (tree_classdef_events_block *evb) + : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst (), + m_doc_string (evb ? get_doc_string (evb->leading_comment ()) : "") + { + append (evb); + } + + tree_classdef_body::tree_classdef_body (tree_classdef_enum_block *enb) + : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst (), + m_doc_string (enb ? get_doc_string (enb->leading_comment ()) : "") + { + append (enb); + } + tree_classdef_body::~tree_classdef_body (void) { while (! m_properties_lst.empty ()) @@ -202,6 +278,22 @@ } } + std::string + tree_classdef_body::get_doc_string (comment_list *comments) const + { + // Grab the first comment from the list and use it as the doc string + // for this classdef body. + + if (comments) + { + comment_elt first_elt = comments->front (); + + return first_elt.text (); + } + + return ""; + } + // Classdef octave_value diff -r a3ea758870dc -r 59dfd9ed72a3 libinterp/parse-tree/pt-classdef.h --- a/libinterp/parse-tree/pt-classdef.h Fri May 29 11:00:41 2020 -0400 +++ b/libinterp/parse-tree/pt-classdef.h Thu May 28 13:48:37 2020 -0400 @@ -272,9 +272,8 @@ public: tree_classdef_element (tree_classdef_attribute_list *a, - base_list *elist, - comment_list *lc, comment_list *tc, - int l = -1, int c = -1) + base_list *elist, comment_list *lc, + comment_list *tc, int l = -1, int c = -1) : tree (l, c), m_attr_list (a), m_elt_list (elist), m_lead_comm (lc), m_trail_comm (tc) { } @@ -311,10 +310,10 @@ // The list of objects contained in this block. base_list *m_elt_list; - // Comment preceding the token marking the beginning of the block. + // Comments preceding the token marking the beginning of the block. comment_list *m_lead_comm; - // Comment preceding END token. + // Comments preceding the END token marking the end of the block. comment_list *m_trail_comm; }; @@ -322,10 +321,11 @@ { public: - tree_classdef_property (tree_identifier *i = nullptr, - tree_expression *e = nullptr) - : m_id (i), m_expr (e) - { } + tree_classdef_property (tree_identifier *i, + comment_list *comments = nullptr); + + tree_classdef_property (tree_identifier *i, tree_expression *e, + comment_list *comments = nullptr); // No copying! @@ -343,6 +343,14 @@ tree_expression * expression (void) { return m_expr; } + comment_list * comments (void) const { return m_comments; } + + void doc_string (const std::string& txt) { m_doc_string = txt; } + + std::string doc_string (void) const { return m_doc_string; } + + bool have_doc_string (void) const { return ! m_doc_string.empty (); } + void accept (tree_walker& tw) { tw.visit_classdef_property (*this); @@ -352,6 +360,8 @@ tree_identifier *m_id; tree_expression *m_expr; + comment_list *m_comments; + std::string m_doc_string; }; class tree_classdef_property_list : public base_list @@ -387,8 +397,7 @@ tree_classdef_properties_block (tree_classdef_attribute_list *a, tree_classdef_property_list *plist, - comment_list *lc, - comment_list *tc, + comment_list *lc, comment_list *tc, int l = -1, int c = -1) : tree_classdef_element (a, plist, lc, tc, l, c) { } @@ -440,8 +449,8 @@ tree_classdef_methods_block (tree_classdef_attribute_list *a, tree_classdef_methods_list *mlist, - comment_list *lc, - comment_list *tc, int l = -1, int c = -1) + comment_list *lc, comment_list *tc, + int l = -1, int c = -1) : tree_classdef_element (a, mlist, lc, tc, l, c) { } @@ -464,7 +473,8 @@ { public: - tree_classdef_event (tree_identifier *i = nullptr) : m_id (i) { } + tree_classdef_event (tree_identifier *i = nullptr, + comment_list *comments = nullptr); // No copying! @@ -479,6 +489,14 @@ tree_identifier * ident (void) { return m_id; } + comment_list * comments (void) const { return m_comments; } + + void doc_string (const std::string& txt) { m_doc_string = txt; } + + std::string doc_string (void) const { return m_doc_string; } + + bool have_doc_string (void) const { return ! m_doc_string.empty (); } + void accept (tree_walker& tw) { tw.visit_classdef_event (*this); @@ -487,6 +505,8 @@ private: tree_identifier *m_id; + comment_list *m_comments; + std::string m_doc_string; }; class tree_classdef_events_list : public base_list @@ -523,8 +543,8 @@ tree_classdef_events_block (tree_classdef_attribute_list *a, tree_classdef_events_list *elist, - comment_list *lc, - comment_list *tc, int l = -1, int c = -1) + comment_list *lc, comment_list *tc, + int l = -1, int c = -1) : tree_classdef_element (a, elist, lc, tc, l, c) { } @@ -547,11 +567,8 @@ { public: - tree_classdef_enum (void) : m_id (nullptr), m_expr (nullptr) { } - - tree_classdef_enum (tree_identifier *i, tree_expression *e) - : m_id (i), m_expr (e) - { } + tree_classdef_enum (tree_identifier *i, tree_expression *e, + comment_list *comments); // No copying! @@ -569,6 +586,14 @@ tree_expression * expression (void) { return m_expr; } + comment_list * comments (void) const { return m_comments; } + + void doc_string (const std::string& txt) { m_doc_string = txt; } + + std::string doc_string (void) const { return m_doc_string; } + + bool have_doc_string (void) const { return ! m_doc_string.empty (); } + void accept (tree_walker& tw) { tw.visit_classdef_enum (*this); @@ -578,6 +603,8 @@ tree_identifier *m_id; tree_expression *m_expr; + comment_list *m_comments; + std::string m_doc_string; }; class tree_classdef_enum_list : public base_list @@ -613,8 +640,8 @@ tree_classdef_enum_block (tree_classdef_attribute_list *a, tree_classdef_enum_list *elist, - comment_list *lc, - comment_list *tc, int l = -1, int c = -1) + comment_list *lc, comment_list *tc, + int l = -1, int c = -1) : tree_classdef_element (a, elist, lc, tc, l, c) { } @@ -655,33 +682,15 @@ typedef std::list::const_iterator enum_list_const_iterator; - tree_classdef_body (void) - : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst () - { } + tree_classdef_body (void); - tree_classdef_body (tree_classdef_properties_block *pb) - : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst () - { - append (pb); - } + tree_classdef_body (tree_classdef_properties_block *pb); - tree_classdef_body (tree_classdef_methods_block *mb) - : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst () - { - append (mb); - } + tree_classdef_body (tree_classdef_methods_block *mb); - tree_classdef_body (tree_classdef_events_block *evb) - : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst () - { - append (evb); - } + tree_classdef_body (tree_classdef_events_block *evb); - tree_classdef_body (tree_classdef_enum_block *enb) - : m_properties_lst (), m_methods_lst (), m_events_lst (), m_enum_lst () - { - append (enb); - } + tree_classdef_body (tree_classdef_enum_block *enb); // No copying! @@ -731,6 +740,12 @@ return m_enum_lst; } + void doc_string (const std::string& txt) { m_doc_string = txt; } + + std::string doc_string (void) const { return m_doc_string; } + + bool have_doc_string (void) const { return ! m_doc_string.empty (); } + void accept (tree_walker& tw) { tw.visit_classdef_body (*this); @@ -738,6 +753,8 @@ private: + std::string get_doc_string (comment_list *comment) const; + std::list m_properties_lst; std::list m_methods_lst; @@ -745,6 +762,8 @@ std::list m_events_lst; std::list m_enum_lst; + + std::string m_doc_string; }; // Classdef definition. @@ -757,9 +776,8 @@ tree_classdef_attribute_list *a, tree_identifier *i, tree_classdef_superclass_list *sc, tree_classdef_body *b, comment_list *lc, - comment_list *tc, - const std::string& pn = "", int l = -1, - int c = -1) + comment_list *tc, const std::string& pn = "", + int l = -1, int c = -1) : tree_command (l, c), m_scope (scope), m_attr_list (a), m_id (i), m_supclass_list (sc), m_element_list (b), m_lead_comm (lc), m_trail_comm (tc), m_pack_name (pn) @@ -796,11 +814,16 @@ comment_list * leading_comment (void) { return m_lead_comm; } comment_list * trailing_comment (void) { return m_trail_comm; } - const std::string& package_name (void) const { return m_pack_name; } + std::string package_name (void) const { return m_pack_name; } octave_value make_meta_class (interpreter& interp, bool is_at_folder = false); + std::string doc_string (void) const + { + return m_element_list ? m_element_list->doc_string () : ""; + } + void accept (tree_walker& tw) { tw.visit_classdef (*this);