Mercurial > octave
changeset 33123:6b0f33f522ad bytecode-interpreter
maint: merge default to bytecode-interpreter
| author | John W. Eaton <jwe@octave.org> |
|---|---|
| date | Sat, 02 Mar 2024 15:36:26 -0500 |
| parents | 9acadc2d8336 (current diff) 2b736846a81d (diff) |
| children | 801e5f2f84f3 |
| files | |
| diffstat | 6 files changed, 439 insertions(+), 442 deletions(-) [+] |
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h Thu Feb 29 15:32:28 2024 -0500 +++ b/libinterp/parse-tree/lex.h Sat Mar 02 15:36:26 2024 -0500 @@ -325,9 +325,12 @@ void reset (); - int previous_token_value () const; + int previous_token_id () const; + token * previous_token (); + const token * previous_token () const; - bool previous_token_value_is (int tok_val) const; + bool previous_token_is (int tok_id) const; + bool previous_token_is (const token *tok) const; void mark_previous_token_trailing_space (); @@ -578,7 +581,7 @@ // Caller is expected to delete the returned value. - comment_list * get_comment () + comment_list * get_comment_list () { comment_list *retval = m_comment_list; @@ -644,7 +647,7 @@ bool inside_any_object_index (); - int make_keyword_token (const std::string& s); + token * make_keyword_token (const std::string& s); bool fq_identifier_contains_keyword (const std::string& s); @@ -658,7 +661,7 @@ void finish_comment (comment_elt::comment_type typ); - comment_list * get_comment () { return m_comment_buf.get_comment (); } + comment_list * get_comment_list () { return m_comment_buf.get_comment_list (); } int handle_close_bracket (int bracket_type); @@ -666,9 +669,9 @@ int handle_superclass_identifier (); - int handle_meta_identifier (); + token * make_meta_identifier_token (); - int handle_fq_identifier (); + token * make_fq_identifier_token (); int handle_identifier (); @@ -692,7 +695,7 @@ std::size_t pending_token_count () const; - void display_token (int tok); + void display_token (int tok_id); void fatal_error (const char *msg); @@ -733,19 +736,20 @@ void display_start_state () const; - bool maybe_unput_comma_before_unary_op (int tok); + bool maybe_unput_comma_before_unary_op (int tok_id); - int handle_op (int tok, bool bos = false, bool compat = true); + int handle_op (int tok_id, bool bos = false, bool compat = true); int finish_command_arg (); - int handle_token (int tok, token *tok_val = nullptr); - - int count_token (int tok); + int handle_token (int tok_id, token *tok = nullptr); + int handle_token (token *tok); - int count_token_internal (int tok); + int count_token (int tok_id); - int show_token (int tok); + int count_token_internal (int tok_id); + + int show_token (int tok_id); protected:
--- a/libinterp/parse-tree/lex.ll Thu Feb 29 15:32:28 2024 -0500 +++ b/libinterp/parse-tree/lex.ll Sat Mar 02 15:36:26 2024 -0500 @@ -168,22 +168,22 @@ #define YY_FATAL_ERROR(msg) \ (yyget_extra (yyscanner))->fatal_error (msg) -#define CMD_OR_OP(PATTERN, TOK, COMPAT) \ - do \ - { \ - curr_lexer->lexer_debug (PATTERN); \ - \ - if (curr_lexer->looks_like_command_arg ()) \ - { \ - yyless (0); \ - curr_lexer->push_start_state (COMMAND_START); \ - } \ - else \ - return curr_lexer->handle_op (TOK, false, COMPAT); \ - } \ +#define CMD_OR_OP(PATTERN, TOK_ID, COMPAT) \ + do \ + { \ + curr_lexer->lexer_debug (PATTERN); \ + \ + if (curr_lexer->looks_like_command_arg ()) \ + { \ + yyless (0); \ + curr_lexer->push_start_state (COMMAND_START); \ + } \ + else \ + return curr_lexer->handle_op (TOK_ID, false, COMPAT); \ + } \ while (0) -#define CMD_OR_UNARY_OP(PATTERN, TOK, COMPAT) \ +#define CMD_OR_UNARY_OP(PATTERN, TOK_ID, COMPAT) \ do \ { \ curr_lexer->lexer_debug (PATTERN); \ @@ -196,17 +196,17 @@ curr_lexer->push_start_state (COMMAND_START); \ } \ else \ - return curr_lexer->handle_op (TOK, false, COMPAT); \ + return curr_lexer->handle_op (TOK_ID, false, COMPAT); \ } \ else \ { \ - if (curr_lexer->maybe_unput_comma_before_unary_op (TOK)) \ + if (curr_lexer->maybe_unput_comma_before_unary_op (TOK_ID)) \ { \ yyless (0); \ curr_lexer->xunput (','); \ } \ else \ - return curr_lexer->handle_op (TOK, false, COMPAT); \ + return curr_lexer->handle_op (TOK_ID, false, COMPAT); \ } \ } \ while (0) @@ -225,57 +225,57 @@ } \ while (0) -// If we are at the end of the buffer, ask for more input. -// If we are at the end of the file, deal with it. -// Otherwise, just keep going with the text from the current buffer. -#define HANDLE_STRING_CONTINUATION \ - do \ - { \ - curr_lexer->m_filepos.next_line (); \ - \ - HANDLE_EOB_OR_EOF (-1); \ - } \ + // If we are at the end of the buffer, ask for more input. + // If we are at the end of the file, deal with it. + // Otherwise, just keep going with the text from the current buffer. +#define HANDLE_STRING_CONTINUATION \ + do \ + { \ + curr_lexer->m_filepos.next_line (); \ + \ + HANDLE_EOB_OR_EOF (-1); \ + } \ while (0) -#define HANDLE_NUMBER(PATTERN, BASE) \ - do \ - { \ - curr_lexer->lexer_debug (PATTERN); \ - \ - if (curr_lexer->previous_token_may_be_command () \ - && curr_lexer->space_follows_previous_token ()) \ - { \ - yyless (0); \ - curr_lexer->push_start_state (COMMAND_START); \ - } \ - else \ - { \ - int tok = curr_lexer->previous_token_value (); \ - \ - if (curr_lexer->whitespace_is_significant () \ - && curr_lexer->space_follows_previous_token () \ - && ! (tok == '[' || tok == '{' \ - || curr_lexer->previous_token_is_binop ())) \ - { \ - yyless (0); \ - curr_lexer->xunput (','); \ - } \ - else \ - return curr_lexer->handle_number<BASE> (); \ - } \ - } \ - while (0) +#define HANDLE_NUMBER(PATTERN, BASE) \ + do \ + { \ + curr_lexer->lexer_debug (PATTERN); \ + \ + if (curr_lexer->previous_token_may_be_command () \ + && curr_lexer->space_follows_previous_token ()) \ + { \ + yyless (0); \ + curr_lexer->push_start_state (COMMAND_START); \ + } \ + else \ + { \ + int tok_id = curr_lexer->previous_token_id (); \ + \ + if (curr_lexer->whitespace_is_significant () \ + && curr_lexer->space_follows_previous_token () \ + && ! (tok_id == '[' || tok_id == '{' \ + || curr_lexer->previous_token_is_binop ())) \ + { \ + yyless (0); \ + curr_lexer->xunput (','); \ + } \ + else \ + return curr_lexer->handle_number<BASE> (); \ + } \ + } \ + while (0) #define HANDLE_IDENTIFIER(pattern, get_set) \ do \ { \ curr_lexer->lexer_debug (pattern); \ \ - int tok = curr_lexer->previous_token_value (); \ + int tok_id = curr_lexer->previous_token_id (); \ \ if (curr_lexer->whitespace_is_significant () \ && curr_lexer->space_follows_previous_token () \ - && ! (tok == '[' || tok == '{' \ + && ! (tok_id == '[' || tok_id == '{' \ || curr_lexer->previous_token_is_binop ())) \ { \ yyless (0); \ @@ -590,9 +590,9 @@ curr_lexer->warn_language_extension ("bare newline inside parentheses"); else { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == ';' || tok == '[' || tok == '{')) + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == ';' || tok_id == '[' || tok_id == '{')) curr_lexer->xunput (';'); } } @@ -655,9 +655,9 @@ if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token ()) { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == '[' || tok == '{' + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) unput_comma = true; } @@ -992,11 +992,10 @@ curr_lexer->m_at_beginning_of_statement = false; octave::token *tok = new octave::token (DQ_STRING, curr_lexer->m_string_text, curr_lexer->m_tok_beg, curr_lexer->m_tok_end); - curr_lexer->push_token (tok); curr_lexer->m_string_text = ""; - return curr_lexer->count_token_internal (DQ_STRING); + return curr_lexer->handle_token (tok); } } @@ -1014,9 +1013,7 @@ std::string msg {"invalid octal escape sequence in character string"}; octave::token *tok = new octave::token (LEXICAL_ERROR, msg, curr_lexer->m_tok_beg, curr_lexer->m_tok_end); - curr_lexer->push_token (tok); - - return curr_lexer->count_token_internal (LEXICAL_ERROR); + return curr_lexer->handle_token (tok); } else curr_lexer->m_string_text += static_cast<unsigned char> (result); @@ -1120,11 +1117,9 @@ std::string msg {"unterminated character string constant"}; octave::token *tok = new octave::token (LEXICAL_ERROR, msg, curr_lexer->m_filepos, curr_lexer->m_filepos); - curr_lexer->push_token (tok); - curr_lexer->m_filepos.next_line (); - return curr_lexer->count_token_internal (LEXICAL_ERROR); + return curr_lexer->handle_token (tok); } %{ @@ -1154,11 +1149,9 @@ octave::token *tok = new octave::token (SQ_STRING, curr_lexer->m_string_text, curr_lexer->m_tok_beg, curr_lexer->m_tok_end); - curr_lexer->push_token (tok); - curr_lexer->m_string_text = ""; - return curr_lexer->count_token_internal (SQ_STRING); + return curr_lexer->handle_token (tok); } } @@ -1176,11 +1169,9 @@ std::string msg {"unterminated character string constant"}; octave::token *tok = new octave::token (LEXICAL_ERROR, msg, curr_lexer->m_filepos, curr_lexer->m_filepos); - curr_lexer->push_token (tok); - curr_lexer->m_filepos.next_line (); - return curr_lexer->count_token_internal (LEXICAL_ERROR); + return curr_lexer->handle_token (tok); } %{ @@ -1194,14 +1185,9 @@ curr_lexer->update_token_positions (yyleng); - int id_tok = curr_lexer->handle_fq_identifier (); - - if (id_tok >= 0) - { - curr_lexer->m_looking_for_object_index = true; - - return curr_lexer->count_token_internal (id_tok); - } + octave::token *tok = curr_lexer->make_fq_identifier_token (); + + return curr_lexer->handle_token (tok); } <FQ_IDENT_START>{S}+ { @@ -1355,14 +1341,9 @@ { curr_lexer->update_token_positions (yyleng); - int id_tok = curr_lexer->handle_meta_identifier (); - - if (id_tok >= 0) - { - curr_lexer->m_looking_for_object_index = true; - - return curr_lexer->count_token_internal (id_tok); - } + octave::token *tok = curr_lexer->make_meta_identifier_token (); + + return curr_lexer->handle_token (tok); } } @@ -1378,11 +1359,11 @@ } else { - int tok_val = curr_lexer->previous_token_value (); + int tok_id = curr_lexer->previous_token_id (); if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token () - && ! (tok_val == '[' || tok_val == '{' + && ! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) { yyless (0); @@ -1423,9 +1404,7 @@ tok = new octave::token (FCN_HANDLE, ident, curr_lexer->m_tok_beg, curr_lexer->m_tok_end); } - curr_lexer->push_token (tok); - - return curr_lexer->count_token_internal (tok->token_value ()); + return curr_lexer->handle_token (tok); } } } @@ -1467,9 +1446,7 @@ std::string msg {"unexpected internal lexer error"}; octave::token *tok = new octave::token (LEXICAL_ERROR, msg, curr_lexer->m_filepos, curr_lexer->m_filepos); - curr_lexer->push_token (tok); - - return curr_lexer->count_token_internal (LEXICAL_ERROR); + return curr_lexer->handle_token (tok); } } @@ -1495,13 +1472,13 @@ } else { - int tok = curr_lexer->previous_token_value (); + int tok_id = curr_lexer->previous_token_id (); if (curr_lexer->whitespace_is_significant ()) { if (curr_lexer->space_follows_previous_token ()) { - if (tok == '[' || tok == '{' + if (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ()) { curr_lexer->m_filepos.increment_column (); @@ -1515,7 +1492,7 @@ } else { - if (tok == '[' || tok == '{' + if (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop () || curr_lexer->previous_token_is_keyword ()) { @@ -1531,7 +1508,7 @@ } else { - if (! tok || tok == '[' || tok == '{' || tok == '(' + if (! tok_id || tok_id == '[' || tok_id == '{' || tok_id == '(' || curr_lexer->previous_token_is_binop () || curr_lexer->previous_token_is_keyword ()) { @@ -1563,13 +1540,13 @@ } else { - int tok = curr_lexer->previous_token_value (); + int tok_id = curr_lexer->previous_token_id (); if (curr_lexer->whitespace_is_significant ()) { if (curr_lexer->space_follows_previous_token ()) { - if (tok == '[' || tok == '{' + if (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ()) { curr_lexer->m_filepos.increment_column (); @@ -1679,9 +1656,9 @@ if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token ()) { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == '[' || tok == '{' + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) unput_comma = true; } @@ -1789,9 +1766,9 @@ if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token ()) { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == '[' || tok == '{' + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) unput_comma = true; } @@ -1871,11 +1848,9 @@ std::string msg {"unexpected internal lexer error"}; octave::token *tok = new octave::token (LEXICAL_ERROR, buf.str (), msg, curr_lexer->m_filepos, curr_lexer->m_filepos); - curr_lexer->push_token (tok); - curr_lexer->m_filepos.increment_column (); - return curr_lexer->count_token_internal (LEXICAL_ERROR); + return curr_lexer->handle_token (tok); } } @@ -2231,60 +2206,79 @@ m_tokens.clear (); } - int - lexical_feedback::previous_token_value () const + token * + lexical_feedback::previous_token () + { + return m_tokens.front (); + } + + const token * + lexical_feedback::previous_token () const { - const token *tok = m_tokens.front (); - return tok ? tok->token_value () : 0; + return m_tokens.front (); + } + + int + lexical_feedback::previous_token_id () const + { + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->token_id () : 0; } bool - lexical_feedback::previous_token_value_is (int tok_val) const + lexical_feedback::previous_token_is (int tok_id) const { - const token *tok = m_tokens.front (); - return tok ? tok->token_value_is (tok_val) : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->token_is (tok_id) : false; + } + + bool + lexical_feedback::previous_token_is (const token *tok) const + { + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->token_is (tok) : false; } void lexical_feedback::mark_previous_token_trailing_space () { - token *tok = m_tokens.front (); - if (tok && ! previous_token_value_is ('\n')) - tok->mark_trailing_space (); + token *prev_tok = previous_token (); + if (prev_tok && ! previous_token_is ('\n')) + prev_tok->mark_trailing_space (); } bool lexical_feedback::space_follows_previous_token () const { - const token *tok = m_tokens.front (); - return tok ? tok->space_follows_token () : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->space_follows_token () : false; } bool lexical_feedback::previous_token_is_binop () const { - int tok = previous_token_value (); - - return (tok == '+' || tok == '-' || tok == '@' || tok == '~' || tok == '!' - || tok == ',' || tok == ';' || tok == '*' || tok == '/' - || tok == ':' || tok == '=' || tok == ADD_EQ - || tok == AND_EQ || tok == DIV_EQ || tok == EDIV - || tok == EDIV_EQ || tok == ELEFTDIV || tok == ELEFTDIV_EQ - || tok == EMUL || tok == EMUL_EQ - || tok == EPOW || tok == EPOW_EQ || tok == EXPR_AND - || tok == EXPR_AND_AND || tok == EXPR_EQ || tok == EXPR_GE - || tok == EXPR_GT || tok == EXPR_LE || tok == EXPR_LT - || tok == EXPR_NE || tok == EXPR_OR - || tok == EXPR_OR_OR || tok == LEFTDIV || tok == LEFTDIV_EQ - || tok == MUL_EQ || tok == OR_EQ || tok == POW - || tok == POW_EQ || tok == SUB_EQ); + int tok_id = previous_token_id (); + + return (tok_id == '+' || tok_id == '-' || tok_id == '@' || tok_id == '~' || tok_id == '!' + || tok_id == ',' || tok_id == ';' || tok_id == '*' || tok_id == '/' + || tok_id == ':' || tok_id == '=' || tok_id == ADD_EQ + || tok_id == AND_EQ || tok_id == DIV_EQ || tok_id == EDIV + || tok_id == EDIV_EQ || tok_id == ELEFTDIV || tok_id == ELEFTDIV_EQ + || tok_id == EMUL || tok_id == EMUL_EQ + || tok_id == EPOW || tok_id == EPOW_EQ || tok_id == EXPR_AND + || tok_id == EXPR_AND_AND || tok_id == EXPR_EQ || tok_id == EXPR_GE + || tok_id == EXPR_GT || tok_id == EXPR_LE || tok_id == EXPR_LT + || tok_id == EXPR_NE || tok_id == EXPR_OR + || tok_id == EXPR_OR_OR || tok_id == LEFTDIV || tok_id == LEFTDIV_EQ + || tok_id == MUL_EQ || tok_id == OR_EQ || tok_id == POW + || tok_id == POW_EQ || tok_id == SUB_EQ); } bool lexical_feedback::previous_token_is_keyword () const { - const token *tok = m_tokens.front (); - return tok ? tok->iskeyword () : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->iskeyword () : false; } void @@ -2311,8 +2305,8 @@ if (! m_allow_command_syntax) return false; - const token *tok = m_tokens.front (); - return tok ? tok->may_be_command () : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->may_be_command () : false; } static bool @@ -2487,11 +2481,9 @@ error ("block comment unterminated at end of input"); } - token *tok_val = new token (END_OF_INPUT, m_tok_beg, m_tok_end); - - push_token (tok_val); - - return count_token_internal (END_OF_INPUT); + token *tok = new token (END_OF_INPUT, m_tok_beg, m_tok_end); + + return handle_token (tok); } char * @@ -2602,7 +2594,7 @@ return retval; } - int + token * base_lexer::make_keyword_token (const std::string& s) { // Token positions should have already been updated before this @@ -2613,14 +2605,14 @@ const octave_kw *kw = octave_kw_hash::in_word_set (s.c_str (), slen); if (! kw) - return 0; + return nullptr; bool previous_at_bos = m_at_beginning_of_statement; // May be reset to true for some token types. m_at_beginning_of_statement = false; - token *tok_val = nullptr; + token *tok = nullptr; switch (kw->kw_id) { @@ -2651,84 +2643,84 @@ || m_parsed_function_name.top ()))) { m_at_beginning_of_statement = previous_at_bos; - return 0; + return nullptr; } - tok_val = new token (kw->tok, token::simple_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::simple_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case end_try_catch_kw: - tok_val = new token (kw->tok, token::try_catch_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::try_catch_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case end_unwind_protect_kw: - tok_val = new token (kw->tok, token::unwind_protect_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::unwind_protect_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endfor_kw: - tok_val = new token (kw->tok, token::for_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::for_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endfunction_kw: - tok_val = new token (kw->tok, token::function_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::function_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endif_kw: - tok_val = new token (kw->tok, token::if_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::if_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endparfor_kw: - tok_val = new token (kw->tok, token::parfor_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::parfor_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endswitch_kw: - tok_val = new token (kw->tok, token::switch_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::switch_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endwhile_kw: - tok_val = new token (kw->tok, token::while_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::while_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endarguments_kw: #if defined (DISABLE_ARGUMENTS_VALIDATION_BLOCK) - return 0; + return nullptr; #else - tok_val = new token (kw->tok, token::arguments_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::arguments_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; #endif case endclassdef_kw: - tok_val = new token (kw->tok, token::classdef_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::classdef_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endenumeration_kw: - tok_val = new token (kw->tok, token::enumeration_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::enumeration_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endevents_kw: - tok_val = new token (kw->tok, token::events_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::events_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endmethods_kw: - tok_val = new token (kw->tok, token::methods_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::methods_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endproperties_kw: - tok_val = new token (kw->tok, token::properties_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::properties_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; @@ -2759,7 +2751,7 @@ if (! m_maybe_classdef_get_set_method) { m_at_beginning_of_statement = previous_at_bos; - return 0; + return nullptr; } break; @@ -2772,7 +2764,7 @@ if (! m_classdef_element_names_are_keywords) { m_at_beginning_of_statement = previous_at_bos; - return 0; + return nullptr; } // fall through ... @@ -2819,10 +2811,10 @@ case arguments_kw: #if defined (DISABLE_ARGUMENTS_VALIDATION_BLOCK) - return 0; + return nullptr; #else if (! m_arguments_is_keyword) - return 0; + return nullptr; break; #endif @@ -2831,7 +2823,7 @@ break; case endspmd_kw: - tok_val = new token (kw->tok, token::spmd_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::spmd_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; @@ -2840,9 +2832,9 @@ if ((m_reading_fcn_file || m_reading_script_file || m_reading_classdef_file) && ! m_fcn_file_full_name.empty ()) - tok_val = new token (kw->tok, m_fcn_file_full_name, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, m_fcn_file_full_name, m_tok_beg, m_tok_end); else - tok_val = new token (kw->tok, "stdin", m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, "stdin", m_tok_beg, m_tok_end); } break; @@ -2850,7 +2842,7 @@ { int l = m_tok_beg.line (); octave_value ov_value (static_cast<double> (l)); - tok_val = new token (kw->tok, ov_value, "", m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, ov_value, "", m_tok_beg, m_tok_end); } break; @@ -2858,12 +2850,10 @@ panic_impossible (); } - if (! tok_val) - tok_val = new token (kw->tok, true, m_tok_beg, m_tok_end); - - push_token (tok_val); - - return kw->tok; + if (! tok) + tok = new token (kw->tok_id, true, m_tok_beg, m_tok_end); + + return tok; } /* @@ -3022,9 +3012,7 @@ std::string msg {"too many digits for binary constant"}; token *tok = new token (LEXICAL_ERROR, msg, m_tok_beg, m_tok_end); - push_token (tok); - - return count_token_internal (LEXICAL_ERROR); + return handle_token (tok); } // FIXME: is there a better way? Can uintmax_t be anything other @@ -3052,9 +3040,9 @@ update_token_positions (flex_yyleng ()); - push_token (new token (NUMBER, ov_value, yytxt, m_tok_beg, m_tok_end)); - - return count_token_internal (NUMBER); + token *tok = new token (NUMBER, ov_value, yytxt, m_tok_beg, m_tok_end); + + return handle_token (tok); } static uint64_t @@ -3173,9 +3161,9 @@ ? octave_value (Complex (0.0, value)) : octave_value (value)); - push_token (new token (NUMBER, ov_value, yytxt, m_tok_beg, m_tok_end)); - - return count_token_internal (NUMBER); + token *tok = new token (NUMBER, ov_value, yytxt, m_tok_beg, m_tok_end); + + return handle_token (tok); } template <> @@ -3228,9 +3216,7 @@ std::string msg {"too many digits for hexadecimal constant"}; token *tok = new token (LEXICAL_ERROR, msg, m_tok_beg, m_tok_end); - push_token (tok); - - return count_token_internal (LEXICAL_ERROR); + return handle_token (tok); } // Assert here because if yytext doesn't contain a valid number, we @@ -3248,9 +3234,9 @@ update_token_positions (flex_yyleng ()); - push_token (new token (NUMBER, ov_value, yytxt, m_tok_beg, m_tok_end)); - - return count_token_internal (NUMBER); + token *tok = new token (NUMBER, ov_value, yytxt, m_tok_beg, m_tok_end); + + return handle_token (tok); } void @@ -3391,20 +3377,18 @@ std::string msg {"method, class, and package names may not be keywords"}; token *tok = new token (LEXICAL_ERROR, msg, m_tok_beg, m_tok_end); - push_token (tok); - - return count_token_internal (LEXICAL_ERROR); + return handle_token (tok); } - push_token (new token (SUPERCLASSREF, meth, cls, m_tok_beg, m_tok_end)); + token *tok = new token (SUPERCLASSREF, meth, cls, m_tok_beg, m_tok_end); m_filepos.increment_column (flex_yyleng ()); - return count_token_internal (SUPERCLASSREF); + return handle_token (tok); } - int - base_lexer::handle_meta_identifier () + token * + base_lexer::make_meta_identifier_token () { std::string txt = flex_yytext (); @@ -3417,25 +3401,27 @@ // Token positions should have already been updated before this // function is called. + token *tok; + if (fq_identifier_contains_keyword (cls)) { std::string msg {"class and package names may not be keywords"}; - token *tok = new token (LEXICAL_ERROR, msg, m_tok_beg, m_tok_end); - - push_token (tok); - - return count_token_internal (LEXICAL_ERROR); + tok = new token (LEXICAL_ERROR, msg, m_tok_beg, m_tok_end); } - - push_token (new token (METAQUERY, cls, m_tok_beg, m_tok_end)); - - m_filepos.increment_column (flex_yyleng ()); - - return METAQUERY; + else + { + m_looking_for_object_index = true; + + tok = new token (METAQUERY, cls, m_tok_beg, m_tok_end); + + m_filepos.increment_column (flex_yyleng ()); + } + + return tok; } - int - base_lexer::handle_fq_identifier () + token * + base_lexer::make_fq_identifier_token () { std::string txt = flex_yytext (); @@ -3445,21 +3431,23 @@ // Token positions should have already been updated before this // function is called. + token *tok; + if (fq_identifier_contains_keyword (txt)) { std::string msg {"function, method, class, and package names may not be keywords"}; - token *tok = new token (LEXICAL_ERROR, msg, m_tok_beg, m_tok_end); - - push_token (tok); - - return count_token_internal (LEXICAL_ERROR); + tok = new token (LEXICAL_ERROR, msg, m_tok_beg, m_tok_end); } - - push_token (new token (FQ_IDENT, txt, m_tok_beg, m_tok_end)); - - m_filepos.increment_column (flex_yyleng ()); - - return FQ_IDENT; + else + { + m_looking_for_object_index = true; + + tok = new token (FQ_IDENT, txt, m_tok_beg, m_tok_end); + + m_filepos.increment_column (flex_yyleng ()); + } + + return tok; } // Figure out exactly what kind of token to return when we have seen @@ -3479,32 +3467,32 @@ if (m_looking_at_indirect_ref) { - push_token (new token (STRUCT_ELT, ident, m_tok_beg, m_tok_end)); + token *tok = new token (STRUCT_ELT, ident, m_tok_beg, m_tok_end); m_looking_for_object_index = true; - return STRUCT_ELT; + return handle_token (tok); } // If ident is a keyword token, then make_keyword_token will set // m_at_beginning_of_statement. For example, if tok is an IF // token, then m_at_beginning_of_statement will be false. - int kw_token = make_keyword_token (ident); + token *tok = make_keyword_token (ident); // If we have a regular keyword, return it. // Keywords can be followed by identifiers. - if (kw_token) + if (tok) { m_looking_for_object_index = false; // The call to make_keyword_token set m_at_beginning_of_statement. - return count_token_internal (kw_token); + return handle_token (tok); } - token *tok = new token (NAME, ident, m_tok_beg, m_tok_end); + tok = new token (NAME, ident, m_tok_beg, m_tok_end); // For compatibility with Matlab, the following symbols are // handled specially so that things like @@ -3523,8 +3511,6 @@ || ident == "NaN" || ident == "nan")) tok->mark_may_be_command (); - push_token (tok); - // The magic end index can't be indexed. if (ident != "end") @@ -3532,7 +3518,7 @@ m_at_beginning_of_statement = false; - return count_token_internal (NAME); + return handle_token (tok); } void @@ -3616,7 +3602,7 @@ base_lexer::push_token (token *tok) { YYSTYPE *lval = yyget_lval (m_scanner); - lval->tok_val = tok; + lval->tok = tok; m_tokens.push (tok); } @@ -3624,7 +3610,7 @@ base_lexer::current_token () { YYSTYPE *lval = yyget_lval (m_scanner); - return lval->tok_val; + return lval->tok; } std::size_t @@ -3634,9 +3620,9 @@ } void - base_lexer::display_token (int tok) + base_lexer::display_token (int tok_id) { - switch (tok) + switch (tok_id) { case '=': std::cerr << "'='\n"; break; case ':': std::cerr << "':'\n"; break; @@ -3681,9 +3667,9 @@ case NUMBER: { - token *tok_val = current_token (); + token *tok = current_token (); std::cerr << "NUMBER ["; - octave_value num = tok_val->number (); + octave_value num = tok->number (); num.print_raw (std::cerr); std::cerr << "]\n"; } @@ -3691,15 +3677,15 @@ case STRUCT_ELT: { - token *tok_val = current_token (); - std::cerr << "STRUCT_ELT [" << tok_val->text () << "]\n"; + token *tok = current_token (); + std::cerr << "STRUCT_ELT [" << tok->text () << "]\n"; } break; case NAME: { - token *tok_val = current_token (); - std::cerr << "NAME [" << tok_val->text () << "]\n"; + token *tok = current_token (); + std::cerr << "NAME [" << tok->text () << "]\n"; } break; @@ -3708,10 +3694,10 @@ case DQ_STRING: case SQ_STRING: { - token *tok_val = current_token (); - - std::cerr << (tok == DQ_STRING ? "DQ_STRING" : "SQ_STRING") - << " [" << tok_val->text () << "]\n"; + token *tok = current_token (); + + std::cerr << (tok_id == DQ_STRING ? "DQ_STRING" : "SQ_STRING") + << " [" << tok->text () << "]\n"; } break; @@ -3752,10 +3738,10 @@ case '\t': std::cerr << "TAB\n"; break; default: { - if (tok < 256 && tok > 31) - std::cerr << static_cast<char> (tok) << "\n"; + if (tok_id < 256 && tok_id > 31) + std::cerr << static_cast<char> (tok_id) << "\n"; else - std::cerr << "UNKNOWN(" << tok << ")\n"; + std::cerr << "UNKNOWN(" << tok_id << ")\n"; } break; } @@ -3891,9 +3877,9 @@ } bool - base_lexer::maybe_unput_comma_before_unary_op (int tok) + base_lexer::maybe_unput_comma_before_unary_op (int tok_id) { - int prev_tok = previous_token_value (); + int prev_tok_id = previous_token_id (); bool unput_comma = false; @@ -3904,9 +3890,9 @@ bool space_after = is_space_or_tab (c); - if (! (prev_tok == '[' || prev_tok == '{' + if (! (prev_tok_id == '[' || prev_tok_id == '{' || previous_token_is_binop () - || ((tok == '+' || tok == '-') && space_after))) + || ((tok_id == '+' || tok_id == '-') && space_after))) unput_comma = true; } @@ -3914,19 +3900,19 @@ } int - base_lexer::handle_op (int tok, bool bos, bool compat) + base_lexer::handle_op (int tok_id, bool bos, bool compat) { if (! compat) warn_language_extension_operator (flex_yytext ()); update_token_positions (flex_yyleng ()); - push_token (new token (tok, m_tok_beg, m_tok_end)); + token *tok = new token (tok_id, m_tok_beg, m_tok_end); m_looking_for_object_index = false; m_at_beginning_of_statement = bos; - switch (tok) + switch (tok_id) { case EXPR_LT: if (m_parsing_classdef_decl) @@ -3945,7 +3931,7 @@ break; } - return count_token_internal (tok); + return handle_token (tok); } // When a command argument boundary is detected, push out the current @@ -3955,61 +3941,61 @@ int base_lexer::finish_command_arg () { - int tok = SQ_STRING; - - token *tok_val = new token (tok, m_string_text, m_tok_beg, m_tok_end); + token *tok = new token (SQ_STRING, m_string_text, m_tok_beg, m_tok_end); m_string_text = ""; m_command_arg_paren_count = 0; - return handle_token (tok, tok_val); + return handle_token (tok); } int - base_lexer::handle_token (int tok, token *tok_val) + base_lexer::handle_token (int tok_id, token *tok) { - if (! tok_val) - tok_val = new token (tok, m_tok_beg, m_tok_end); - - push_token (tok_val); - - return count_token_internal (tok); + if (! tok) + tok = new token (tok_id, m_tok_beg, m_tok_end); + + return handle_token (tok); } int - base_lexer::count_token (int tok) + base_lexer::handle_token (token *tok) { - token *tok_val = new token (tok, m_tok_beg, m_tok_end); - - push_token (tok_val); - - return count_token_internal (tok); + push_token (tok); + + return count_token_internal (tok->token_id ()); + } + + int + base_lexer::count_token (int tok_id) + { + return handle_token (new token (tok_id, m_tok_beg, m_tok_end)); } int - base_lexer::count_token_internal (int tok) + base_lexer::count_token_internal (int tok_id) { - if (tok != '\n') + if (tok_id != '\n') increment_token_count (); - return show_token (tok); + return show_token (tok_id); } int - base_lexer::show_token (int tok) + base_lexer::show_token (int tok_id) { if (display_tokens ()) - display_token (tok); + display_token (tok_id); if (debug_flag ()) { std::cerr << "R: "; - display_token (tok); + display_token (tok_id); std::cerr << std::endl; } - return tok; + return tok_id; } int
--- a/libinterp/parse-tree/oct-parse.yy Thu Feb 29 15:32:28 2024 -0500 +++ b/libinterp/parse-tree/oct-parse.yy Sat Mar 02 15:36:26 2024 -0500 @@ -140,7 +140,7 @@ int dummy_type; // The type of the basic tokens returned by the lexer. - octave::token *tok_val; + octave::token *tok; // Comment strings that we need to deal with mid-rule. octave::comment_list *comment_type; @@ -202,41 +202,41 @@ } // Tokens with line and column information. -%token <tok_val> '=' ':' '-' '+' '*' '/' '~' '!' -%token <tok_val> '(' ')' '[' ']' '{' '}' '.' '@' -%token <tok_val> ',' ';' '\n' -%token <tok_val> ADD_EQ SUB_EQ MUL_EQ DIV_EQ LEFTDIV_EQ POW_EQ -%token <tok_val> EMUL_EQ EDIV_EQ ELEFTDIV_EQ EPOW_EQ AND_EQ OR_EQ -%token <tok_val> EXPR_AND_AND EXPR_OR_OR -%token <tok_val> EXPR_AND EXPR_OR -%token <tok_val> EXPR_LT EXPR_LE EXPR_EQ EXPR_NE EXPR_GE EXPR_GT -%token <tok_val> LEFTDIV EMUL EDIV ELEFTDIV -%token <tok_val> HERMITIAN TRANSPOSE -%token <tok_val> PLUS_PLUS MINUS_MINUS POW EPOW -%token <tok_val> NUMBER -%token <tok_val> STRUCT_ELT -%token <tok_val> NAME -%token <tok_val> END -%token <tok_val> DQ_STRING SQ_STRING -%token <tok_val> FOR PARFOR WHILE DO UNTIL -%token <tok_val> SPMD -%token <tok_val> IF ELSEIF ELSE -%token <tok_val> SWITCH CASE OTHERWISE -%token <tok_val> BREAK CONTINUE FUNC_RET -%token <tok_val> UNWIND CLEANUP -%token <tok_val> TRY CATCH -%token <tok_val> GLOBAL PERSISTENT -%token <tok_val> FCN_HANDLE -%token <tok_val> CLASSDEF -%token <tok_val> PROPERTIES METHODS EVENTS ENUMERATION -%token <tok_val> METAQUERY -%token <tok_val> SUPERCLASSREF -%token <tok_val> FQ_IDENT -%token <tok_val> GET SET -%token <tok_val> FCN -%token <tok_val> ARGUMENTS -%token <tok_val> LEXICAL_ERROR -%token <tok_val> END_OF_INPUT +%token <tok> '=' ':' '-' '+' '*' '/' '~' '!' +%token <tok> '(' ')' '[' ']' '{' '}' '.' '@' +%token <tok> ',' ';' '\n' +%token <tok> ADD_EQ SUB_EQ MUL_EQ DIV_EQ LEFTDIV_EQ POW_EQ +%token <tok> EMUL_EQ EDIV_EQ ELEFTDIV_EQ EPOW_EQ AND_EQ OR_EQ +%token <tok> EXPR_AND_AND EXPR_OR_OR +%token <tok> EXPR_AND EXPR_OR +%token <tok> EXPR_LT EXPR_LE EXPR_EQ EXPR_NE EXPR_GE EXPR_GT +%token <tok> LEFTDIV EMUL EDIV ELEFTDIV +%token <tok> HERMITIAN TRANSPOSE +%token <tok> PLUS_PLUS MINUS_MINUS POW EPOW +%token <tok> NUMBER +%token <tok> STRUCT_ELT +%token <tok> NAME +%token <tok> END +%token <tok> DQ_STRING SQ_STRING +%token <tok> FOR PARFOR WHILE DO UNTIL +%token <tok> SPMD +%token <tok> IF ELSEIF ELSE +%token <tok> SWITCH CASE OTHERWISE +%token <tok> BREAK CONTINUE FUNC_RET +%token <tok> UNWIND CLEANUP +%token <tok> TRY CATCH +%token <tok> GLOBAL PERSISTENT +%token <tok> FCN_HANDLE +%token <tok> CLASSDEF +%token <tok> PROPERTIES METHODS EVENTS ENUMERATION +%token <tok> METAQUERY +%token <tok> SUPERCLASSREF +%token <tok> FQ_IDENT +%token <tok> GET SET +%token <tok> FCN +%token <tok> ARGUMENTS +%token <tok> LEXICAL_ERROR +%token <tok> END_OF_INPUT // Other tokens. %token <dummy_type> INPUT_FILE @@ -248,8 +248,8 @@ %type <dummy_type> param_list_beg param_list_end stmt_begin anon_fcn_begin %type <dummy_type> parsing_local_fcns parse_error at_first_executable_stmt %type <comment_type> stash_comment -%type <tok_val> function_beg classdef_beg arguments_beg -%type <tok_val> properties_beg methods_beg events_beg enumeration_beg +%type <tok> function_beg classdef_beg arguments_beg +%type <tok> properties_beg methods_beg events_beg enumeration_beg %type <punct_type> sep_no_nl opt_sep_no_nl nl opt_nl sep opt_sep %type <tree_type> input %type <tree_constant_type> string constant magic_colon @@ -337,7 +337,7 @@ // cases (for example, a new semantic type is added but not handled // here). -%destructor { } <tok_val> +%destructor { } <tok> %destructor { } <punct_type> %destructor { } <comment_type> %destructor { } <> @@ -1278,7 +1278,7 @@ OCTAVE_YYUSE ($3); octave::comment_list *lc = $2; - octave::comment_list *tc = lexer.get_comment (); + octave::comment_list *tc = lexer.get_comment_list (); if (! ($$ = parser.make_spmd_command ($1, $4, $5, lc, tc))) { @@ -1710,7 +1710,7 @@ OCTAVE_YYUSE ($3, $6); octave::comment_list *lc = $2; - octave::comment_list *tc = lexer.get_comment (); + octave::comment_list *tc = lexer.get_comment_list (); if (! ($$ = parser.make_arguments_block ($1, $4, $5, $7, lc, tc))) { @@ -1991,7 +1991,7 @@ OCTAVE_YYUSE ($3); octave::comment_list *lc = $2; - octave::comment_list *tc = lexer.get_comment (); + octave::comment_list *tc = lexer.get_comment_list (); if (! ($$ = parser.make_classdef_properties_block ($1, $4, $5, $6, lc, tc))) @@ -2067,7 +2067,7 @@ OCTAVE_YYUSE ($3); octave::comment_list *lc = $2; - octave::comment_list *tc = lexer.get_comment (); + octave::comment_list *tc = lexer.get_comment_list (); if (! ($$ = parser.make_classdef_methods_block ($1, $4, $5, $6, lc, tc))) @@ -2151,7 +2151,7 @@ OCTAVE_YYUSE ($3); octave::comment_list *lc = $2; - octave::comment_list *tc = lexer.get_comment (); + octave::comment_list *tc = lexer.get_comment_list (); if (! ($$ = parser.make_classdef_events_block ($1, $4, $5, $6, lc, tc))) @@ -2203,7 +2203,7 @@ OCTAVE_YYUSE ($3); octave::comment_list *lc = $2; - octave::comment_list *tc = lexer.get_comment (); + octave::comment_list *tc = lexer.get_comment_list (); if (! ($$ = parser.make_classdef_enum_block ($1, $4, $5, $6, lc, tc))) @@ -2275,7 +2275,7 @@ stash_comment : // empty { - $$ = lexer.get_comment (); + $$ = lexer.get_comment_list (); } ; @@ -2826,12 +2826,12 @@ // Make a constant. tree_constant * - base_parser::make_constant (token *tok_val) - { - int l = tok_val->line (); - int c = tok_val->column (); - - int op = tok_val->token_value (); + base_parser::make_constant (token *tok) + { + int l = tok->line (); + int c = tok->column (); + + int op = tok->token_id (); tree_constant *retval = nullptr; @@ -2846,15 +2846,15 @@ case NUMBER: { - retval = new tree_constant (tok_val->number (), l, c); - retval->stash_original_text (tok_val->text_rep ()); + retval = new tree_constant (tok->number (), l, c); + retval->stash_original_text (tok->text_rep ()); } break; case DQ_STRING: case SQ_STRING: { - std::string txt = tok_val->text (); + std::string txt = tok->text (); char delim = op == DQ_STRING ? '"' : '\''; octave_value tmp (txt, delim); @@ -2873,7 +2873,7 @@ txt = undo_string_escapes (txt); // FIXME: maybe this should also be handled by - // tok_val->text_rep () for character strings? + // tok->text_rep () for character strings? retval->stash_original_text (delim + txt + delim); } break; @@ -2895,12 +2895,12 @@ // Make a function handle. tree_fcn_handle * - base_parser::make_fcn_handle (token *tok_val) - { - int l = tok_val->line (); - int c = tok_val->column (); - - tree_fcn_handle *retval = new tree_fcn_handle (tok_val->text (), l, c); + base_parser::make_fcn_handle (token *tok) + { + int l = tok->line (); + int c = tok->column (); + + tree_fcn_handle *retval = new tree_fcn_handle (tok->text (), l, c); return retval; } @@ -3055,7 +3055,7 @@ tree_expression * base_parser::make_binary_op (int op, tree_expression *op1, - token *tok_val, tree_expression *op2) + token *tok, tree_expression *op2) { octave_value::binary_op t = octave_value::unknown_binary_op; @@ -3138,8 +3138,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return maybe_compound_binary_expression (op1, op2, l, c, t); } @@ -3183,7 +3183,7 @@ tree_expression * base_parser::make_boolean_op (int op, tree_expression *op1, - token *tok_val, tree_expression *op2) + token *tok, tree_expression *op2) { tree_boolean_expression::type t; @@ -3202,8 +3202,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return new tree_boolean_expression (op1, op2, l, c, t); } @@ -3211,7 +3211,7 @@ // Build a prefix expression. tree_expression * - base_parser::make_prefix_op (int op, tree_expression *op1, token *tok_val) + base_parser::make_prefix_op (int op, tree_expression *op1, token *tok) { octave_value::unary_op t = octave_value::unknown_unary_op; @@ -3243,8 +3243,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return new tree_prefix_expression (op1, l, c, t); } @@ -3252,7 +3252,7 @@ // Build a postfix expression. tree_expression * - base_parser::make_postfix_op (int op, tree_expression *op1, token *tok_val) + base_parser::make_postfix_op (int op, tree_expression *op1, token *tok) { octave_value::unary_op t = octave_value::unknown_unary_op; @@ -3279,8 +3279,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return new tree_postfix_expression (op1, l, c, t); } @@ -3299,7 +3299,7 @@ if (end_token_ok (end_tok, token::unwind_protect_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); int l = unwind_tok->line (); int c = unwind_tok->column (); @@ -3333,7 +3333,7 @@ if (end_token_ok (end_tok, token::try_catch_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); int l = try_tok->line (); int c = try_tok->column (); @@ -3389,7 +3389,7 @@ if (end_token_ok (end_tok, token::while_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); m_lexer.m_looping--; @@ -3419,7 +3419,7 @@ { maybe_warn_assign_as_truth_value (expr); - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); m_lexer.m_looping--; @@ -3448,7 +3448,7 @@ { expr->mark_as_for_cmd_expr (); - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); m_lexer.m_looping--; @@ -3596,7 +3596,7 @@ if (end_token_ok (end_tok, token::if_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); int l = if_tok->line (); int c = if_tok->column (); @@ -3670,7 +3670,7 @@ if (end_token_ok (end_tok, token::switch_end)) { - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); int l = switch_tok->line (); int c = switch_tok->column (); @@ -4044,7 +4044,7 @@ = new octave_user_function (m_lexer.m_symtab_context.curr_scope (), param_list, nullptr, body); - comment_list *tc = m_lexer.m_comment_buf.get_comment (); + comment_list *tc = m_lexer.m_comment_buf.get_comment_list (); fcn->stash_trailing_comment (tc); fcn->stash_fcn_end_location (end_fcn_stmt->line (), @@ -4377,7 +4377,7 @@ // and the final end token for the classdef block. tree_classdef * - base_parser::make_classdef (token *tok_val, + base_parser::make_classdef (token *tok, tree_classdef_attribute_list *a, tree_identifier *id, tree_classdef_superclass_list *sc, @@ -4419,8 +4419,8 @@ { if (end_token_ok (end_tok, token::classdef_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); // First non-copyright comments found above and below // function keyword are candidates for the documentation @@ -4470,7 +4470,7 @@ // find the doc string for the final property in the list. tree_classdef_properties_block * - base_parser::make_classdef_properties_block (token *tok_val, + base_parser::make_classdef_properties_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_property_list *plist, token *end_tok, @@ -4481,8 +4481,8 @@ if (end_token_ok (end_tok, token::properties_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (plist) { @@ -4549,7 +4549,7 @@ // classdef block. tree_classdef_methods_block * - base_parser::make_classdef_methods_block (token *tok_val, + base_parser::make_classdef_methods_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_methods_list *mlist, token *end_tok, comment_list *lc, @@ -4559,8 +4559,8 @@ if (end_token_ok (end_tok, token::methods_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (! mlist) mlist = new tree_classdef_methods_list (); @@ -4590,7 +4590,7 @@ // the doc string for the final event in the list. tree_classdef_events_block * - base_parser::make_classdef_events_block (token *tok_val, + base_parser::make_classdef_events_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_events_list *elist, token *end_tok, @@ -4601,8 +4601,8 @@ if (end_token_ok (end_tok, token::events_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (! elist) elist = new tree_classdef_events_list (); @@ -4645,7 +4645,7 @@ // list. tree_classdef_enum_block * - base_parser::make_classdef_enum_block (token *tok_val, + base_parser::make_classdef_enum_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_enum_list *elist, token *end_tok, @@ -4656,8 +4656,8 @@ if (end_token_ok (end_tok, token::enumeration_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (! elist) elist = new tree_classdef_enum_list (); @@ -5069,18 +5069,18 @@ // Make a declaration command. tree_decl_command * - base_parser::make_decl_command (int tok, token *tok_val, + base_parser::make_decl_command (int tok_id, token *tok, tree_decl_init_list *lst) { tree_decl_command *retval = nullptr; - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (lst) m_lexer.mark_as_variables (lst->variable_names ()); - switch (tok) + switch (tok_id) { case GLOBAL: { @@ -5485,7 +5485,7 @@ tree_statement * base_parser::make_statement (T *arg) { - comment_list *comment = m_lexer.get_comment (); + comment_list *comment = m_lexer.get_comment_list (); return new tree_statement (arg, comment); } @@ -5698,16 +5698,16 @@ { YYSTYPE lval; - int token = octave_lex (&lval, m_lexer.m_scanner); - - if (token < 0) + int tok_id = octave_lex (&lval, m_lexer.m_scanner); + + if (tok_id < 0) { // TOKEN == -2 means that the lexer recognized a comment // and we should be at the end of the buffer but not the // end of the file so we should return 0 to indicate // "complete input" instead of -1 to request more input. - status = (token == -2 ? 0 : -1); + status = (tok_id == -2 ? 0 : -1); if (! eof && m_lexer.at_end_of_buffer ()) return status; @@ -5719,7 +5719,7 @@ try { - status = octave_push_parse (pstate, token, &lval, *this); + status = octave_push_parse (pstate, tok_id, &lval, *this); } catch (execution_exception& e) {
--- a/libinterp/parse-tree/octave.gperf Thu Feb 29 15:32:28 2024 -0500 +++ b/libinterp/parse-tree/octave.gperf Sat Mar 02 15:36:26 2024 -0500 @@ -83,7 +83,7 @@ %} -struct octave_kw { const char *name; int tok; octave_kw_id kw_id; }; +struct octave_kw { const char *name; int tok_id; octave_kw_id kw_id; }; %% arguments, ARGUMENTS, arguments_kw
--- a/libinterp/parse-tree/parse.h Thu Feb 29 15:32:28 2024 -0500 +++ b/libinterp/parse-tree/parse.h Sat Mar 02 15:36:26 2024 -0500 @@ -234,7 +234,7 @@ OCTINTERP_API bool push_fcn_symtab (); // Build a constant. - OCTINTERP_API tree_constant * make_constant (token *tok_val); + OCTINTERP_API tree_constant * make_constant (token *tok); OCTINTERP_API tree_black_hole * make_black_hole (); @@ -249,7 +249,7 @@ append_cell_row (tree_cell *cell, tree_argument_list *row); // Build a function handle. - OCTINTERP_API tree_fcn_handle * make_fcn_handle (token *tok_val); + OCTINTERP_API tree_fcn_handle * make_fcn_handle (token *tok); // Build an anonymous function handle. OCTINTERP_API tree_anon_fcn_handle * @@ -263,7 +263,7 @@ // Build a binary expression. OCTINTERP_API tree_expression * - make_binary_op (int op, tree_expression *op1, token *tok_val, + make_binary_op (int op, tree_expression *op1, token *tok, tree_expression *op2); // Maybe convert EXPR to a braindead_shortcircuit expression. @@ -272,16 +272,16 @@ // Build a boolean expression. OCTINTERP_API tree_expression * - make_boolean_op (int op, tree_expression *op1, token *tok_val, + make_boolean_op (int op, tree_expression *op1, token *tok, tree_expression *op2); // Build a prefix expression. OCTINTERP_API tree_expression * - make_prefix_op (int op, tree_expression *op1, token *tok_val); + make_prefix_op (int op, tree_expression *op1, token *tok); // Build a postfix expression. OCTINTERP_API tree_expression * - make_postfix_op (int op, tree_expression *op1, token *tok_val); + make_postfix_op (int op, tree_expression *op1, token *tok); // Build an unwind-protect command. OCTINTERP_API tree_command * @@ -454,13 +454,13 @@ recover_from_parsing_function (); OCTINTERP_API tree_classdef * - make_classdef (token *tok_val, tree_classdef_attribute_list *a, + make_classdef (token *tok, tree_classdef_attribute_list *a, tree_identifier *id, tree_classdef_superclass_list *sc, tree_classdef_body *body, token *end_tok, comment_list *lc, comment_list *bc, comment_list *tc); OCTINTERP_API tree_classdef_properties_block * - make_classdef_properties_block (token *tok_val, + make_classdef_properties_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_property_list *plist, token *end_tok, comment_list *lc, @@ -478,14 +478,14 @@ tree_classdef_property *elt); OCTINTERP_API tree_classdef_methods_block * - make_classdef_methods_block (token *tok_val, + make_classdef_methods_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_methods_list *mlist, token *end_tok, comment_list *lc, comment_list *tc); OCTINTERP_API tree_classdef_events_block * - make_classdef_events_block (token *tok_val, + make_classdef_events_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_events_list *elist, token *end_tok, comment_list *lc, @@ -502,7 +502,7 @@ tree_classdef_event *elt); OCTINTERP_API tree_classdef_enum_block * - make_classdef_enum_block (token *tok_val, + make_classdef_enum_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_enum_list *elist, token *end_tok, comment_list *lc, @@ -606,7 +606,7 @@ // Make a declaration command. OCTINTERP_API tree_decl_command * - make_decl_command (int tok, token *tok_val, tree_decl_init_list *lst); + make_decl_command (int tok_id, token *tok, tree_decl_init_list *lst); OCTINTERP_API tree_decl_init_list * make_decl_init_list (tree_decl_elt *elt);
--- a/libinterp/parse-tree/token.h Thu Feb 29 15:32:28 2024 -0500 +++ b/libinterp/parse-tree/token.h Sat Mar 02 15:36:26 2024 -0500 @@ -73,32 +73,32 @@ public: - token (int tv, const filepos& beg_pos, const filepos& end_pos) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv) + token (int id, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_comment_list (lst) { } - token (int tv, bool is_kw, const filepos& beg_pos, const filepos& end_pos) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (is_kw ? keyword_token : generic_token) + token (int id, bool is_kw, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (is_kw ? keyword_token : generic_token), m_comment_list (lst) { } - token (int tv, const char *s, const filepos& beg_pos, const filepos& end_pos) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (string_token), m_tok_info (s) + token (int id, const char *s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (string_token), m_tok_info (s), m_comment_list (lst) { } - token (int tv, const std::string& s, const filepos& beg_pos, const filepos& end_pos) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (string_token), m_tok_info (s) + token (int id, const std::string& s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (string_token), m_tok_info (s), m_comment_list (lst) { } - token (int tv, const octave_value& val, const std::string& s, const filepos& beg_pos, const filepos& end_pos) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (numeric_token), m_tok_info (val), m_orig_text (s) + token (int id, const octave_value& val, const std::string& s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (numeric_token), m_tok_info (val), m_orig_text (s), m_comment_list (lst) { } - token (int tv, end_tok_type t, const filepos& beg_pos, const filepos& end_pos) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (ettype_token), m_tok_info (t) + token (int id, end_tok_type t, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (ettype_token), m_tok_info (t), m_comment_list (lst) { } - token (int tv, const std::string& meth, const std::string& cls, const filepos& beg_pos, const filepos& end_pos) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (scls_name_token), m_tok_info (meth, cls) + token (int id, const std::string& meth, const std::string& cls, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (scls_name_token), m_tok_info (meth, cls), m_comment_list (lst) { } OCTAVE_DEFAULT_COPY_MOVE_DELETE (token) @@ -109,8 +109,13 @@ void mark_trailing_space () { m_tspc = true; } bool space_follows_token () const { return m_tspc; } - int token_value () const { return m_tok_val; } - bool token_value_is (int tv) const { return tv == m_tok_val; } + int token_id () const { return m_tok_id; } + + bool token_is (int id) const { return m_tok_id == id; } + bool token_is (const token *tok) const + { + return tok ? token_is (tok->token_id ()) : false; + } filepos beg_pos () const { return m_beg_pos; } filepos end_pos () const { return m_end_pos; } @@ -148,7 +153,7 @@ filepos m_beg_pos; filepos m_end_pos; - int m_tok_val; + int m_tok_id; token_type m_type_tag {generic_token}; @@ -235,6 +240,8 @@ tok_info m_tok_info; std::string m_orig_text; + + comment_list *m_comment_list; }; OCTAVE_END_NAMESPACE(octave)