Mercurial > octave
changeset 33119:f83d0ab96c16
use more consistent naming in the lexer and parser for token IDs and values
Try to consistently use TOK_ID as the name for the numeric code that
identifies a token and TOK for the name of the octave::token object
that holds the information gathered about the token.
Affected files: lex.h, lex.ll, oct-parse.yy, octave.gperf, parse.h,
token.h.
| author | John W. Eaton <jwe@octave.org> |
|---|---|
| date | Sat, 02 Mar 2024 14:27:30 -0500 |
| parents | 8ec8df064d8f |
| children | b8919be4c931 |
| files | libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll libinterp/parse-tree/oct-parse.yy libinterp/parse-tree/octave.gperf libinterp/parse-tree/parse.h libinterp/parse-tree/token.h |
| diffstat | 6 files changed, 352 insertions(+), 325 deletions(-) [+] |
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h Sat Mar 02 12:07:44 2024 -0500 +++ b/libinterp/parse-tree/lex.h Sat Mar 02 14:27:30 2024 -0500 @@ -325,9 +325,12 @@ void reset (); - int previous_token_value () const; + int previous_token_id () const; + token * previous_token (); + const token * previous_token () const; - bool previous_token_value_is (int tok_val) const; + bool previous_token_is (int tok_id) const; + bool previous_token_is (const token *tok) const; void mark_previous_token_trailing_space (); @@ -692,7 +695,7 @@ std::size_t pending_token_count () const; - void display_token (int tok); + void display_token (int tok_id); void fatal_error (const char *msg); @@ -733,19 +736,19 @@ void display_start_state () const; - bool maybe_unput_comma_before_unary_op (int tok); + bool maybe_unput_comma_before_unary_op (int tok_id); - int handle_op (int tok, bool bos = false, bool compat = true); + int handle_op (int tok_id, bool bos = false, bool compat = true); int finish_command_arg (); - int handle_token (int tok, token *tok_val = nullptr); + int handle_token (int tok_id, token *tok = nullptr); - int count_token (int tok); + int count_token (int tok_id); - int count_token_internal (int tok); + int count_token_internal (int tok_id); - int show_token (int tok); + int show_token (int tok_id); protected:
--- a/libinterp/parse-tree/lex.ll Sat Mar 02 12:07:44 2024 -0500 +++ b/libinterp/parse-tree/lex.ll Sat Mar 02 14:27:30 2024 -0500 @@ -168,22 +168,22 @@ #define YY_FATAL_ERROR(msg) \ (yyget_extra (yyscanner))->fatal_error (msg) -#define CMD_OR_OP(PATTERN, TOK, COMPAT) \ - do \ - { \ - curr_lexer->lexer_debug (PATTERN); \ - \ - if (curr_lexer->looks_like_command_arg ()) \ - { \ - yyless (0); \ - curr_lexer->push_start_state (COMMAND_START); \ - } \ - else \ - return curr_lexer->handle_op (TOK, false, COMPAT); \ - } \ +#define CMD_OR_OP(PATTERN, TOK_ID, COMPAT) \ + do \ + { \ + curr_lexer->lexer_debug (PATTERN); \ + \ + if (curr_lexer->looks_like_command_arg ()) \ + { \ + yyless (0); \ + curr_lexer->push_start_state (COMMAND_START); \ + } \ + else \ + return curr_lexer->handle_op (TOK_ID, false, COMPAT); \ + } \ while (0) -#define CMD_OR_UNARY_OP(PATTERN, TOK, COMPAT) \ +#define CMD_OR_UNARY_OP(PATTERN, TOK_ID, COMPAT) \ do \ { \ curr_lexer->lexer_debug (PATTERN); \ @@ -196,17 +196,17 @@ curr_lexer->push_start_state (COMMAND_START); \ } \ else \ - return curr_lexer->handle_op (TOK, false, COMPAT); \ + return curr_lexer->handle_op (TOK_ID, false, COMPAT); \ } \ else \ { \ - if (curr_lexer->maybe_unput_comma_before_unary_op (TOK)) \ + if (curr_lexer->maybe_unput_comma_before_unary_op (TOK_ID)) \ { \ yyless (0); \ curr_lexer->xunput (','); \ } \ else \ - return curr_lexer->handle_op (TOK, false, COMPAT); \ + return curr_lexer->handle_op (TOK_ID, false, COMPAT); \ } \ } \ while (0) @@ -225,57 +225,57 @@ } \ while (0) -// If we are at the end of the buffer, ask for more input. -// If we are at the end of the file, deal with it. -// Otherwise, just keep going with the text from the current buffer. -#define HANDLE_STRING_CONTINUATION \ - do \ - { \ - curr_lexer->m_filepos.next_line (); \ - \ - HANDLE_EOB_OR_EOF (-1); \ - } \ + // If we are at the end of the buffer, ask for more input. + // If we are at the end of the file, deal with it. + // Otherwise, just keep going with the text from the current buffer. +#define HANDLE_STRING_CONTINUATION \ + do \ + { \ + curr_lexer->m_filepos.next_line (); \ + \ + HANDLE_EOB_OR_EOF (-1); \ + } \ while (0) -#define HANDLE_NUMBER(PATTERN, BASE) \ - do \ - { \ - curr_lexer->lexer_debug (PATTERN); \ - \ - if (curr_lexer->previous_token_may_be_command () \ - && curr_lexer->space_follows_previous_token ()) \ - { \ - yyless (0); \ - curr_lexer->push_start_state (COMMAND_START); \ - } \ - else \ - { \ - int tok = curr_lexer->previous_token_value (); \ - \ - if (curr_lexer->whitespace_is_significant () \ - && curr_lexer->space_follows_previous_token () \ - && ! (tok == '[' || tok == '{' \ - || curr_lexer->previous_token_is_binop ())) \ - { \ - yyless (0); \ - curr_lexer->xunput (','); \ - } \ - else \ - return curr_lexer->handle_number<BASE> (); \ - } \ - } \ - while (0) +#define HANDLE_NUMBER(PATTERN, BASE) \ + do \ + { \ + curr_lexer->lexer_debug (PATTERN); \ + \ + if (curr_lexer->previous_token_may_be_command () \ + && curr_lexer->space_follows_previous_token ()) \ + { \ + yyless (0); \ + curr_lexer->push_start_state (COMMAND_START); \ + } \ + else \ + { \ + int tok_id = curr_lexer->previous_token_id (); \ + \ + if (curr_lexer->whitespace_is_significant () \ + && curr_lexer->space_follows_previous_token () \ + && ! (tok_id == '[' || tok_id == '{' \ + || curr_lexer->previous_token_is_binop ())) \ + { \ + yyless (0); \ + curr_lexer->xunput (','); \ + } \ + else \ + return curr_lexer->handle_number<BASE> (); \ + } \ + } \ + while (0) #define HANDLE_IDENTIFIER(pattern, get_set) \ do \ { \ curr_lexer->lexer_debug (pattern); \ \ - int tok = curr_lexer->previous_token_value (); \ + int tok_id = curr_lexer->previous_token_id (); \ \ if (curr_lexer->whitespace_is_significant () \ && curr_lexer->space_follows_previous_token () \ - && ! (tok == '[' || tok == '{' \ + && ! (tok_id == '[' || tok_id == '{' \ || curr_lexer->previous_token_is_binop ())) \ { \ yyless (0); \ @@ -590,9 +590,9 @@ curr_lexer->warn_language_extension ("bare newline inside parentheses"); else { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == ';' || tok == '[' || tok == '{')) + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == ';' || tok_id == '[' || tok_id == '{')) curr_lexer->xunput (';'); } } @@ -655,9 +655,9 @@ if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token ()) { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == '[' || tok == '{' + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) unput_comma = true; } @@ -1194,13 +1194,13 @@ curr_lexer->update_token_positions (yyleng); - int id_tok = curr_lexer->handle_fq_identifier (); - - if (id_tok >= 0) + int tok_id = curr_lexer->handle_fq_identifier (); + + if (tok_id >= 0) { curr_lexer->m_looking_for_object_index = true; - return curr_lexer->count_token_internal (id_tok); + return curr_lexer->count_token_internal (tok_id); } } @@ -1355,13 +1355,13 @@ { curr_lexer->update_token_positions (yyleng); - int id_tok = curr_lexer->handle_meta_identifier (); - - if (id_tok >= 0) + int tok_id = curr_lexer->handle_meta_identifier (); + + if (tok_id >= 0) { curr_lexer->m_looking_for_object_index = true; - return curr_lexer->count_token_internal (id_tok); + return curr_lexer->count_token_internal (tok_id); } } } @@ -1378,11 +1378,11 @@ } else { - int tok_val = curr_lexer->previous_token_value (); + int tok_id = curr_lexer->previous_token_id (); if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token () - && ! (tok_val == '[' || tok_val == '{' + && ! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) { yyless (0); @@ -1425,7 +1425,7 @@ curr_lexer->push_token (tok); - return curr_lexer->count_token_internal (tok->token_value ()); + return curr_lexer->count_token_internal (tok->token_id ()); } } } @@ -1495,13 +1495,13 @@ } else { - int tok = curr_lexer->previous_token_value (); + int tok_id = curr_lexer->previous_token_id (); if (curr_lexer->whitespace_is_significant ()) { if (curr_lexer->space_follows_previous_token ()) { - if (tok == '[' || tok == '{' + if (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ()) { curr_lexer->m_filepos.increment_column (); @@ -1515,7 +1515,7 @@ } else { - if (tok == '[' || tok == '{' + if (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop () || curr_lexer->previous_token_is_keyword ()) { @@ -1531,7 +1531,7 @@ } else { - if (! tok || tok == '[' || tok == '{' || tok == '(' + if (! tok_id || tok_id == '[' || tok_id == '{' || tok_id == '(' || curr_lexer->previous_token_is_binop () || curr_lexer->previous_token_is_keyword ()) { @@ -1563,13 +1563,13 @@ } else { - int tok = curr_lexer->previous_token_value (); + int tok_id = curr_lexer->previous_token_id (); if (curr_lexer->whitespace_is_significant ()) { if (curr_lexer->space_follows_previous_token ()) { - if (tok == '[' || tok == '{' + if (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ()) { curr_lexer->m_filepos.increment_column (); @@ -1679,9 +1679,9 @@ if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token ()) { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == '[' || tok == '{' + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) unput_comma = true; } @@ -1789,9 +1789,9 @@ if (curr_lexer->whitespace_is_significant () && curr_lexer->space_follows_previous_token ()) { - int tok = curr_lexer->previous_token_value (); - - if (! (tok == '[' || tok == '{' + int tok_id = curr_lexer->previous_token_id (); + + if (! (tok_id == '[' || tok_id == '{' || curr_lexer->previous_token_is_binop ())) unput_comma = true; } @@ -2231,60 +2231,79 @@ m_tokens.clear (); } - int - lexical_feedback::previous_token_value () const + token * + lexical_feedback::previous_token () + { + return m_tokens.front (); + } + + const token * + lexical_feedback::previous_token () const { - const token *tok = m_tokens.front (); - return tok ? tok->token_value () : 0; + return m_tokens.front (); + } + + int + lexical_feedback::previous_token_id () const + { + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->token_id () : 0; } bool - lexical_feedback::previous_token_value_is (int tok_val) const + lexical_feedback::previous_token_is (int tok_id) const { - const token *tok = m_tokens.front (); - return tok ? tok->token_value_is (tok_val) : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->token_is (tok_id) : false; + } + + bool + lexical_feedback::previous_token_is (const token *tok) const + { + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->token_is (tok) : false; } void lexical_feedback::mark_previous_token_trailing_space () { - token *tok = m_tokens.front (); - if (tok && ! previous_token_value_is ('\n')) - tok->mark_trailing_space (); + token *prev_tok = previous_token (); + if (prev_tok && ! previous_token_is ('\n')) + prev_tok->mark_trailing_space (); } bool lexical_feedback::space_follows_previous_token () const { - const token *tok = m_tokens.front (); - return tok ? tok->space_follows_token () : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->space_follows_token () : false; } bool lexical_feedback::previous_token_is_binop () const { - int tok = previous_token_value (); - - return (tok == '+' || tok == '-' || tok == '@' || tok == '~' || tok == '!' - || tok == ',' || tok == ';' || tok == '*' || tok == '/' - || tok == ':' || tok == '=' || tok == ADD_EQ - || tok == AND_EQ || tok == DIV_EQ || tok == EDIV - || tok == EDIV_EQ || tok == ELEFTDIV || tok == ELEFTDIV_EQ - || tok == EMUL || tok == EMUL_EQ - || tok == EPOW || tok == EPOW_EQ || tok == EXPR_AND - || tok == EXPR_AND_AND || tok == EXPR_EQ || tok == EXPR_GE - || tok == EXPR_GT || tok == EXPR_LE || tok == EXPR_LT - || tok == EXPR_NE || tok == EXPR_OR - || tok == EXPR_OR_OR || tok == LEFTDIV || tok == LEFTDIV_EQ - || tok == MUL_EQ || tok == OR_EQ || tok == POW - || tok == POW_EQ || tok == SUB_EQ); + int tok_id = previous_token_id (); + + return (tok_id == '+' || tok_id == '-' || tok_id == '@' || tok_id == '~' || tok_id == '!' + || tok_id == ',' || tok_id == ';' || tok_id == '*' || tok_id == '/' + || tok_id == ':' || tok_id == '=' || tok_id == ADD_EQ + || tok_id == AND_EQ || tok_id == DIV_EQ || tok_id == EDIV + || tok_id == EDIV_EQ || tok_id == ELEFTDIV || tok_id == ELEFTDIV_EQ + || tok_id == EMUL || tok_id == EMUL_EQ + || tok_id == EPOW || tok_id == EPOW_EQ || tok_id == EXPR_AND + || tok_id == EXPR_AND_AND || tok_id == EXPR_EQ || tok_id == EXPR_GE + || tok_id == EXPR_GT || tok_id == EXPR_LE || tok_id == EXPR_LT + || tok_id == EXPR_NE || tok_id == EXPR_OR + || tok_id == EXPR_OR_OR || tok_id == LEFTDIV || tok_id == LEFTDIV_EQ + || tok_id == MUL_EQ || tok_id == OR_EQ || tok_id == POW + || tok_id == POW_EQ || tok_id == SUB_EQ); } bool lexical_feedback::previous_token_is_keyword () const { - const token *tok = m_tokens.front (); - return tok ? tok->iskeyword () : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->iskeyword () : false; } void @@ -2311,8 +2330,8 @@ if (! m_allow_command_syntax) return false; - const token *tok = m_tokens.front (); - return tok ? tok->may_be_command () : false; + const token *prev_tok = previous_token (); + return prev_tok ? prev_tok->may_be_command () : false; } static bool @@ -2487,9 +2506,9 @@ error ("block comment unterminated at end of input"); } - token *tok_val = new token (END_OF_INPUT, m_tok_beg, m_tok_end); - - push_token (tok_val); + token *tok = new token (END_OF_INPUT, m_tok_beg, m_tok_end); + + push_token (tok); return count_token_internal (END_OF_INPUT); } @@ -2620,7 +2639,7 @@ // May be reset to true for some token types. m_at_beginning_of_statement = false; - token *tok_val = nullptr; + token *tok = nullptr; switch (kw->kw_id) { @@ -2654,47 +2673,47 @@ return 0; } - tok_val = new token (kw->tok, token::simple_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::simple_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case end_try_catch_kw: - tok_val = new token (kw->tok, token::try_catch_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::try_catch_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case end_unwind_protect_kw: - tok_val = new token (kw->tok, token::unwind_protect_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::unwind_protect_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endfor_kw: - tok_val = new token (kw->tok, token::for_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::for_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endfunction_kw: - tok_val = new token (kw->tok, token::function_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::function_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endif_kw: - tok_val = new token (kw->tok, token::if_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::if_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endparfor_kw: - tok_val = new token (kw->tok, token::parfor_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::parfor_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endswitch_kw: - tok_val = new token (kw->tok, token::switch_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::switch_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endwhile_kw: - tok_val = new token (kw->tok, token::while_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::while_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; @@ -2702,33 +2721,33 @@ #if defined (DISABLE_ARGUMENTS_VALIDATION_BLOCK) return 0; #else - tok_val = new token (kw->tok, token::arguments_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::arguments_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; #endif case endclassdef_kw: - tok_val = new token (kw->tok, token::classdef_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::classdef_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endenumeration_kw: - tok_val = new token (kw->tok, token::enumeration_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::enumeration_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endevents_kw: - tok_val = new token (kw->tok, token::events_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::events_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endmethods_kw: - tok_val = new token (kw->tok, token::methods_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::methods_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; case endproperties_kw: - tok_val = new token (kw->tok, token::properties_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::properties_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; @@ -2831,7 +2850,7 @@ break; case endspmd_kw: - tok_val = new token (kw->tok, token::spmd_end, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, token::spmd_end, m_tok_beg, m_tok_end); m_at_beginning_of_statement = true; break; @@ -2840,9 +2859,9 @@ if ((m_reading_fcn_file || m_reading_script_file || m_reading_classdef_file) && ! m_fcn_file_full_name.empty ()) - tok_val = new token (kw->tok, m_fcn_file_full_name, m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, m_fcn_file_full_name, m_tok_beg, m_tok_end); else - tok_val = new token (kw->tok, "stdin", m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, "stdin", m_tok_beg, m_tok_end); } break; @@ -2850,7 +2869,7 @@ { int l = m_tok_beg.line (); octave_value ov_value (static_cast<double> (l)); - tok_val = new token (kw->tok, ov_value, "", m_tok_beg, m_tok_end); + tok = new token (kw->tok_id, ov_value, "", m_tok_beg, m_tok_end); } break; @@ -2858,12 +2877,12 @@ panic_impossible (); } - if (! tok_val) - tok_val = new token (kw->tok, true, m_tok_beg, m_tok_end); - - push_token (tok_val); - - return kw->tok; + if (! tok) + tok = new token (kw->tok_id, true, m_tok_beg, m_tok_end); + + push_token (tok); + + return kw->tok_id; } /* @@ -3616,7 +3635,7 @@ base_lexer::push_token (token *tok) { YYSTYPE *lval = yyget_lval (m_scanner); - lval->tok_val = tok; + lval->tok = tok; m_tokens.push (tok); } @@ -3624,7 +3643,7 @@ base_lexer::current_token () { YYSTYPE *lval = yyget_lval (m_scanner); - return lval->tok_val; + return lval->tok; } std::size_t @@ -3634,9 +3653,9 @@ } void - base_lexer::display_token (int tok) + base_lexer::display_token (int tok_id) { - switch (tok) + switch (tok_id) { case '=': std::cerr << "'='\n"; break; case ':': std::cerr << "':'\n"; break; @@ -3681,9 +3700,9 @@ case NUMBER: { - token *tok_val = current_token (); + token *tok = current_token (); std::cerr << "NUMBER ["; - octave_value num = tok_val->number (); + octave_value num = tok->number (); num.print_raw (std::cerr); std::cerr << "]\n"; } @@ -3691,15 +3710,15 @@ case STRUCT_ELT: { - token *tok_val = current_token (); - std::cerr << "STRUCT_ELT [" << tok_val->text () << "]\n"; + token *tok = current_token (); + std::cerr << "STRUCT_ELT [" << tok->text () << "]\n"; } break; case NAME: { - token *tok_val = current_token (); - std::cerr << "NAME [" << tok_val->text () << "]\n"; + token *tok = current_token (); + std::cerr << "NAME [" << tok->text () << "]\n"; } break; @@ -3708,10 +3727,10 @@ case DQ_STRING: case SQ_STRING: { - token *tok_val = current_token (); - - std::cerr << (tok == DQ_STRING ? "DQ_STRING" : "SQ_STRING") - << " [" << tok_val->text () << "]\n"; + token *tok = current_token (); + + std::cerr << (tok_id == DQ_STRING ? "DQ_STRING" : "SQ_STRING") + << " [" << tok->text () << "]\n"; } break; @@ -3752,10 +3771,10 @@ case '\t': std::cerr << "TAB\n"; break; default: { - if (tok < 256 && tok > 31) - std::cerr << static_cast<char> (tok) << "\n"; + if (tok_id < 256 && tok_id > 31) + std::cerr << static_cast<char> (tok_id) << "\n"; else - std::cerr << "UNKNOWN(" << tok << ")\n"; + std::cerr << "UNKNOWN(" << tok_id << ")\n"; } break; } @@ -3891,9 +3910,9 @@ } bool - base_lexer::maybe_unput_comma_before_unary_op (int tok) + base_lexer::maybe_unput_comma_before_unary_op (int tok_id) { - int prev_tok = previous_token_value (); + int prev_tok_id = previous_token_id (); bool unput_comma = false; @@ -3904,9 +3923,9 @@ bool space_after = is_space_or_tab (c); - if (! (prev_tok == '[' || prev_tok == '{' + if (! (prev_tok_id == '[' || prev_tok_id == '{' || previous_token_is_binop () - || ((tok == '+' || tok == '-') && space_after))) + || ((tok_id == '+' || tok_id == '-') && space_after))) unput_comma = true; } @@ -3914,19 +3933,19 @@ } int - base_lexer::handle_op (int tok, bool bos, bool compat) + base_lexer::handle_op (int tok_id, bool bos, bool compat) { if (! compat) warn_language_extension_operator (flex_yytext ()); update_token_positions (flex_yyleng ()); - push_token (new token (tok, m_tok_beg, m_tok_end)); + push_token (new token (tok_id, m_tok_beg, m_tok_end)); m_looking_for_object_index = false; m_at_beginning_of_statement = bos; - switch (tok) + switch (tok_id) { case EXPR_LT: if (m_parsing_classdef_decl) @@ -3945,7 +3964,7 @@ break; } - return count_token_internal (tok); + return count_token_internal (tok_id); } // When a command argument boundary is detected, push out the current @@ -3955,61 +3974,61 @@ int base_lexer::finish_command_arg () { - int tok = SQ_STRING; - - token *tok_val = new token (tok, m_string_text, m_tok_beg, m_tok_end); + int tok_id = SQ_STRING; + + token *tok = new token (tok_id, m_string_text, m_tok_beg, m_tok_end); m_string_text = ""; m_command_arg_paren_count = 0; - return handle_token (tok, tok_val); + return handle_token (tok_id, tok); } int - base_lexer::handle_token (int tok, token *tok_val) + base_lexer::handle_token (int tok_id, token *tok) { - if (! tok_val) - tok_val = new token (tok, m_tok_beg, m_tok_end); - - push_token (tok_val); - - return count_token_internal (tok); + if (! tok) + tok = new token (tok_id, m_tok_beg, m_tok_end); + + push_token (tok); + + return count_token_internal (tok_id); } int - base_lexer::count_token (int tok) + base_lexer::count_token (int tok_id) { - token *tok_val = new token (tok, m_tok_beg, m_tok_end); - - push_token (tok_val); - - return count_token_internal (tok); + token *tok = new token (tok_id, m_tok_beg, m_tok_end); + + push_token (tok); + + return count_token_internal (tok_id); } int - base_lexer::count_token_internal (int tok) + base_lexer::count_token_internal (int tok_id) { - if (tok != '\n') + if (tok_id != '\n') increment_token_count (); - return show_token (tok); + return show_token (tok_id); } int - base_lexer::show_token (int tok) + base_lexer::show_token (int tok_id) { if (display_tokens ()) - display_token (tok); + display_token (tok_id); if (debug_flag ()) { std::cerr << "R: "; - display_token (tok); + display_token (tok_id); std::cerr << std::endl; } - return tok; + return tok_id; } int
--- a/libinterp/parse-tree/oct-parse.yy Sat Mar 02 12:07:44 2024 -0500 +++ b/libinterp/parse-tree/oct-parse.yy Sat Mar 02 14:27:30 2024 -0500 @@ -140,7 +140,7 @@ int dummy_type; // The type of the basic tokens returned by the lexer. - octave::token *tok_val; + octave::token *tok; // Comment strings that we need to deal with mid-rule. octave::comment_list *comment_type; @@ -202,41 +202,41 @@ } // Tokens with line and column information. -%token <tok_val> '=' ':' '-' '+' '*' '/' '~' '!' -%token <tok_val> '(' ')' '[' ']' '{' '}' '.' '@' -%token <tok_val> ',' ';' '\n' -%token <tok_val> ADD_EQ SUB_EQ MUL_EQ DIV_EQ LEFTDIV_EQ POW_EQ -%token <tok_val> EMUL_EQ EDIV_EQ ELEFTDIV_EQ EPOW_EQ AND_EQ OR_EQ -%token <tok_val> EXPR_AND_AND EXPR_OR_OR -%token <tok_val> EXPR_AND EXPR_OR -%token <tok_val> EXPR_LT EXPR_LE EXPR_EQ EXPR_NE EXPR_GE EXPR_GT -%token <tok_val> LEFTDIV EMUL EDIV ELEFTDIV -%token <tok_val> HERMITIAN TRANSPOSE -%token <tok_val> PLUS_PLUS MINUS_MINUS POW EPOW -%token <tok_val> NUMBER -%token <tok_val> STRUCT_ELT -%token <tok_val> NAME -%token <tok_val> END -%token <tok_val> DQ_STRING SQ_STRING -%token <tok_val> FOR PARFOR WHILE DO UNTIL -%token <tok_val> SPMD -%token <tok_val> IF ELSEIF ELSE -%token <tok_val> SWITCH CASE OTHERWISE -%token <tok_val> BREAK CONTINUE FUNC_RET -%token <tok_val> UNWIND CLEANUP -%token <tok_val> TRY CATCH -%token <tok_val> GLOBAL PERSISTENT -%token <tok_val> FCN_HANDLE -%token <tok_val> CLASSDEF -%token <tok_val> PROPERTIES METHODS EVENTS ENUMERATION -%token <tok_val> METAQUERY -%token <tok_val> SUPERCLASSREF -%token <tok_val> FQ_IDENT -%token <tok_val> GET SET -%token <tok_val> FCN -%token <tok_val> ARGUMENTS -%token <tok_val> LEXICAL_ERROR -%token <tok_val> END_OF_INPUT +%token <tok> '=' ':' '-' '+' '*' '/' '~' '!' +%token <tok> '(' ')' '[' ']' '{' '}' '.' '@' +%token <tok> ',' ';' '\n' +%token <tok> ADD_EQ SUB_EQ MUL_EQ DIV_EQ LEFTDIV_EQ POW_EQ +%token <tok> EMUL_EQ EDIV_EQ ELEFTDIV_EQ EPOW_EQ AND_EQ OR_EQ +%token <tok> EXPR_AND_AND EXPR_OR_OR +%token <tok> EXPR_AND EXPR_OR +%token <tok> EXPR_LT EXPR_LE EXPR_EQ EXPR_NE EXPR_GE EXPR_GT +%token <tok> LEFTDIV EMUL EDIV ELEFTDIV +%token <tok> HERMITIAN TRANSPOSE +%token <tok> PLUS_PLUS MINUS_MINUS POW EPOW +%token <tok> NUMBER +%token <tok> STRUCT_ELT +%token <tok> NAME +%token <tok> END +%token <tok> DQ_STRING SQ_STRING +%token <tok> FOR PARFOR WHILE DO UNTIL +%token <tok> SPMD +%token <tok> IF ELSEIF ELSE +%token <tok> SWITCH CASE OTHERWISE +%token <tok> BREAK CONTINUE FUNC_RET +%token <tok> UNWIND CLEANUP +%token <tok> TRY CATCH +%token <tok> GLOBAL PERSISTENT +%token <tok> FCN_HANDLE +%token <tok> CLASSDEF +%token <tok> PROPERTIES METHODS EVENTS ENUMERATION +%token <tok> METAQUERY +%token <tok> SUPERCLASSREF +%token <tok> FQ_IDENT +%token <tok> GET SET +%token <tok> FCN +%token <tok> ARGUMENTS +%token <tok> LEXICAL_ERROR +%token <tok> END_OF_INPUT // Other tokens. %token <dummy_type> INPUT_FILE @@ -248,8 +248,8 @@ %type <dummy_type> param_list_beg param_list_end stmt_begin anon_fcn_begin %type <dummy_type> parsing_local_fcns parse_error at_first_executable_stmt %type <comment_type> stash_comment -%type <tok_val> function_beg classdef_beg arguments_beg -%type <tok_val> properties_beg methods_beg events_beg enumeration_beg +%type <tok> function_beg classdef_beg arguments_beg +%type <tok> properties_beg methods_beg events_beg enumeration_beg %type <punct_type> sep_no_nl opt_sep_no_nl nl opt_nl sep opt_sep %type <tree_type> input %type <tree_constant_type> string constant magic_colon @@ -337,7 +337,7 @@ // cases (for example, a new semantic type is added but not handled // here). -%destructor { } <tok_val> +%destructor { } <tok> %destructor { } <punct_type> %destructor { } <comment_type> %destructor { } <> @@ -2826,12 +2826,12 @@ // Make a constant. tree_constant * - base_parser::make_constant (token *tok_val) - { - int l = tok_val->line (); - int c = tok_val->column (); - - int op = tok_val->token_value (); + base_parser::make_constant (token *tok) + { + int l = tok->line (); + int c = tok->column (); + + int op = tok->token_id (); tree_constant *retval = nullptr; @@ -2846,15 +2846,15 @@ case NUMBER: { - retval = new tree_constant (tok_val->number (), l, c); - retval->stash_original_text (tok_val->text_rep ()); + retval = new tree_constant (tok->number (), l, c); + retval->stash_original_text (tok->text_rep ()); } break; case DQ_STRING: case SQ_STRING: { - std::string txt = tok_val->text (); + std::string txt = tok->text (); char delim = op == DQ_STRING ? '"' : '\''; octave_value tmp (txt, delim); @@ -2873,7 +2873,7 @@ txt = undo_string_escapes (txt); // FIXME: maybe this should also be handled by - // tok_val->text_rep () for character strings? + // tok->text_rep () for character strings? retval->stash_original_text (delim + txt + delim); } break; @@ -2895,12 +2895,12 @@ // Make a function handle. tree_fcn_handle * - base_parser::make_fcn_handle (token *tok_val) - { - int l = tok_val->line (); - int c = tok_val->column (); - - tree_fcn_handle *retval = new tree_fcn_handle (tok_val->text (), l, c); + base_parser::make_fcn_handle (token *tok) + { + int l = tok->line (); + int c = tok->column (); + + tree_fcn_handle *retval = new tree_fcn_handle (tok->text (), l, c); return retval; } @@ -3055,7 +3055,7 @@ tree_expression * base_parser::make_binary_op (int op, tree_expression *op1, - token *tok_val, tree_expression *op2) + token *tok, tree_expression *op2) { octave_value::binary_op t = octave_value::unknown_binary_op; @@ -3138,8 +3138,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return maybe_compound_binary_expression (op1, op2, l, c, t); } @@ -3183,7 +3183,7 @@ tree_expression * base_parser::make_boolean_op (int op, tree_expression *op1, - token *tok_val, tree_expression *op2) + token *tok, tree_expression *op2) { tree_boolean_expression::type t; @@ -3202,8 +3202,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return new tree_boolean_expression (op1, op2, l, c, t); } @@ -3211,7 +3211,7 @@ // Build a prefix expression. tree_expression * - base_parser::make_prefix_op (int op, tree_expression *op1, token *tok_val) + base_parser::make_prefix_op (int op, tree_expression *op1, token *tok) { octave_value::unary_op t = octave_value::unknown_unary_op; @@ -3243,8 +3243,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return new tree_prefix_expression (op1, l, c, t); } @@ -3252,7 +3252,7 @@ // Build a postfix expression. tree_expression * - base_parser::make_postfix_op (int op, tree_expression *op1, token *tok_val) + base_parser::make_postfix_op (int op, tree_expression *op1, token *tok) { octave_value::unary_op t = octave_value::unknown_unary_op; @@ -3279,8 +3279,8 @@ break; } - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); return new tree_postfix_expression (op1, l, c, t); } @@ -4377,7 +4377,7 @@ // and the final end token for the classdef block. tree_classdef * - base_parser::make_classdef (token *tok_val, + base_parser::make_classdef (token *tok, tree_classdef_attribute_list *a, tree_identifier *id, tree_classdef_superclass_list *sc, @@ -4419,8 +4419,8 @@ { if (end_token_ok (end_tok, token::classdef_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); // First non-copyright comments found above and below // function keyword are candidates for the documentation @@ -4470,7 +4470,7 @@ // find the doc string for the final property in the list. tree_classdef_properties_block * - base_parser::make_classdef_properties_block (token *tok_val, + base_parser::make_classdef_properties_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_property_list *plist, token *end_tok, @@ -4481,8 +4481,8 @@ if (end_token_ok (end_tok, token::properties_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (plist) { @@ -4549,7 +4549,7 @@ // classdef block. tree_classdef_methods_block * - base_parser::make_classdef_methods_block (token *tok_val, + base_parser::make_classdef_methods_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_methods_list *mlist, token *end_tok, comment_list *lc, @@ -4559,8 +4559,8 @@ if (end_token_ok (end_tok, token::methods_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (! mlist) mlist = new tree_classdef_methods_list (); @@ -4590,7 +4590,7 @@ // the doc string for the final event in the list. tree_classdef_events_block * - base_parser::make_classdef_events_block (token *tok_val, + base_parser::make_classdef_events_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_events_list *elist, token *end_tok, @@ -4601,8 +4601,8 @@ if (end_token_ok (end_tok, token::events_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (! elist) elist = new tree_classdef_events_list (); @@ -4645,7 +4645,7 @@ // list. tree_classdef_enum_block * - base_parser::make_classdef_enum_block (token *tok_val, + base_parser::make_classdef_enum_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_enum_list *elist, token *end_tok, @@ -4656,8 +4656,8 @@ if (end_token_ok (end_tok, token::enumeration_end)) { - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (! elist) elist = new tree_classdef_enum_list (); @@ -5069,18 +5069,18 @@ // Make a declaration command. tree_decl_command * - base_parser::make_decl_command (int tok, token *tok_val, + base_parser::make_decl_command (int tok_id, token *tok, tree_decl_init_list *lst) { tree_decl_command *retval = nullptr; - int l = tok_val->line (); - int c = tok_val->column (); + int l = tok->line (); + int c = tok->column (); if (lst) m_lexer.mark_as_variables (lst->variable_names ()); - switch (tok) + switch (tok_id) { case GLOBAL: { @@ -5698,16 +5698,16 @@ { YYSTYPE lval; - int token = octave_lex (&lval, m_lexer.m_scanner); - - if (token < 0) + int tok_id = octave_lex (&lval, m_lexer.m_scanner); + + if (tok_id < 0) { // TOKEN == -2 means that the lexer recognized a comment // and we should be at the end of the buffer but not the // end of the file so we should return 0 to indicate // "complete input" instead of -1 to request more input. - status = (token == -2 ? 0 : -1); + status = (tok_id == -2 ? 0 : -1); if (! eof && m_lexer.at_end_of_buffer ()) return status; @@ -5719,7 +5719,7 @@ try { - status = octave_push_parse (pstate, token, &lval, *this); + status = octave_push_parse (pstate, tok_id, &lval, *this); } catch (execution_exception& e) {
--- a/libinterp/parse-tree/octave.gperf Sat Mar 02 12:07:44 2024 -0500 +++ b/libinterp/parse-tree/octave.gperf Sat Mar 02 14:27:30 2024 -0500 @@ -83,7 +83,7 @@ %} -struct octave_kw { const char *name; int tok; octave_kw_id kw_id; }; +struct octave_kw { const char *name; int tok_id; octave_kw_id kw_id; }; %% arguments, ARGUMENTS, arguments_kw
--- a/libinterp/parse-tree/parse.h Sat Mar 02 12:07:44 2024 -0500 +++ b/libinterp/parse-tree/parse.h Sat Mar 02 14:27:30 2024 -0500 @@ -234,7 +234,7 @@ OCTINTERP_API bool push_fcn_symtab (); // Build a constant. - OCTINTERP_API tree_constant * make_constant (token *tok_val); + OCTINTERP_API tree_constant * make_constant (token *tok); OCTINTERP_API tree_black_hole * make_black_hole (); @@ -249,7 +249,7 @@ append_cell_row (tree_cell *cell, tree_argument_list *row); // Build a function handle. - OCTINTERP_API tree_fcn_handle * make_fcn_handle (token *tok_val); + OCTINTERP_API tree_fcn_handle * make_fcn_handle (token *tok); // Build an anonymous function handle. OCTINTERP_API tree_anon_fcn_handle * @@ -263,7 +263,7 @@ // Build a binary expression. OCTINTERP_API tree_expression * - make_binary_op (int op, tree_expression *op1, token *tok_val, + make_binary_op (int op, tree_expression *op1, token *tok, tree_expression *op2); // Maybe convert EXPR to a braindead_shortcircuit expression. @@ -272,16 +272,16 @@ // Build a boolean expression. OCTINTERP_API tree_expression * - make_boolean_op (int op, tree_expression *op1, token *tok_val, + make_boolean_op (int op, tree_expression *op1, token *tok, tree_expression *op2); // Build a prefix expression. OCTINTERP_API tree_expression * - make_prefix_op (int op, tree_expression *op1, token *tok_val); + make_prefix_op (int op, tree_expression *op1, token *tok); // Build a postfix expression. OCTINTERP_API tree_expression * - make_postfix_op (int op, tree_expression *op1, token *tok_val); + make_postfix_op (int op, tree_expression *op1, token *tok); // Build an unwind-protect command. OCTINTERP_API tree_command * @@ -454,13 +454,13 @@ recover_from_parsing_function (); OCTINTERP_API tree_classdef * - make_classdef (token *tok_val, tree_classdef_attribute_list *a, + make_classdef (token *tok, tree_classdef_attribute_list *a, tree_identifier *id, tree_classdef_superclass_list *sc, tree_classdef_body *body, token *end_tok, comment_list *lc, comment_list *bc, comment_list *tc); OCTINTERP_API tree_classdef_properties_block * - make_classdef_properties_block (token *tok_val, + make_classdef_properties_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_property_list *plist, token *end_tok, comment_list *lc, @@ -478,14 +478,14 @@ tree_classdef_property *elt); OCTINTERP_API tree_classdef_methods_block * - make_classdef_methods_block (token *tok_val, + make_classdef_methods_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_methods_list *mlist, token *end_tok, comment_list *lc, comment_list *tc); OCTINTERP_API tree_classdef_events_block * - make_classdef_events_block (token *tok_val, + make_classdef_events_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_events_list *elist, token *end_tok, comment_list *lc, @@ -502,7 +502,7 @@ tree_classdef_event *elt); OCTINTERP_API tree_classdef_enum_block * - make_classdef_enum_block (token *tok_val, + make_classdef_enum_block (token *tok, tree_classdef_attribute_list *a, tree_classdef_enum_list *elist, token *end_tok, comment_list *lc, @@ -606,7 +606,7 @@ // Make a declaration command. OCTINTERP_API tree_decl_command * - make_decl_command (int tok, token *tok_val, tree_decl_init_list *lst); + make_decl_command (int tok_id, token *tok, tree_decl_init_list *lst); OCTINTERP_API tree_decl_init_list * make_decl_init_list (tree_decl_elt *elt);
--- a/libinterp/parse-tree/token.h Sat Mar 02 12:07:44 2024 -0500 +++ b/libinterp/parse-tree/token.h Sat Mar 02 14:27:30 2024 -0500 @@ -73,32 +73,32 @@ public: - token (int tv, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_comment_list (lst) + token (int id, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_comment_list (lst) { } - token (int tv, bool is_kw, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (is_kw ? keyword_token : generic_token), m_comment_list (lst) + token (int id, bool is_kw, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (is_kw ? keyword_token : generic_token), m_comment_list (lst) { } - token (int tv, const char *s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (string_token), m_tok_info (s), m_comment_list (lst) + token (int id, const char *s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (string_token), m_tok_info (s), m_comment_list (lst) { } - token (int tv, const std::string& s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (string_token), m_tok_info (s), m_comment_list (lst) + token (int id, const std::string& s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (string_token), m_tok_info (s), m_comment_list (lst) { } - token (int tv, const octave_value& val, const std::string& s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (numeric_token), m_tok_info (val), m_orig_text (s), m_comment_list (lst) + token (int id, const octave_value& val, const std::string& s, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (numeric_token), m_tok_info (val), m_orig_text (s), m_comment_list (lst) { } - token (int tv, end_tok_type t, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (ettype_token), m_tok_info (t), m_comment_list (lst) + token (int id, end_tok_type t, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (ettype_token), m_tok_info (t), m_comment_list (lst) { } - token (int tv, const std::string& meth, const std::string& cls, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) - : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_val (tv), m_type_tag (scls_name_token), m_tok_info (meth, cls), m_comment_list (lst) + token (int id, const std::string& meth, const std::string& cls, const filepos& beg_pos, const filepos& end_pos, comment_list *lst = nullptr) + : m_beg_pos (beg_pos), m_end_pos (end_pos), m_tok_id (id), m_type_tag (scls_name_token), m_tok_info (meth, cls), m_comment_list (lst) { } OCTAVE_DEFAULT_COPY_MOVE_DELETE (token) @@ -109,8 +109,13 @@ void mark_trailing_space () { m_tspc = true; } bool space_follows_token () const { return m_tspc; } - int token_value () const { return m_tok_val; } - bool token_value_is (int tv) const { return tv == m_tok_val; } + int token_id () const { return m_tok_id; } + + bool token_is (int id) const { return m_tok_id == id; } + bool token_is (const token *tok) const + { + return tok ? token_is (tok->token_id ()) : false; + } filepos beg_pos () const { return m_beg_pos; } filepos end_pos () const { return m_end_pos; } @@ -148,7 +153,7 @@ filepos m_beg_pos; filepos m_end_pos; - int m_tok_val; + int m_tok_id; token_type m_type_tag {generic_token};