Mercurial > octave
diff libinterp/parse-tree/lex.ll @ 27769:07ffed9878ad
store begin and end position in tokens
* token.h, token.cc (token::m_beg_pos): Rename from m_pos.
(token::m_end_pos): New member variable.
(token::beg_pos, token::end_pos): New functions.
(token::token): Accept beginning and ending positions for tokens as
arguments. Change all uses but store the same position for both
beginning and end in most cases. Subsequent changes will store the
correct beginning and ending positions.
author | John W. Eaton <jwe@octave.org> |
---|---|
date | Mon, 02 Dec 2019 22:58:55 -0600 |
parents | d6701f835496 |
children | 7a06e352ac61 |
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.ll Mon Dec 02 22:37:05 2019 -0600 +++ b/libinterp/parse-tree/lex.ll Mon Dec 02 22:58:55 2019 -0600 @@ -840,7 +840,8 @@ curr_lexer->push_token (new octave::token (DQ_STRING, curr_lexer->m_string_text, - curr_lexer->m_beg_string)); + curr_lexer->m_beg_string, + curr_lexer->m_filepos)); curr_lexer->m_string_text = ""; @@ -861,7 +862,7 @@ octave::token *tok = new octave::token (LEXICAL_ERROR, "invalid octal escape sequence in character string", - curr_lexer->m_filepos); + curr_lexer->m_filepos, curr_lexer->m_filepos); curr_lexer->push_token (tok); @@ -1002,7 +1003,7 @@ octave::token *tok = new octave::token (LEXICAL_ERROR, "unterminated character string constant", - curr_lexer->m_filepos); + curr_lexer->m_filepos, curr_lexer->m_filepos); curr_lexer->push_token (tok); @@ -1036,7 +1037,8 @@ curr_lexer->push_token (new octave::token (SQ_STRING, curr_lexer->m_string_text, - curr_lexer->m_beg_string)); + curr_lexer->m_beg_string, + curr_lexer->m_filepos)); curr_lexer->m_string_text = ""; @@ -1057,7 +1059,7 @@ octave::token *tok = new octave::token (LEXICAL_ERROR, "unterminated character string constant", - curr_lexer->m_filepos); + curr_lexer->m_filepos, curr_lexer->m_filepos); curr_lexer->push_token (tok); @@ -1368,12 +1370,14 @@ if (kw_token) tok = new octave::token (LEXICAL_ERROR, "function handles may not refer to keywords", + curr_lexer->m_filepos, curr_lexer->m_filepos); else { curr_lexer->m_looking_for_object_index = true; tok = new octave::token (FCN_HANDLE, ident, + curr_lexer->m_filepos, curr_lexer->m_filepos); } @@ -1416,7 +1420,7 @@ octave::token *tok = new octave::token (LEXICAL_ERROR, "unexpected internal lexer error", - curr_lexer->m_filepos); + curr_lexer->m_filepos, curr_lexer->m_filepos); curr_lexer->push_token (tok); @@ -1806,7 +1810,7 @@ octave::token *tok = new octave::token (LEXICAL_ERROR, buf.str (), - curr_lexer->m_filepos); + curr_lexer->m_filepos, curr_lexer->m_filepos); curr_lexer->push_token (tok); @@ -2575,73 +2579,86 @@ return 0; } - tok_val = new token (end_kw, token::simple_end, m_filepos); + tok_val = new token (end_kw, token::simple_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case end_try_catch_kw: - tok_val = new token (end_try_catch_kw, token::try_catch_end, m_filepos); + tok_val = new token (end_try_catch_kw, token::try_catch_end, + m_filepos, m_filepos); m_at_beginning_of_statement = true; break; case end_unwind_protect_kw: tok_val = new token (end_unwind_protect_kw, - token::unwind_protect_end, m_filepos); + token::unwind_protect_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case endfor_kw: - tok_val = new token (endfor_kw, token::for_end, m_filepos); + tok_val = new token (endfor_kw, token::for_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case endfunction_kw: - tok_val = new token (endfunction_kw, token::function_end, m_filepos); + tok_val = new token (endfunction_kw, token::function_end, + m_filepos, m_filepos); m_at_beginning_of_statement = true; break; case endif_kw: - tok_val = new token (endif_kw, token::if_end, m_filepos); + tok_val = new token (endif_kw, token::if_end, m_filepos, m_filepos); m_at_beginning_of_statement = true; break; case endparfor_kw: - tok_val = new token (endparfor_kw, token::parfor_end, m_filepos); + tok_val = new token (endparfor_kw, token::parfor_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case endswitch_kw: - tok_val = new token (endswitch_kw, token::switch_end, m_filepos); + tok_val = new token (endswitch_kw, token::switch_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case endwhile_kw: - tok_val = new token (endwhile_kw, token::while_end, m_filepos); + tok_val = new token (endwhile_kw, token::while_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case endclassdef_kw: - tok_val = new token (endclassdef_kw, token::classdef_end, m_filepos); + tok_val = new token (endclassdef_kw, token::classdef_end, + m_filepos, m_filepos); m_at_beginning_of_statement = true; break; case endenumeration_kw: - tok_val = new token (endenumeration_kw, token::enumeration_end, m_filepos); + tok_val = new token (endenumeration_kw, token::enumeration_end, + m_filepos, m_filepos); m_at_beginning_of_statement = true; break; case endevents_kw: - tok_val = new token (endevents_kw, token::events_end, m_filepos); + tok_val = new token (endevents_kw, token::events_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case endmethods_kw: - tok_val = new token (endmethods_kw, token::methods_end, m_filepos); + tok_val = new token (endmethods_kw, token::methods_end, m_filepos, + m_filepos); m_at_beginning_of_statement = true; break; case endproperties_kw: - tok_val = new token (endproperties_kw, token::properties_end, m_filepos); + tok_val = new token (endproperties_kw, token::properties_end, + m_filepos, m_filepos); m_at_beginning_of_statement = true; break; @@ -2729,9 +2746,11 @@ if ((m_reading_fcn_file || m_reading_script_file || m_reading_classdef_file) && ! m_fcn_file_full_name.empty ()) - tok_val = new token (magic_file_kw, m_fcn_file_full_name, m_filepos); + tok_val = new token (magic_file_kw, m_fcn_file_full_name, + m_filepos, m_filepos); else - tok_val = new token (magic_file_kw, "stdin", m_filepos); + tok_val = new token (magic_file_kw, "stdin", m_filepos, + m_filepos); } break; @@ -2739,7 +2758,7 @@ { int l = m_filepos.line (); tok_val = new token (magic_line_kw, static_cast<double> (l), - "", m_filepos); + "", m_filepos, m_filepos); } break; @@ -2748,7 +2767,7 @@ } if (! tok_val) - tok_val = new token (kw->tok, true, m_filepos); + tok_val = new token (kw->tok, true, m_filepos, m_filepos); push_token (tok_val); @@ -2874,7 +2893,7 @@ m_looking_for_object_index = false; m_at_beginning_of_statement = false; - push_token (new token (NUM, value, yytxt, m_filepos)); + push_token (new token (NUM, value, yytxt, m_filepos, m_filepos)); m_filepos.increment_column (flex_yyleng ()); } @@ -3018,15 +3037,14 @@ token *tok = new token (LEXICAL_ERROR, "method, class, and package names may not be keywords", - m_filepos); + m_filepos, m_filepos); push_token (tok); return count_token_internal (LEXICAL_ERROR); } - push_token (new token (SUPERCLASSREF, meth, cls, - m_filepos)); + push_token (new token (SUPERCLASSREF, meth, cls, m_filepos, m_filepos)); m_filepos.increment_column (flex_yyleng ()); @@ -3048,13 +3066,13 @@ { token *tok = new token (LEXICAL_ERROR, "class and package names may not be keywords", - m_filepos); + m_filepos, m_filepos); push_token (tok); return count_token_internal (LEXICAL_ERROR); } - push_token (new token (METAQUERY, cls, m_filepos)); + push_token (new token (METAQUERY, cls, m_filepos, m_filepos)); m_filepos.increment_column (flex_yyleng ()); @@ -3074,14 +3092,14 @@ token *tok = new token (LEXICAL_ERROR, "function, method, class, and package names may not be keywords", - m_filepos); + m_filepos, m_filepos); push_token (tok); return count_token_internal (LEXICAL_ERROR); } - push_token (new token (FQ_IDENT, txt, m_filepos)); + push_token (new token (FQ_IDENT, txt, m_filepos, m_filepos)); m_filepos.increment_column (flex_yyleng ()); @@ -3103,7 +3121,7 @@ if (m_looking_at_indirect_ref) { - push_token (new token (STRUCT_ELT, ident, m_filepos)); + push_token (new token (STRUCT_ELT, ident, m_filepos, m_filepos)); m_looking_for_object_index = true; @@ -3140,7 +3158,7 @@ symbol_record sr = (scope ? scope.insert (ident) : symbol_record (ident)); - token *tok = new token (NAME, sr, m_filepos); + token *tok = new token (NAME, sr, m_filepos, m_filepos); // The following symbols are handled specially so that things like // @@ -3574,7 +3592,7 @@ if (! compat) warn_language_extension_operator (flex_yytext ()); - push_token (new token (tok, m_filepos)); + push_token (new token (tok, m_filepos, m_filepos)); m_filepos.increment_column (flex_yyleng ()); m_looking_for_object_index = false; @@ -3605,7 +3623,7 @@ int base_lexer::handle_token (const std::string& name, int tok) { - token *tok_val = new token (tok, name, m_filepos); + token *tok_val = new token (tok, name, m_filepos, m_filepos); return handle_token (tok, tok_val); } @@ -3614,7 +3632,7 @@ base_lexer::handle_token (int tok, token *tok_val) { if (! tok_val) - tok_val = new token (tok, m_filepos); + tok_val = new token (tok, m_filepos, m_filepos); push_token (tok_val); @@ -3626,7 +3644,7 @@ int base_lexer::count_token (int tok) { - token *tok_val = new token (tok, m_filepos); + token *tok_val = new token (tok, m_filepos, m_filepos); push_token (tok_val);