octave: libinterp/parse-tree/lex.ll comparison

comparison libinterp/parse-tree/lex.ll @ 27769:07ffed9878ad

store begin and end position in tokens * token.h, token.cc (token::m_beg_pos): Rename from m_pos. (token::m_end_pos): New member variable. (token::beg_pos, token::end_pos): New functions. (token::token): Accept beginning and ending positions for tokens as arguments. Change all uses but store the same position for both beginning and end in most cases. Subsequent changes will store the correct beginning and ending positions.

author	John W. Eaton <jwe@octave.org>
date	Mon, 02 Dec 2019 22:58:55 -0600
parents	d6701f835496
children	7a06e352ac61

comparison

equal deleted inserted replaced

-:d6701f835496
+:07ffed9878ad
 curr_lexer->m_looking_for_object_index = true;
 curr_lexer->m_at_beginning_of_statement = false;
 curr_lexer->push_token (new octave::token (DQ_STRING,
 curr_lexer->m_string_text,
-curr_lexer->m_beg_string));
+curr_lexer->m_beg_string,
+curr_lexer->m_filepos));
 curr_lexer->m_string_text = "";
 return curr_lexer->count_token_internal (DQ_STRING);
 }
 if (result > 0xff)
 {
 octave::token *tok
 = new octave::token (LEXICAL_ERROR,
 "invalid octal escape sequence in character string",
-curr_lexer->m_filepos);
+curr_lexer->m_filepos, curr_lexer->m_filepos);
 curr_lexer->push_token (tok);
 return curr_lexer->count_token_internal (LEXICAL_ERROR);
 }
 curr_lexer->lexer_debug ("<DQ_STRING_START>{NL}");
 octave::token *tok
 = new octave::token (LEXICAL_ERROR,
 "unterminated character string constant",
-curr_lexer->m_filepos);
+curr_lexer->m_filepos, curr_lexer->m_filepos);
 curr_lexer->push_token (tok);
 curr_lexer->m_filepos.next_line ();
 curr_lexer->m_looking_for_object_index = true;
 curr_lexer->m_at_beginning_of_statement = false;
 curr_lexer->push_token (new octave::token (SQ_STRING,
 curr_lexer->m_string_text,
-curr_lexer->m_beg_string));
+curr_lexer->m_beg_string,
+curr_lexer->m_filepos));
 curr_lexer->m_string_text = "";
 return curr_lexer->count_token_internal (SQ_STRING);
 }
 curr_lexer->lexer_debug ("<SQ_STRING_START>{NL}");
 octave::token *tok
 = new octave::token (LEXICAL_ERROR,
 "unterminated character string constant",
-curr_lexer->m_filepos);
+curr_lexer->m_filepos, curr_lexer->m_filepos);
 curr_lexer->push_token (tok);
 curr_lexer->m_filepos.next_line ();
 octave::token *tok;
 if (kw_token)
 tok = new octave::token (LEXICAL_ERROR,
 "function handles may not refer to keywords",
+curr_lexer->m_filepos,
 curr_lexer->m_filepos);
 else
 {
 curr_lexer->m_looking_for_object_index = true;
 tok = new octave::token (FCN_HANDLE, ident,
+curr_lexer->m_filepos,
 curr_lexer->m_filepos);
 }
 curr_lexer->push_token (tok);
 else if (curr_lexer->m_nesting_level.is_bracket_or_brace ())
 {
 octave::token *tok
 = new octave::token (LEXICAL_ERROR,
 "unexpected internal lexer error",
-curr_lexer->m_filepos);
+curr_lexer->m_filepos, curr_lexer->m_filepos);
 curr_lexer->push_token (tok);
 curr_lexer->m_filepos.next_line ();
 << octave::undo_string_escape (static_cast<char> (c))
 << "' (ASCII " << c << ")";
 octave::token *tok
 = new octave::token (LEXICAL_ERROR, buf.str (),
-curr_lexer->m_filepos);
+curr_lexer->m_filepos, curr_lexer->m_filepos);
 curr_lexer->push_token (tok);
 curr_lexer->m_filepos.increment_column ();
 {
 m_at_beginning_of_statement = previous_at_bos;
 return 0;
 }
-tok_val = new token (end_kw, token::simple_end, m_filepos);
+tok_val = new token (end_kw, token::simple_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case end_try_catch_kw:
-tok_val = new token (end_try_catch_kw, token::try_catch_end, m_filepos);
+tok_val = new token (end_try_catch_kw, token::try_catch_end,
+m_filepos, m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case end_unwind_protect_kw:
 tok_val = new token (end_unwind_protect_kw,
-token::unwind_protect_end, m_filepos);
+token::unwind_protect_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endfor_kw:
-tok_val = new token (endfor_kw, token::for_end, m_filepos);
+tok_val = new token (endfor_kw, token::for_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endfunction_kw:
-tok_val = new token (endfunction_kw, token::function_end, m_filepos);
+tok_val = new token (endfunction_kw, token::function_end,
+m_filepos, m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endif_kw:
-tok_val = new token (endif_kw, token::if_end, m_filepos);
+tok_val = new token (endif_kw, token::if_end, m_filepos, m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endparfor_kw:
-tok_val = new token (endparfor_kw, token::parfor_end, m_filepos);
+tok_val = new token (endparfor_kw, token::parfor_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endswitch_kw:
-tok_val = new token (endswitch_kw, token::switch_end, m_filepos);
+tok_val = new token (endswitch_kw, token::switch_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endwhile_kw:
-tok_val = new token (endwhile_kw, token::while_end, m_filepos);
+tok_val = new token (endwhile_kw, token::while_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endclassdef_kw:
-tok_val = new token (endclassdef_kw, token::classdef_end, m_filepos);
+tok_val = new token (endclassdef_kw, token::classdef_end,
+m_filepos, m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endenumeration_kw:
-tok_val = new token (endenumeration_kw, token::enumeration_end, m_filepos);
+tok_val = new token (endenumeration_kw, token::enumeration_end,
+m_filepos, m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endevents_kw:
-tok_val = new token (endevents_kw, token::events_end, m_filepos);
+tok_val = new token (endevents_kw, token::events_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endmethods_kw:
-tok_val = new token (endmethods_kw, token::methods_end, m_filepos);
+tok_val = new token (endmethods_kw, token::methods_end, m_filepos,
+m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case endproperties_kw:
-tok_val = new token (endproperties_kw, token::properties_end, m_filepos);
+tok_val = new token (endproperties_kw, token::properties_end,
+m_filepos, m_filepos);
 m_at_beginning_of_statement = true;
 break;
 case for_kw:
 case magic_file_kw:
 {
 if ((m_reading_fcn_file || m_reading_script_file
 || m_reading_classdef_file)
 && ! m_fcn_file_full_name.empty ())
-tok_val = new token (magic_file_kw, m_fcn_file_full_name, m_filepos);
+tok_val = new token (magic_file_kw, m_fcn_file_full_name,
+m_filepos, m_filepos);
 else
-tok_val = new token (magic_file_kw, "stdin", m_filepos);
+tok_val = new token (magic_file_kw, "stdin", m_filepos,
+m_filepos);
 }
 break;
 case magic_line_kw:
 {
 int l = m_filepos.line ();
 tok_val = new token (magic_line_kw, static_cast<double> (l),
-"", m_filepos);
+"", m_filepos, m_filepos);
 }
 break;
 default:
 panic_impossible ();
 }
 if (! tok_val)
-tok_val = new token (kw->tok, true, m_filepos);
+tok_val = new token (kw->tok, true, m_filepos, m_filepos);
 push_token (tok_val);
 return kw->tok;
 }
 assert (nread == 1);
 m_looking_for_object_index = false;
 m_at_beginning_of_statement = false;
-push_token (new token (NUM, value, yytxt, m_filepos));
+push_token (new token (NUM, value, yytxt, m_filepos, m_filepos));
 m_filepos.increment_column (flex_yyleng ());
 }
 void
 if (kw_token)
 {
 token *tok
 = new token (LEXICAL_ERROR,
 "method, class, and package names may not be keywords",
-m_filepos);
+m_filepos, m_filepos);
 push_token (tok);
 return count_token_internal (LEXICAL_ERROR);
 }
-push_token (new token (SUPERCLASSREF, meth, cls,
+push_token (new token (SUPERCLASSREF, meth, cls, m_filepos, m_filepos));
-m_filepos));
 m_filepos.increment_column (flex_yyleng ());
 return SUPERCLASSREF;
 }
 if (fq_identifier_contains_keyword (cls))
 {
 token *tok = new token (LEXICAL_ERROR,
 "class and package names may not be keywords",
-m_filepos);
+m_filepos, m_filepos);
 push_token (tok);
 return count_token_internal (LEXICAL_ERROR);
 }
-push_token (new token (METAQUERY, cls, m_filepos));
+push_token (new token (METAQUERY, cls, m_filepos, m_filepos));
 m_filepos.increment_column (flex_yyleng ());
 return METAQUERY;
 }
 if (fq_identifier_contains_keyword (txt))
 {
 token *tok
 = new token (LEXICAL_ERROR,
 "function, method, class, and package names may not be keywords",
-m_filepos);
+m_filepos, m_filepos);
 push_token (tok);
 return count_token_internal (LEXICAL_ERROR);
 }
-push_token (new token (FQ_IDENT, txt, m_filepos));
+push_token (new token (FQ_IDENT, txt, m_filepos, m_filepos));
 m_filepos.increment_column (flex_yyleng ());
 return FQ_IDENT;
 }
 // keywords and other special names and return STRUCT_ELT, which is
 // a string that is also a valid identifier.
 if (m_looking_at_indirect_ref)
 {
-push_token (new token (STRUCT_ELT, ident, m_filepos));
+push_token (new token (STRUCT_ELT, ident, m_filepos, m_filepos));
 m_looking_for_object_index = true;
 m_filepos.increment_column (flex_yyleng ());
 symbol_scope scope = m_symtab_context.curr_scope ();
 symbol_record sr = (scope ? scope.insert (ident) : symbol_record (ident));
-token *tok = new token (NAME, sr, m_filepos);
+token *tok = new token (NAME, sr, m_filepos, m_filepos);
 // The following symbols are handled specially so that things like
 //
 //   pi +1
 //
 base_lexer::handle_op_internal (int tok, bool bos, bool compat)
 {
 if (! compat)
 warn_language_extension_operator (flex_yytext ());
-push_token (new token (tok, m_filepos));
+push_token (new token (tok, m_filepos, m_filepos));
 m_filepos.increment_column (flex_yyleng ());
 m_looking_for_object_index = false;
 m_at_beginning_of_statement = bos;
 }
 int
 base_lexer::handle_token (const std::string& name, int tok)
 {
-token *tok_val = new token (tok, name, m_filepos);
+token *tok_val = new token (tok, name, m_filepos, m_filepos);
 return handle_token (tok, tok_val);
 }
 int
 base_lexer::handle_token (int tok, token *tok_val)
 {
 if (! tok_val)
-tok_val = new token (tok, m_filepos);
+tok_val = new token (tok, m_filepos, m_filepos);
 push_token (tok_val);
 m_filepos.increment_column (flex_yyleng ());
 }
 int
 base_lexer::count_token (int tok)
 {
-token *tok_val = new token (tok, m_filepos);
+token *tok_val = new token (tok, m_filepos, m_filepos);
 push_token (tok_val);
 return count_token_internal (tok);
 }

Mercurial > octave

comparison libinterp/parse-tree/lex.ll @ 27769:07ffed9878ad