octave: libinterp/parse-tree/lex.ll comparison

comparison libinterp/parse-tree/lex.ll @ 16119:b31eb56f4d84

maint: reorder class definitions in lex.ll

author	John W. Eaton <jwe@octave.org>
date	Tue, 26 Feb 2013 12:01:54 -0500
parents	f8e463523229
children	4b68eb9b98b0

comparison

equal deleted inserted replaced

-:f8e463523229
+:b31eb56f4d84
 if (! eof)
 YY_FATAL_ERROR ("octave_read () in flex scanner failed");
 }
 return status;
+}
+DEFUN (__display_tokens__, args, nargout,
+"-*- texinfo -*-\n\
+@deftypefn {Built-in Function} {} __display_tokens__ ()\n\
+Query or set the internal variable that determines whether Octave's\n\
+lexer displays tokens as they are read.\n\
+@end deftypefn")
+{
+return SET_INTERNAL_VARIABLE (display_tokens);
+}
+DEFUN (__token_count__, , ,
+"-*- texinfo -*-\n\
+@deftypefn {Built-in Function} {} __token_count__ ()\n\
+Number of language tokens processed since Octave startup.\n\
+@end deftypefn")
+{
+return octave_value (Vtoken_count);
+}
+DEFUN (__lexer_debug_flag__, args, nargout,
+"-*- texinfo -*-\n\
+@deftypefn {Built-in Function} {@var{old_val} =} __lexer_debug_flag__ (@var{new_val}))\n\
+Undocumented internal function.\n\
+@end deftypefn")
+{
+octave_value retval;
+retval = set_internal_variable (lexer_debug_flag, args, nargout,
+"__lexer_debug_flag__");
+return retval;
+}
+class
+flex_stream_reader : public stream_reader
+{
+public:
+flex_stream_reader (lexical_feedback *l, char *buf_arg)
+: stream_reader (), lexer (l), buf (buf_arg)
+{ }
+int getc (void) { return lexer->text_yyinput (); }
+int ungetc (int c) { lexer->xunput (c, buf); return 0; }
+private:
+// No copying!
+flex_stream_reader (const flex_stream_reader&);
+flex_stream_reader& operator = (const flex_stream_reader&);
+lexical_feedback *lexer;
+char *buf;
+};
+lexical_feedback::~lexical_feedback (void)
+{
+// Clear out the stack of token info used to track line and
+// column numbers.
+while (! token_stack.empty ())
+{
+delete token_stack.top ();
+token_stack.pop ();
+}
+}
+// GAG.
+//
+// If we're reading a matrix and the next character is '[', make sure
+// that we insert a comma ahead of it.
+void
+lexical_feedback::do_comma_insert_check (void)
+{
+bool spc_gobbled = (eat_continuation () != lexical_feedback::NO_WHITESPACE);
+int c = text_yyinput ();
+xunput (c, yytext);
+if (spc_gobbled)
+xunput (' ', yytext);
+do_comma_insert = (! looking_at_object_index.front ()
+&& bracketflag && c == '[');
+}
+int
+lexical_feedback::text_yyinput (void)
+{
+int c = yyinput ();
+if (lexer_debug_flag)
+{
+std::cerr << "I: ";
+display_character (c);
+std::cerr << std::endl;
+}
+// Convert CRLF into just LF and single CR into LF.
+if (c == '\r')
+{
+c = yyinput ();
+if (lexer_debug_flag)
+{
+std::cerr << "I: ";
+display_character (c);
+std::cerr << std::endl;
+}
+if (c != '\n')
+{
+xunput (c, yytext);
+c = '\n';
+}
+}
+if (c == '\n')
+input_line_number++;
+return c;
+}
+void
+lexical_feedback::xunput (char c, char *buf)
+{
+if (lexer_debug_flag)
+{
+std::cerr << "U: ";
+display_character (c);
+std::cerr << std::endl;
+}
+if (c == '\n')
+input_line_number--;
+yyunput (c, buf);
+}
+// If we read some newlines, we need figure out what column we're
+// really looking at.
+void
+lexical_feedback::fixup_column_count (char *s)
+{
+char c;
+while ((c = *s++) != '\0')
+{
+if (c == '\n')
+{
+input_line_number++;
+current_input_column = 1;
+}
+else
+current_input_column++;
+}
+}
+bool
+lexical_feedback::inside_any_object_index (void)
+{
+bool retval = false;
+for (std::list<bool>::const_iterator i = looking_at_object_index.begin ();
+i != looking_at_object_index.end (); i++)
+{
+if (*i)
+{
+retval = true;
+break;
+}
+}
+return retval;
+}
+// Handle keywords.  Return -1 if the keyword should be ignored.
+int
+lexical_feedback::is_keyword_token (const std::string& s)
+{
+int l = input_line_number;
+int c = current_input_column;
+int len = s.length ();
+const octave_kw *kw = octave_kw_hash::in_word_set (s.c_str (), len);
+if (kw)
+{
+yylval.tok_val = 0;
+switch (kw->kw_id)
+{
+case break_kw:
+case catch_kw:
+case continue_kw:
+case else_kw:
+case otherwise_kw:
+case return_kw:
+case unwind_protect_cleanup_kw:
+at_beginning_of_statement = true;
+break;
+case static_kw:
+if ((reading_fcn_file || reading_script_file
+|| reading_classdef_file)
+&& ! curr_fcn_file_full_name.empty ())
+warning_with_id ("Octave:deprecated-keyword",
+"the 'static' keyword is obsolete and will be removed from a future version of Octave; please use 'persistent' instead; near line %d of file '%s'",
+input_line_number,
+curr_fcn_file_full_name.c_str ());
+else
+warning_with_id ("Octave:deprecated-keyword",
+"the 'static' keyword is obsolete and will be removed from a future version of Octave; please use 'persistent' instead; near line %d",
+input_line_number);
+// fall through ...
+case persistent_kw:
+break;
+case case_kw:
+case elseif_kw:
+case global_kw:
+case until_kw:
+break;
+case end_kw:
+if (inside_any_object_index ()
+|| (! reading_classdef_file
+&& (defining_func
+&& ! (looking_at_return_list
+|| parsed_function_name.top ()))))
+return 0;
+yylval.tok_val = new token (token::simple_end, l, c);
+at_beginning_of_statement = true;
+break;
+case end_try_catch_kw:
+yylval.tok_val = new token (token::try_catch_end, l, c);
+at_beginning_of_statement = true;
+break;
+case end_unwind_protect_kw:
+yylval.tok_val = new token (token::unwind_protect_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endfor_kw:
+yylval.tok_val = new token (token::for_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endfunction_kw:
+yylval.tok_val = new token (token::function_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endif_kw:
+yylval.tok_val = new token (token::if_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endparfor_kw:
+yylval.tok_val = new token (token::parfor_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endswitch_kw:
+yylval.tok_val = new token (token::switch_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endwhile_kw:
+yylval.tok_val = new token (token::while_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endclassdef_kw:
+yylval.tok_val = new token (token::classdef_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endenumeration_kw:
+yylval.tok_val = new token (token::enumeration_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endevents_kw:
+yylval.tok_val = new token (token::events_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endmethods_kw:
+yylval.tok_val = new token (token::methods_end, l, c);
+at_beginning_of_statement = true;
+break;
+case endproperties_kw:
+yylval.tok_val = new token (token::properties_end, l, c);
+at_beginning_of_statement = true;
+break;
+case for_kw:
+case parfor_kw:
+case while_kw:
+promptflag--;
+looping++;
+break;
+case do_kw:
+at_beginning_of_statement = true;
+promptflag--;
+looping++;
+break;
+case try_kw:
+case unwind_protect_kw:
+at_beginning_of_statement = true;
+promptflag--;
+break;
+case if_kw:
+case switch_kw:
+promptflag--;
+break;
+case get_kw:
+case set_kw:
+// 'get' and 'set' are keywords in classdef method
+// declarations.
+if (! maybe_classdef_get_set_method)
+return 0;
+break;
+case enumeration_kw:
+case events_kw:
+case methods_kw:
+case properties_kw:
+// 'properties', 'methods' and 'events' are keywords for
+// classdef blocks.
+if (! parsing_classdef)
+return 0;
+// fall through ...
+case classdef_kw:
+// 'classdef' is always a keyword.
+promptflag--;
+break;
+case function_kw:
+promptflag--;
+defining_func++;
+parsed_function_name.push (false);
+if (! (reading_fcn_file || reading_script_file
+|| reading_classdef_file))
+input_line_number = 1;
+break;
+case magic_file_kw:
+{
+if ((reading_fcn_file || reading_script_file
+|| reading_classdef_file)
+&& ! curr_fcn_file_full_name.empty ())
+yylval.tok_val = new token (curr_fcn_file_full_name, l, c);
+else
+yylval.tok_val = new token ("stdin", l, c);
+}
+break;
+case magic_line_kw:
+yylval.tok_val = new token (static_cast<double> (l), "", l, c);
+break;
+default:
+panic_impossible ();
+}
+if (! yylval.tok_val)
+yylval.tok_val = new token (l, c);
+token_stack.push (yylval.tok_val);
+return kw->tok;
+}
+return 0;
+}
+bool
+lexical_feedback::is_variable (const std::string& name)
+{
+return (symbol_table::is_variable (name)
+|| (pending_local_variables.find (name)
+!= pending_local_variables.end ()));
+}
+std::string
+lexical_feedback::grab_block_comment (stream_reader& reader, bool& eof)
+{
+std::string buf;
+bool at_bol = true;
+bool look_for_marker = false;
+bool warned_incompatible = false;
+int c = 0;
+while ((c = reader.getc ()) != EOF)
+{
+current_input_column++;
+if (look_for_marker)
+{
+at_bol = false;
+look_for_marker = false;
+if (c == '{' || c == '}')
+{
+std::string tmp_buf (1, static_cast<char> (c));
+int type = c;
+bool done = false;
+while ((c = reader.getc ()) != EOF && ! done)
+{
+current_input_column++;
+switch (c)
+{
+case ' ':
+case '\t':
+tmp_buf += static_cast<char> (c);
+break;
+case '\n':
+{
+current_input_column = 0;
+at_bol = true;
+done = true;
+if (type == '{')
+{
+block_comment_nesting_level++;
+promptflag--;
+}
+else
+{
+block_comment_nesting_level--;
+promptflag++;
+if (block_comment_nesting_level == 0)
+{
+buf += grab_comment_block (reader, true, eof);
+return buf;
+}
+}
+}
+break;
+default:
+at_bol = false;
+tmp_buf += static_cast<char> (c);
+buf += tmp_buf;
+done = true;
+break;
+}
+}
+}
+}
+if (at_bol && (c == '%' || c == '#'))
+{
+if (c == '#' && ! warned_incompatible)
+{
+warned_incompatible = true;
+maybe_gripe_matlab_incompatible_comment (c);
+}
+at_bol = false;
+look_for_marker = true;
+}
+else
+{
+buf += static_cast<char> (c);
+if (c == '\n')
+{
+current_input_column = 0;
+at_bol = true;
+}
+}
+}
+if (c == EOF)
+eof = true;
+return buf;
+}
+std::string
+lexical_feedback::grab_comment_block (stream_reader& reader, bool at_bol,
+bool& eof)
+{
+std::string buf;
+// TRUE means we are at the beginning of a comment block.
+bool begin_comment = false;
+// TRUE means we are currently reading a comment block.
+bool in_comment = false;
+bool warned_incompatible = false;
+int c = 0;
+while ((c = reader.getc ()) != EOF)
+{
+current_input_column++;
+if (begin_comment)
+{
+if (c == '%' || c == '#')
+{
+at_bol = false;
+continue;
+}
+else if (at_bol && c == '{')
+{
+std::string tmp_buf (1, static_cast<char> (c));
+bool done = false;
+while ((c = reader.getc ()) != EOF && ! done)
+{
+current_input_column++;
+switch (c)
+{
+case ' ':
+case '\t':
+tmp_buf += static_cast<char> (c);
+break;
+case '\n':
+{
+current_input_column = 0;
+at_bol = true;
+done = true;
+block_comment_nesting_level++;
+promptflag--;
+buf += grab_block_comment (reader, eof);
+in_comment = false;
+if (eof)
+goto done;
+}
+break;
+default:
+at_bol = false;
+tmp_buf += static_cast<char> (c);
+buf += tmp_buf;
+done = true;
+break;
+}
+}
+}
+else
+{
+at_bol = false;
+begin_comment = false;
+}
+}
+if (in_comment)
+{
+buf += static_cast<char> (c);
+if (c == '\n')
+{
+at_bol = true;
+current_input_column = 0;
+in_comment = false;
+// FIXME -- bailing out here prevents things like
+//
+//    octave> # comment
+//    octave> x = 1
+//
+// from failing at the command line, while still
+// allowing blocks of comments to be grabbed properly
+// for function doc strings.  But only the first line of
+// a mult-line doc string will be picked up for
+// functions defined on the command line.  We need a
+// better way of collecting these comments...
+if (! (reading_fcn_file || reading_script_file))
+goto done;
+}
+}
+else
+{
+switch (c)
+{
+case ' ':
+case '\t':
+break;
+case '#':
+if (! warned_incompatible)
+{
+warned_incompatible = true;
+maybe_gripe_matlab_incompatible_comment (c);
+}
+// fall through...
+case '%':
+in_comment = true;
+begin_comment = true;
+break;
+default:
+current_input_column--;
+reader.ungetc (c);
+goto done;
+}
+}
+}
+done:
+if (c == EOF)
+eof = true;
+return buf;
+}
+int
+lexical_feedback::process_comment (bool start_in_block, bool& eof)
+{
+eof = false;
+std::string help_txt;
+if (! help_buf.empty ())
+help_txt = help_buf.top ();
+flex_stream_reader flex_reader (this, yytext);
+// process_comment is only supposed to be called when we are not
+// initially looking at a block comment.
+std::string txt = start_in_block
+? grab_block_comment (flex_reader, eof)
+: grab_comment_block (flex_reader, false, eof);
+if (lexer_debug_flag)
+std::cerr << "C: " << txt << std::endl;
+if (help_txt.empty () && nesting_level.none ())
+{
+if (! help_buf.empty ())
+help_buf.pop ();
+help_buf.push (txt);
+}
+octave_comment_buffer::append (txt);
+current_input_column = 1;
+quote_is_transpose = false;
+convert_spaces_to_comma = true;
+at_beginning_of_statement = true;
+if (YY_START == COMMAND_START)
+BEGIN (INITIAL);
+if (nesting_level.none ())
+return '\n';
+else if (nesting_level.is_bracket_or_brace ())
+return ';';
+else
+return 0;
+}
+// Recognize separators.  If the separator is a CRLF pair, it is
+// replaced by a single LF.
+bool
+lexical_feedback::next_token_is_sep_op (void)
+{
+bool retval = false;
+int c = text_yyinput ();
+retval = match_any (c, ",;\n]");
+xunput (c, yytext);
+return retval;
+}
+// Try to determine if the next token should be treated as a postfix
+// unary operator.  This is ugly, but it seems to do the right thing.
+bool
+lexical_feedback::next_token_is_postfix_unary_op (bool spc_prev)
+{
+bool un_op = false;
+int c0 = text_yyinput ();
+if (c0 == '\'' && ! spc_prev)
+{
+un_op = true;
+}
+else if (c0 == '.')
+{
+int c1 = text_yyinput ();
+un_op = (c1 == '\'');
+xunput (c1, yytext);
+}
+else if (c0 == '+')
+{
+int c1 = text_yyinput ();
+un_op = (c1 == '+');
+xunput (c1, yytext);
+}
+else if (c0 == '-')
+{
+int c1 = text_yyinput ();
+un_op = (c1 == '-');
+xunput (c1, yytext);
+}
+xunput (c0, yytext);
+return un_op;
+}
+// Try to determine if the next token should be treated as a binary
+// operator.
+//
+// This kluge exists because whitespace is not always ignored inside
+// the square brackets that are used to create matrix objects (though
+// spacing only really matters in the cases that can be interpreted
+// either as binary ops or prefix unary ops: currently just +, -).
+//
+// Note that a line continuation directly following a + or - operator
+// (e.g., the characters '[' 'a' ' ' '+' '\' LFD 'b' ']') will be
+// parsed as a binary operator.
+bool
+lexical_feedback::next_token_is_bin_op (bool spc_prev)
+{
+bool bin_op = false;
+int c0 = text_yyinput ();
+switch (c0)
+{
+case '+':
+case '-':
+{
+int c1 = text_yyinput ();
+switch (c1)
+{
+case '+':
+case '-':
+// Unary ops, spacing doesn't matter.
+break;
+case '=':
+// Binary ops, spacing doesn't matter.
+bin_op = true;
+break;
+default:
+// Could be either, spacing matters.
+bin_op = looks_like_bin_op (spc_prev, c1);
+break;
+}
+xunput (c1, yytext);
+}
+break;
+case ':':
+case '/':
+case '\\':
+case '^':
+// Always a binary op (may also include /=, \=, and ^=).
+bin_op = true;
+break;
+// .+ .- ./ .\ .^ .* .**
+case '.':
+{
+int c1 = text_yyinput ();
+if (match_any (c1, "+-/\\^*"))
+// Always a binary op (may also include .+=, .-=, ./=, ...).
+bin_op = true;
+else if (! isdigit (c1) && c1 != ' ' && c1 != '\t' && c1 != '.')
+// A structure element reference is a binary op.
+bin_op = true;
+xunput (c1, yytext);
+}
+break;
+// = == & && | || * **
+case '=':
+case '&':
+case '|':
+case '*':
+// Always a binary op (may also include ==, &&, ||, **).
+bin_op = true;
+break;
+// < <= <> > >=
+case '<':
+case '>':
+// Always a binary op (may also include <=, <>, >=).
+bin_op = true;
+break;
+// ~= !=
+case '~':
+case '!':
+{
+int c1 = text_yyinput ();
+// ~ and ! can be unary ops, so require following =.
+if (c1 == '=')
+bin_op = true;
+xunput (c1, yytext);
+}
+break;
+default:
+break;
+}
+xunput (c0, yytext);
+return bin_op;
+}
+// FIXME -- we need to handle block comments here.
+void
+lexical_feedback::scan_for_comments (const char *text)
+{
+std::string comment_buf;
+bool in_comment = false;
+bool beginning_of_comment = false;
+int len = strlen (text);
+int i = 0;
+while (i < len)
+{
+char c = text[i++];
+switch (c)
+{
+case '%':
+case '#':
+if (in_comment)
+{
+if (! beginning_of_comment)
+comment_buf += static_cast<char> (c);
+}
+else
+{
+maybe_gripe_matlab_incompatible_comment (c);
+in_comment = true;
+beginning_of_comment = true;
+}
+break;
+case '\n':
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+octave_comment_buffer::append (comment_buf);
+comment_buf.resize (0);
+in_comment = false;
+beginning_of_comment = false;
+}
+break;
+default:
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+beginning_of_comment = false;
+}
+break;
+}
+}
+if (! comment_buf.empty ())
+octave_comment_buffer::append (comment_buf);
+}
+// Discard whitespace, including comments and continuations.
+// FIXME -- we need to handle block comments here.
+int
+lexical_feedback::eat_whitespace (void)
+{
+int retval = lexical_feedback::NO_WHITESPACE;
+std::string comment_buf;
+bool in_comment = false;
+bool beginning_of_comment = false;
+int c = 0;
+while ((c = text_yyinput ()) != EOF)
+{
+current_input_column++;
+switch (c)
+{
+case ' ':
+case '\t':
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+beginning_of_comment = false;
+}
+retval |= lexical_feedback::SPACE_OR_TAB;
+break;
+case '\n':
+retval |= lexical_feedback::NEWLINE;
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+octave_comment_buffer::append (comment_buf);
+comment_buf.resize (0);
+in_comment = false;
+beginning_of_comment = false;
+}
+current_input_column = 0;
+break;
+case '#':
+case '%':
+if (in_comment)
+{
+if (! beginning_of_comment)
+comment_buf += static_cast<char> (c);
+}
+else
+{
+maybe_gripe_matlab_incompatible_comment (c);
+in_comment = true;
+beginning_of_comment = true;
+}
+break;
+case '.':
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+beginning_of_comment = false;
+break;
+}
+else
+{
+if (have_ellipsis_continuation ())
+break;
+else
+goto done;
+}
+case '\\':
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+beginning_of_comment = false;
+break;
+}
+else
+{
+if (have_continuation ())
+break;
+else
+goto done;
+}
+default:
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+beginning_of_comment = false;
+break;
+}
+else
+goto done;
+}
+}
+if (! comment_buf.empty ())
+octave_comment_buffer::append (comment_buf);
+done:
+xunput (c, yytext);
+current_input_column--;
+return retval;
+}
+static inline bool
+looks_like_hex (const char *s, int len)
+{
+return (len > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X'));
+}
+void
+lexical_feedback::handle_number (void)
+{
+double value = 0.0;
+int nread = 0;
+if (looks_like_hex (yytext, strlen (yytext)))
+{
+unsigned long ival;
+nread = sscanf (yytext, "%lx", &ival);
+value = static_cast<double> (ival);
+}
+else
+{
+char *tmp = strsave (yytext);
+char *idx = strpbrk (tmp, "Dd");
+if (idx)
+*idx = 'e';
+nread = sscanf (tmp, "%lf", &value);
+delete [] tmp;
+}
+// If yytext doesn't contain a valid number, we are in deep doo doo.
+assert (nread == 1);
+quote_is_transpose = true;
+convert_spaces_to_comma = true;
+looking_for_object_index = false;
+at_beginning_of_statement = false;
+yylval.tok_val = new token (value, yytext, input_line_number,
+current_input_column);
+token_stack.push (yylval.tok_val);
+current_input_column += yyleng;
+do_comma_insert_check ();
+}
+// We have seen a backslash and need to find out if it should be
+// treated as a continuation character.  If so, this eats it, up to
+// and including the new line character.
+//
+// Match whitespace only, followed by a comment character or newline.
+// Once a comment character is found, discard all input until newline.
+// If non-whitespace characters are found before comment
+// characters, return 0.  Otherwise, return 1.
+// FIXME -- we need to handle block comments here.
+bool
+lexical_feedback::have_continuation (bool trailing_comments_ok)
+{
+std::ostringstream buf;
+std::string comment_buf;
+bool in_comment = false;
+bool beginning_of_comment = false;
+int c = 0;
+while ((c = text_yyinput ()) != EOF)
+{
+buf << static_cast<char> (c);
+switch (c)
+{
+case ' ':
+case '\t':
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+beginning_of_comment = false;
+}
+break;
+case '%':
+case '#':
+if (trailing_comments_ok)
+{
+if (in_comment)
+{
+if (! beginning_of_comment)
+comment_buf += static_cast<char> (c);
+}
+else
+{
+maybe_gripe_matlab_incompatible_comment (c);
+in_comment = true;
+beginning_of_comment = true;
+}
+}
+else
+goto cleanup;
+break;
+case '\n':
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+octave_comment_buffer::append (comment_buf);
+}
+current_input_column = 0;
+promptflag--;
+gripe_matlab_incompatible_continuation ();
+return true;
+default:
+if (in_comment)
+{
+comment_buf += static_cast<char> (c);
+beginning_of_comment = false;
+}
+else
+goto cleanup;
+break;
+}
+}
+xunput (c, yytext);
+return false;
+cleanup:
+std::string s = buf.str ();
+int len = s.length ();
+while (len--)
+xunput (s[len], yytext);
+return false;
+}
+// We have seen a '.' and need to see if it is the start of a
+// continuation.  If so, this eats it, up to and including the new
+// line character.
+bool
+lexical_feedback::have_ellipsis_continuation (bool trailing_comments_ok)
+{
+char c1 = text_yyinput ();
+if (c1 == '.')
+{
+char c2 = text_yyinput ();
+if (c2 == '.' && have_continuation (trailing_comments_ok))
+return true;
+else
+{
+xunput (c2, yytext);
+xunput (c1, yytext);
+}
+}
+else
+xunput (c1, yytext);
+return false;
+}
+// See if we have a continuation line.  If so, eat it and the leading
+// whitespace on the next line.
+int
+lexical_feedback::eat_continuation (void)
+{
+int retval = lexical_feedback::NO_WHITESPACE;
+int c = text_yyinput ();
+if ((c == '.' && have_ellipsis_continuation ())
+|| (c == '\\' && have_continuation ()))
+retval = eat_whitespace ();
+else
+xunput (c, yytext);
+return retval;
+}
+int
+lexical_feedback::handle_string (char delim)
+{
+std::ostringstream buf;
+int bos_line = input_line_number;
+int bos_col = current_input_column;
+int c;
+int escape_pending = 0;
+while ((c = text_yyinput ()) != EOF)
+{
+current_input_column++;
+if (c == '\\')
+{
+if (delim == '\'' || escape_pending)
+{
+buf << static_cast<char> (c);
+escape_pending = 0;
+}
+else
+{
+if (have_continuation (false))
+escape_pending = 0;
+else
+{
+buf << static_cast<char> (c);
+escape_pending = 1;
+}
+}
+continue;
+}
+else if (c == '.')
+{
+if (delim == '\'' || ! have_ellipsis_continuation (false))
+buf << static_cast<char> (c);
+}
+else if (c == '\n')
+{
+error ("unterminated string constant");
+break;
+}
+else if (c == delim)
+{
+if (escape_pending)
+buf << static_cast<char> (c);
+else
+{
+c = text_yyinput ();
+if (c == delim)
+{
+buf << static_cast<char> (c);
+}
+else
+{
+std::string s;
+xunput (c, yytext);
+if (delim == '\'')
+s = buf.str ();
+else
+s = do_string_escapes (buf.str ());
+quote_is_transpose = true;
+convert_spaces_to_comma = true;
+yylval.tok_val = new token (s, bos_line, bos_col);
+token_stack.push (yylval.tok_val);
+if (delim == '"')
+gripe_matlab_incompatible ("\" used as string delimiter");
+else if (delim == '\'')
+gripe_single_quote_string ();
+looking_for_object_index = true;
+at_beginning_of_statement = false;
+return delim == '"' ? DQ_STRING : SQ_STRING;
+}
+}
+}
+else
+{
+buf << static_cast<char> (c);
+}
+escape_pending = 0;
+}
+return LEXICAL_ERROR;
+}
+bool
+lexical_feedback::next_token_is_assign_op (void)
+{
+bool retval = false;
+int c0 = text_yyinput ();
+switch (c0)
+{
+case '=':
+{
+int c1 = text_yyinput ();
+xunput (c1, yytext);
+if (c1 != '=')
+retval = true;
+}
+break;
+case '+':
+case '-':
+case '*':
+case '/':
+case '\\':
+case '&':
+case '|':
+{
+int c1 = text_yyinput ();
+xunput (c1, yytext);
+if (c1 == '=')
+retval = true;
+}
+break;
+case '.':
+{
+int c1 = text_yyinput ();
+if (match_any (c1, "+-*/\\"))
+{
+int c2 = text_yyinput ();
+xunput (c2, yytext);
+if (c2 == '=')
+retval = true;
+}
+xunput (c1, yytext);
+}
+break;
+case '>':
+{
+int c1 = text_yyinput ();
+if (c1 == '>')
+{
+int c2 = text_yyinput ();
+xunput (c2, yytext);
+if (c2 == '=')
+retval = true;
+}
+xunput (c1, yytext);
+}
+break;
+case '<':
+{
+int c1 = text_yyinput ();
+if (c1 == '<')
+{
+int c2 = text_yyinput ();
+xunput (c2, yytext);
+if (c2 == '=')
+retval = true;
+}
+xunput (c1, yytext);
+}
+break;
+default:
+break;
+}
+xunput (c0, yytext);
+return retval;
+}
+bool
+lexical_feedback::next_token_is_index_op (void)
+{
+int c = text_yyinput ();
+xunput (c, yytext);
+return c == '(' || c == '{';
+}
+int
+lexical_feedback::handle_close_bracket (bool spc_gobbled, int bracket_type)
+{
+int retval = bracket_type;
+if (! nesting_level.none ())
+{
+nesting_level.remove ();
+if (bracket_type == ']')
+bracketflag--;
+else if (bracket_type == '}')
+braceflag--;
+else
+panic_impossible ();
+}
+if (bracketflag == 0 && braceflag == 0)
+BEGIN (INITIAL);
+if (bracket_type == ']'
+&& next_token_is_assign_op ()
+&& ! looking_at_return_list)
+{
+retval = CLOSE_BRACE;
+}
+else if ((bracketflag || braceflag)
+&& convert_spaces_to_comma
+&& (nesting_level.is_bracket ()
+|| (nesting_level.is_brace ()
+&& ! looking_at_object_index.front ())))
+{
+bool index_op = next_token_is_index_op ();
+// Don't insert comma if we are looking at something like
+//
+//   [x{i}{j}] or [x{i}(j)]
+//
+// but do if we are looking at
+//
+//   [x{i} {j}] or [x{i} (j)]
+if (spc_gobbled || ! (bracket_type == '}' && index_op))
+{
+bool bin_op = next_token_is_bin_op (spc_gobbled);
+bool postfix_un_op = next_token_is_postfix_unary_op (spc_gobbled);
+bool sep_op = next_token_is_sep_op ();
+if (! (postfix_un_op || bin_op || sep_op))
+{
+maybe_warn_separator_insert (',');
+xunput (',', yytext);
+return retval;
+}
+}
+}
+quote_is_transpose = true;
+convert_spaces_to_comma = true;
+return retval;
+}
+void
+lexical_feedback::maybe_unput_comma (int spc_gobbled)
+{
+if (nesting_level.is_bracket ()
+|| (nesting_level.is_brace ()
+&& ! looking_at_object_index.front ()))
+{
+int bin_op = next_token_is_bin_op (spc_gobbled);
+int postfix_un_op = next_token_is_postfix_unary_op (spc_gobbled);
+int c1 = text_yyinput ();
+int c2 = text_yyinput ();
+xunput (c2, yytext);
+xunput (c1, yytext);
+int sep_op = next_token_is_sep_op ();
+int dot_op = (c1 == '.'
+&& (isalpha (c2) || isspace (c2) || c2 == '_'));
+if (postfix_un_op || bin_op || sep_op || dot_op)
+return;
+int index_op = (c1 == '(' || c1 == '{');
+// If there is no space before the indexing op, we don't insert
+// a comma.
+if (index_op && ! spc_gobbled)
+return;
+maybe_warn_separator_insert (',');
+xunput (',', yytext);
+}
+}
+bool
+lexical_feedback::next_token_can_follow_bin_op (void)
+{
+std::stack<char> buf;
+int c = EOF;
+// Skip whitespace in current statement on current line
+while (true)
+{
+c = text_yyinput ();
+buf.push (c);
+if (match_any (c, ",;\n") || (c != ' ' && c != '\t'))
+break;
+}
+// Restore input.
+while (! buf.empty ())
+{
+xunput (buf.top (), yytext);
+buf.pop ();
+}
+return (isalnum (c) || match_any (c, "!\"'(-[_{~"));
+}
+static bool
+can_be_command (const std::string& tok)
+{
+// Don't allow these names to be treated as commands to avoid
+// surprises when parsing things like "NaN ^2".
+return ! (tok == "e"
+|| tok == "I" || tok == "i"
+|| tok == "J" || tok == "j"
+|| tok == "Inf" || tok == "inf"
+|| tok == "NaN" || tok == "nan");
+}
+bool
+lexical_feedback::looks_like_command_arg (void)
+{
+bool retval = true;
+int c0 = text_yyinput ();
+switch (c0)
+{
+// = ==
+case '=':
+{
+int c1 = text_yyinput ();
+if (c1 == '=')
+{
+int c2 = text_yyinput ();
+if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c2, yytext);
+}
+else
+retval = false;
+xunput (c1, yytext);
+}
+break;
+case '(':
+case '{':
+// Indexing.
+retval = false;
+break;
+case '\n':
+// EOL.
+break;
+case '\'':
+case '"':
+// Beginning of a character string.
+break;
+// + - ++ -- += -=
+case '+':
+case '-':
+{
+int c1 = text_yyinput ();
+switch (c1)
+{
+case '\n':
+// EOL.
+case '+':
+case '-':
+// Unary ops, spacing doesn't matter.
+break;
+case '\t':
+case ' ':
+{
+if (next_token_can_follow_bin_op ())
+retval = false;
+}
+break;
+case '=':
+{
+int c2 = text_yyinput ();
+if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c2, yytext);
+}
+break;
+}
+xunput (c1, yytext);
+}
+break;
+case ':':
+case '/':
+case '\\':
+case '^':
+{
+int c1 = text_yyinput ();
+if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c1, yytext);
+}
+break;
+// .+ .- ./ .\ .^ .* .**
+case '.':
+{
+int c1 = text_yyinput ();
+if (match_any (c1, "+-/\\^*"))
+{
+int c2 = text_yyinput ();
+if (c2 == '=')
+{
+int c3 = text_yyinput ();
+if (! match_any (c3, ",;\n") && (c3 == ' ' || c3 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c3, yytext);
+}
+else if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c2, yytext);
+}
+else if (! match_any (c1, ",;\n")
+&& (! isdigit (c1) && c1 != ' ' && c1 != '\t'
+&& c1 != '.'))
+{
+// Structure reference.  FIXME -- is this a complete check?
+retval = false;
+}
+xunput (c1, yytext);
+}
+break;
+// & && | || * **
+case '&':
+case '|':
+case '*':
+{
+int c1 = text_yyinput ();
+if (c1 == c0)
+{
+int c2 = text_yyinput ();
+if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c2, yytext);
+}
+else if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c1, yytext);
+}
+break;
+// < <= > >=
+case '<':
+case '>':
+{
+int c1 = text_yyinput ();
+if (c1 == '=')
+{
+int c2 = text_yyinput ();
+if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c2, yytext);
+}
+else if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c1, yytext);
+}
+break;
+// ~= !=
+case '~':
+case '!':
+{
+int c1 = text_yyinput ();
+// ~ and ! can be unary ops, so require following =.
+if (c1 == '=')
+{
+int c2 = text_yyinput ();
+if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c2, yytext);
+}
+else if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
+&& next_token_can_follow_bin_op ())
+retval = false;
+xunput (c1, yytext);
+}
+break;
+default:
+break;
+}
+xunput (c0, yytext);
+return retval;
+}
+int
+lexical_feedback::handle_superclass_identifier (void)
+{
+eat_continuation ();
+std::string pkg;
+std::string meth = strip_trailing_whitespace (yytext);
+size_t pos = meth.find ("@");
+std::string cls = meth.substr (pos).substr (1);
+meth = meth.substr (0, pos - 1);
+pos = cls.find (".");
+if (pos != std::string::npos)
+{
+pkg = cls.substr (pos).substr (1);
+cls = cls.substr (0, pos - 1);
+}
+int kw_token = (is_keyword_token (meth) || is_keyword_token (cls)
+|| is_keyword_token (pkg));
+if (kw_token)
+{
+error ("method, class and package names may not be keywords");
+return LEXICAL_ERROR;
+}
+yylval.tok_val
+= new token (meth.empty () ? 0 : &(symbol_table::insert (meth)),
+cls.empty () ? 0 : &(symbol_table::insert (cls)),
+pkg.empty () ? 0 : &(symbol_table::insert (pkg)),
+input_line_number,
+current_input_column);
+token_stack.push (yylval.tok_val);
+convert_spaces_to_comma = true;
+current_input_column += yyleng;
+return SUPERCLASSREF;
+}
+int
+lexical_feedback::handle_meta_identifier (void)
+{
+eat_continuation ();
+std::string pkg;
+std::string cls = strip_trailing_whitespace (yytext).substr (1);
+size_t pos = cls.find (".");
+if (pos != std::string::npos)
+{
+pkg = cls.substr (pos).substr (1);
+cls = cls.substr (0, pos - 1);
+}
+int kw_token = is_keyword_token (cls) || is_keyword_token (pkg);
+if (kw_token)
+{
+error ("class and package names may not be keywords");
+return LEXICAL_ERROR;
+}
+yylval.tok_val
+= new token (cls.empty () ? 0 : &(symbol_table::insert (cls)),
+pkg.empty () ? 0 : &(symbol_table::insert (pkg)),
+input_line_number,
+current_input_column);
+token_stack.push (yylval.tok_val);
+convert_spaces_to_comma = true;
+current_input_column += yyleng;
+return METAQUERY;
+}
+// Figure out exactly what kind of token to return when we have seen
+// an identifier.  Handles keywords.  Return -1 if the identifier
+// should be ignored.
+int
+lexical_feedback::handle_identifier (void)
+{
+bool at_bos = at_beginning_of_statement;
+std::string tok = strip_trailing_whitespace (yytext);
+int c = yytext[yyleng-1];
+bool cont_is_spc = (eat_continuation () != lexical_feedback::NO_WHITESPACE);
+int spc_gobbled = (cont_is_spc || c == ' ' || c == '\t');
+// If we are expecting a structure element, avoid recognizing
+// keywords and other special names and return STRUCT_ELT, which is
+// a string that is also a valid identifier.  But first, we have to
+// decide whether to insert a comma.
+if (looking_at_indirect_ref)
+{
+do_comma_insert_check ();
+maybe_unput_comma (spc_gobbled);
+yylval.tok_val = new token (tok, input_line_number,
+current_input_column);
+token_stack.push (yylval.tok_val);
+quote_is_transpose = true;
+convert_spaces_to_comma = true;
+looking_for_object_index = true;
+current_input_column += yyleng;
+return STRUCT_ELT;
+}
+at_beginning_of_statement = false;
+// The is_keyword_token may reset
+// at_beginning_of_statement.  For example, if it sees
+// an else token, then the next token is at the beginning of a
+// statement.
+int kw_token = is_keyword_token (tok);
+// If we found a keyword token, then the beginning_of_statement flag
+// is already set.  Otherwise, we won't be at the beginning of a
+// statement.
+if (looking_at_function_handle)
+{
+if (kw_token)
+{
+error ("function handles may not refer to keywords");
+return LEXICAL_ERROR;
+}
+else
+{
+yylval.tok_val = new token (tok, input_line_number,
+current_input_column);
+token_stack.push (yylval.tok_val);
+current_input_column += yyleng;
+quote_is_transpose = false;
+convert_spaces_to_comma = true;
+looking_for_object_index = true;
+return FCN_HANDLE;
+}
+}
+// If we have a regular keyword, return it.
+// Keywords can be followed by identifiers.
+if (kw_token)
+{
+if (kw_token >= 0)
+{
+current_input_column += yyleng;
+quote_is_transpose = false;
+convert_spaces_to_comma = true;
+looking_for_object_index = false;
+}
+return kw_token;
+}
+// See if we have a plot keyword (title, using, with, or clear).
+int c1 = text_yyinput ();
+bool next_tok_is_eq = false;
+if (c1 == '=')
+{
+int c2 = text_yyinput ();
+xunput (c2, yytext);
+if (c2 != '=')
+next_tok_is_eq = true;
+}
+xunput (c1, yytext);
+// Kluge alert.
+//
+// If we are looking at a text style function, set up to gobble its
+// arguments.
+//
+// If the following token is '=', or if we are parsing a function
+// return list or function parameter list, or if we are looking at
+// something like [ab,cd] = foo (), force the symbol to be inserted
+// as a variable in the current symbol table.
+if (! is_variable (tok))
+{
+if (at_bos && spc_gobbled && can_be_command (tok)
+&& looks_like_command_arg ())
+{
+BEGIN (COMMAND_START);
+}
+else if (next_tok_is_eq
+|| looking_at_decl_list
+|| looking_at_return_list
+|| (looking_at_parameter_list
+&& ! looking_at_initializer_expression))
+{
+symbol_table::force_variable (tok);
+}
+else if (looking_at_matrix_or_assign_lhs)
+{
+pending_local_variables.insert (tok);
+}
+}
+// Find the token in the symbol table.  Beware the magic
+// transformation of the end keyword...
+if (tok == "end")
+tok = "__end__";
+yylval.tok_val = new token (&(symbol_table::insert (tok)),
+input_line_number,
+current_input_column);
+token_stack.push (yylval.tok_val);
+// After seeing an identifer, it is ok to convert spaces to a comma
+// (if needed).
+convert_spaces_to_comma = true;
+if (! (next_tok_is_eq || YY_START == COMMAND_START))
+{
+quote_is_transpose = true;
+do_comma_insert_check ();
+maybe_unput_comma (spc_gobbled);
+}
+current_input_column += yyleng;
+if (tok != "__end__")
+looking_for_object_index = true;
+return NAME;
+}
+void
+lexical_feedback::maybe_warn_separator_insert (char sep)
+{
+std::string nm = curr_fcn_file_full_name;
+if (nm.empty ())
+warning_with_id ("Octave:separator-insert",
+"potential auto-insertion of '%c' near line %d",
+sep, input_line_number);
+else
+warning_with_id ("Octave:separator-insert",
+"potential auto-insertion of '%c' near line %d of file %s",
+sep, input_line_number, nm.c_str ());
+}
+void
+lexical_feedback::gripe_single_quote_string (void)
+{
+std::string nm = curr_fcn_file_full_name;
+if (nm.empty ())
+warning_with_id ("Octave:single-quote-string",
+"single quote delimited string near line %d",
+input_line_number);
+else
+warning_with_id ("Octave:single-quote-string",
+"single quote delimited string near line %d of file %s",
+input_line_number, nm.c_str ());
+}
+void
+lexical_feedback::gripe_matlab_incompatible (const std::string& msg)
+{
+std::string nm = curr_fcn_file_full_name;
+if (nm.empty ())
+warning_with_id ("Octave:matlab-incompatible",
+"potential Matlab compatibility problem: %s",
+msg.c_str ());
+else
+warning_with_id ("Octave:matlab-incompatible",
+"potential Matlab compatibility problem: %s near line %d offile %s",
+msg.c_str (), input_line_number, nm.c_str ());
+}
+void
+lexical_feedback::maybe_gripe_matlab_incompatible_comment (char c)
+{
+if (c == '#')
+gripe_matlab_incompatible ("# used as comment character");
+}
+void
+lexical_feedback::gripe_matlab_incompatible_continuation (void)
+{
+gripe_matlab_incompatible ("\\ used as line continuation marker");
+}
+void
+lexical_feedback::gripe_matlab_incompatible_operator (const std::string& op)
+{
+std::string t = op;
+int n = t.length ();
+if (t[n-1] == '\n')
+t.resize (n-1);
+gripe_matlab_incompatible (t + " used as operator");
 }
 void
 lexical_feedback::display_token (int tok)
 {
 display_state (YY_START);
 std::cerr << "P: " << pattern << std::endl;
 std::cerr << "T: " << text << std::endl;
 }
-DEFUN (__display_tokens__, args, nargout,
-"-*- texinfo -*-\n\
-@deftypefn {Built-in Function} {} __display_tokens__ ()\n\
-Query or set the internal variable that determines whether Octave's\n\
-lexer displays tokens as they are read.\n\
-@end deftypefn")
-{
-return SET_INTERNAL_VARIABLE (display_tokens);
-}
-DEFUN (__token_count__, , ,
-"-*- texinfo -*-\n\
-@deftypefn {Built-in Function} {} __token_count__ ()\n\
-Number of language tokens processed since Octave startup.\n\
-@end deftypefn")
-{
-return octave_value (Vtoken_count);
-}
-DEFUN (__lexer_debug_flag__, args, nargout,
-"-*- texinfo -*-\n\
-@deftypefn {Built-in Function} {@var{old_val} =} __lexer_debug_flag__ (@var{new_val}))\n\
-Undocumented internal function.\n\
-@end deftypefn")
-{
-octave_value retval;
-retval = set_internal_variable (lexer_debug_flag, args, nargout,
-"__lexer_debug_flag__");
-return retval;
-}
-class
-flex_stream_reader : public stream_reader
-{
-public:
-flex_stream_reader (lexical_feedback *l, char *buf_arg)
-: stream_reader (), lexer (l), buf (buf_arg)
-{ }
-int getc (void) { return lexer->text_yyinput (); }
-int ungetc (int c) { lexer->xunput (c, buf); return 0; }
-private:
-// No copying!
-flex_stream_reader (const flex_stream_reader&);
-flex_stream_reader& operator = (const flex_stream_reader&);
-lexical_feedback *lexer;
-char *buf;
-};
-lexical_feedback::~lexical_feedback (void)
-{
-// Clear out the stack of token info used to track line and
-// column numbers.
-while (! token_stack.empty ())
-{
-delete token_stack.top ();
-token_stack.pop ();
-}
-}
-// GAG.
-//
-// If we're reading a matrix and the next character is '[', make sure
-// that we insert a comma ahead of it.
-void
-lexical_feedback::do_comma_insert_check (void)
-{
-bool spc_gobbled = (eat_continuation () != lexical_feedback::NO_WHITESPACE);
-int c = text_yyinput ();
-xunput (c, yytext);
-if (spc_gobbled)
-xunput (' ', yytext);
-do_comma_insert = (! looking_at_object_index.front ()
-&& bracketflag && c == '[');
-}
-int
-lexical_feedback::text_yyinput (void)
-{
-int c = yyinput ();
-if (lexer_debug_flag)
-{
-std::cerr << "I: ";
-display_character (c);
-std::cerr << std::endl;
-}
-// Convert CRLF into just LF and single CR into LF.
-if (c == '\r')
-{
-c = yyinput ();
-if (lexer_debug_flag)
-{
-std::cerr << "I: ";
-display_character (c);
-std::cerr << std::endl;
-}
-if (c != '\n')
-{
-xunput (c, yytext);
-c = '\n';
-}
-}
-if (c == '\n')
-input_line_number++;
-return c;
-}
-void
-lexical_feedback::xunput (char c, char *buf)
-{
-if (lexer_debug_flag)
-{
-std::cerr << "U: ";
-display_character (c);
-std::cerr << std::endl;
-}
-if (c == '\n')
-input_line_number--;
-yyunput (c, buf);
-}
-// If we read some newlines, we need figure out what column we're
-// really looking at.
-void
-lexical_feedback::fixup_column_count (char *s)
-{
-char c;
-while ((c = *s++) != '\0')
-{
-if (c == '\n')
-{
-input_line_number++;
-current_input_column = 1;
-}
-else
-current_input_column++;
-}
-}
-bool
-lexical_feedback::inside_any_object_index (void)
-{
-bool retval = false;
-for (std::list<bool>::const_iterator i = looking_at_object_index.begin ();
-i != looking_at_object_index.end (); i++)
-{
-if (*i)
-{
-retval = true;
-break;
-}
-}
-return retval;
-}
-// Handle keywords.  Return -1 if the keyword should be ignored.
-int
-lexical_feedback::is_keyword_token (const std::string& s)
-{
-int l = input_line_number;
-int c = current_input_column;
-int len = s.length ();
-const octave_kw *kw = octave_kw_hash::in_word_set (s.c_str (), len);
-if (kw)
-{
-yylval.tok_val = 0;
-switch (kw->kw_id)
-{
-case break_kw:
-case catch_kw:
-case continue_kw:
-case else_kw:
-case otherwise_kw:
-case return_kw:
-case unwind_protect_cleanup_kw:
-at_beginning_of_statement = true;
-break;
-case static_kw:
-if ((reading_fcn_file || reading_script_file
-|| reading_classdef_file)
-&& ! curr_fcn_file_full_name.empty ())
-warning_with_id ("Octave:deprecated-keyword",
-"the 'static' keyword is obsolete and will be removed from a future version of Octave; please use 'persistent' instead; near line %d of file '%s'",
-input_line_number,
-curr_fcn_file_full_name.c_str ());
-else
-warning_with_id ("Octave:deprecated-keyword",
-"the 'static' keyword is obsolete and will be removed from a future version of Octave; please use 'persistent' instead; near line %d",
-input_line_number);
-// fall through ...
-case persistent_kw:
-break;
-case case_kw:
-case elseif_kw:
-case global_kw:
-case until_kw:
-break;
-case end_kw:
-if (inside_any_object_index ()
-|| (! reading_classdef_file
-&& (defining_func
-&& ! (looking_at_return_list
-|| parsed_function_name.top ()))))
-return 0;
-yylval.tok_val = new token (token::simple_end, l, c);
-at_beginning_of_statement = true;
-break;
-case end_try_catch_kw:
-yylval.tok_val = new token (token::try_catch_end, l, c);
-at_beginning_of_statement = true;
-break;
-case end_unwind_protect_kw:
-yylval.tok_val = new token (token::unwind_protect_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endfor_kw:
-yylval.tok_val = new token (token::for_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endfunction_kw:
-yylval.tok_val = new token (token::function_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endif_kw:
-yylval.tok_val = new token (token::if_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endparfor_kw:
-yylval.tok_val = new token (token::parfor_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endswitch_kw:
-yylval.tok_val = new token (token::switch_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endwhile_kw:
-yylval.tok_val = new token (token::while_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endclassdef_kw:
-yylval.tok_val = new token (token::classdef_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endenumeration_kw:
-yylval.tok_val = new token (token::enumeration_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endevents_kw:
-yylval.tok_val = new token (token::events_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endmethods_kw:
-yylval.tok_val = new token (token::methods_end, l, c);
-at_beginning_of_statement = true;
-break;
-case endproperties_kw:
-yylval.tok_val = new token (token::properties_end, l, c);
-at_beginning_of_statement = true;
-break;
-case for_kw:
-case parfor_kw:
-case while_kw:
-promptflag--;
-looping++;
-break;
-case do_kw:
-at_beginning_of_statement = true;
-promptflag--;
-looping++;
-break;
-case try_kw:
-case unwind_protect_kw:
-at_beginning_of_statement = true;
-promptflag--;
-break;
-case if_kw:
-case switch_kw:
-promptflag--;
-break;
-case get_kw:
-case set_kw:
-// 'get' and 'set' are keywords in classdef method
-// declarations.
-if (! maybe_classdef_get_set_method)
-return 0;
-break;
-case enumeration_kw:
-case events_kw:
-case methods_kw:
-case properties_kw:
-// 'properties', 'methods' and 'events' are keywords for
-// classdef blocks.
-if (! parsing_classdef)
-return 0;
-// fall through ...
-case classdef_kw:
-// 'classdef' is always a keyword.
-promptflag--;
-break;
-case function_kw:
-promptflag--;
-defining_func++;
-parsed_function_name.push (false);
-if (! (reading_fcn_file || reading_script_file
-|| reading_classdef_file))
-input_line_number = 1;
-break;
-case magic_file_kw:
-{
-if ((reading_fcn_file || reading_script_file
-|| reading_classdef_file)
-&& ! curr_fcn_file_full_name.empty ())
-yylval.tok_val = new token (curr_fcn_file_full_name, l, c);
-else
-yylval.tok_val = new token ("stdin", l, c);
-}
-break;
-case magic_line_kw:
-yylval.tok_val = new token (static_cast<double> (l), "", l, c);
-break;
-default:
-panic_impossible ();
-}
-if (! yylval.tok_val)
-yylval.tok_val = new token (l, c);
-token_stack.push (yylval.tok_val);
-return kw->tok;
-}
-return 0;
-}
-bool
-lexical_feedback::is_variable (const std::string& name)
-{
-return (symbol_table::is_variable (name)
-|| (pending_local_variables.find (name)
-!= pending_local_variables.end ()));
-}
-std::string
-lexical_feedback::grab_block_comment (stream_reader& reader, bool& eof)
-{
-std::string buf;
-bool at_bol = true;
-bool look_for_marker = false;
-bool warned_incompatible = false;
-int c = 0;
-while ((c = reader.getc ()) != EOF)
-{
-current_input_column++;
-if (look_for_marker)
-{
-at_bol = false;
-look_for_marker = false;
-if (c == '{' || c == '}')
-{
-std::string tmp_buf (1, static_cast<char> (c));
-int type = c;
-bool done = false;
-while ((c = reader.getc ()) != EOF && ! done)
-{
-current_input_column++;
-switch (c)
-{
-case ' ':
-case '\t':
-tmp_buf += static_cast<char> (c);
-break;
-case '\n':
-{
-current_input_column = 0;
-at_bol = true;
-done = true;
-if (type == '{')
-{
-block_comment_nesting_level++;
-promptflag--;
-}
-else
-{
-block_comment_nesting_level--;
-promptflag++;
-if (block_comment_nesting_level == 0)
-{
-buf += grab_comment_block (reader, true, eof);
-return buf;
-}
-}
-}
-break;
-default:
-at_bol = false;
-tmp_buf += static_cast<char> (c);
-buf += tmp_buf;
-done = true;
-break;
-}
-}
-}
-}
-if (at_bol && (c == '%' || c == '#'))
-{
-if (c == '#' && ! warned_incompatible)
-{
-warned_incompatible = true;
-maybe_gripe_matlab_incompatible_comment (c);
-}
-at_bol = false;
-look_for_marker = true;
-}
-else
-{
-buf += static_cast<char> (c);
-if (c == '\n')
-{
-current_input_column = 0;
-at_bol = true;
-}
-}
-}
-if (c == EOF)
-eof = true;
-return buf;
-}
-std::string
-lexical_feedback::grab_comment_block (stream_reader& reader, bool at_bol,
-bool& eof)
-{
-std::string buf;
-// TRUE means we are at the beginning of a comment block.
-bool begin_comment = false;
-// TRUE means we are currently reading a comment block.
-bool in_comment = false;
-bool warned_incompatible = false;
-int c = 0;
-while ((c = reader.getc ()) != EOF)
-{
-current_input_column++;
-if (begin_comment)
-{
-if (c == '%' || c == '#')
-{
-at_bol = false;
-continue;
-}
-else if (at_bol && c == '{')
-{
-std::string tmp_buf (1, static_cast<char> (c));
-bool done = false;
-while ((c = reader.getc ()) != EOF && ! done)
-{
-current_input_column++;
-switch (c)
-{
-case ' ':
-case '\t':
-tmp_buf += static_cast<char> (c);
-break;
-case '\n':
-{
-current_input_column = 0;
-at_bol = true;
-done = true;
-block_comment_nesting_level++;
-promptflag--;
-buf += grab_block_comment (reader, eof);
-in_comment = false;
-if (eof)
-goto done;
-}
-break;
-default:
-at_bol = false;
-tmp_buf += static_cast<char> (c);
-buf += tmp_buf;
-done = true;
-break;
-}
-}
-}
-else
-{
-at_bol = false;
-begin_comment = false;
-}
-}
-if (in_comment)
-{
-buf += static_cast<char> (c);
-if (c == '\n')
-{
-at_bol = true;
-current_input_column = 0;
-in_comment = false;
-// FIXME -- bailing out here prevents things like
-//
-//    octave> # comment
-//    octave> x = 1
-//
-// from failing at the command line, while still
-// allowing blocks of comments to be grabbed properly
-// for function doc strings.  But only the first line of
-// a mult-line doc string will be picked up for
-// functions defined on the command line.  We need a
-// better way of collecting these comments...
-if (! (reading_fcn_file || reading_script_file))
-goto done;
-}
-}
-else
-{
-switch (c)
-{
-case ' ':
-case '\t':
-break;
-case '#':
-if (! warned_incompatible)
-{
-warned_incompatible = true;
-maybe_gripe_matlab_incompatible_comment (c);
-}
-// fall through...
-case '%':
-in_comment = true;
-begin_comment = true;
-break;
-default:
-current_input_column--;
-reader.ungetc (c);
-goto done;
-}
-}
-}
-done:
-if (c == EOF)
-eof = true;
-return buf;
-}
-int
-lexical_feedback::process_comment (bool start_in_block, bool& eof)
-{
-eof = false;
-std::string help_txt;
-if (! help_buf.empty ())
-help_txt = help_buf.top ();
-flex_stream_reader flex_reader (this, yytext);
-// process_comment is only supposed to be called when we are not
-// initially looking at a block comment.
-std::string txt = start_in_block
-? grab_block_comment (flex_reader, eof)
-: grab_comment_block (flex_reader, false, eof);
-if (lexer_debug_flag)
-std::cerr << "C: " << txt << std::endl;
-if (help_txt.empty () && nesting_level.none ())
-{
-if (! help_buf.empty ())
-help_buf.pop ();
-help_buf.push (txt);
-}
-octave_comment_buffer::append (txt);
-current_input_column = 1;
-quote_is_transpose = false;
-convert_spaces_to_comma = true;
-at_beginning_of_statement = true;
-if (YY_START == COMMAND_START)
-BEGIN (INITIAL);
-if (nesting_level.none ())
-return '\n';
-else if (nesting_level.is_bracket_or_brace ())
-return ';';
-else
-return 0;
-}
-// Recognize separators.  If the separator is a CRLF pair, it is
-// replaced by a single LF.
-bool
-lexical_feedback::next_token_is_sep_op (void)
-{
-bool retval = false;
-int c = text_yyinput ();
-retval = match_any (c, ",;\n]");
-xunput (c, yytext);
-return retval;
-}
-// Try to determine if the next token should be treated as a postfix
-// unary operator.  This is ugly, but it seems to do the right thing.
-bool
-lexical_feedback::next_token_is_postfix_unary_op (bool spc_prev)
-{
-bool un_op = false;
-int c0 = text_yyinput ();
-if (c0 == '\'' && ! spc_prev)
-{
-un_op = true;
-}
-else if (c0 == '.')
-{
-int c1 = text_yyinput ();
-un_op = (c1 == '\'');
-xunput (c1, yytext);
-}
-else if (c0 == '+')
-{
-int c1 = text_yyinput ();
-un_op = (c1 == '+');
-xunput (c1, yytext);
-}
-else if (c0 == '-')
-{
-int c1 = text_yyinput ();
-un_op = (c1 == '-');
-xunput (c1, yytext);
-}
-xunput (c0, yytext);
-return un_op;
-}
-// Try to determine if the next token should be treated as a binary
-// operator.
-//
-// This kluge exists because whitespace is not always ignored inside
-// the square brackets that are used to create matrix objects (though
-// spacing only really matters in the cases that can be interpreted
-// either as binary ops or prefix unary ops: currently just +, -).
-//
-// Note that a line continuation directly following a + or - operator
-// (e.g., the characters '[' 'a' ' ' '+' '\' LFD 'b' ']') will be
-// parsed as a binary operator.
-bool
-lexical_feedback::next_token_is_bin_op (bool spc_prev)
-{
-bool bin_op = false;
-int c0 = text_yyinput ();
-switch (c0)
-{
-case '+':
-case '-':
-{
-int c1 = text_yyinput ();
-switch (c1)
-{
-case '+':
-case '-':
-// Unary ops, spacing doesn't matter.
-break;
-case '=':
-// Binary ops, spacing doesn't matter.
-bin_op = true;
-break;
-default:
-// Could be either, spacing matters.
-bin_op = looks_like_bin_op (spc_prev, c1);
-break;
-}
-xunput (c1, yytext);
-}
-break;
-case ':':
-case '/':
-case '\\':
-case '^':
-// Always a binary op (may also include /=, \=, and ^=).
-bin_op = true;
-break;
-// .+ .- ./ .\ .^ .* .**
-case '.':
-{
-int c1 = text_yyinput ();
-if (match_any (c1, "+-/\\^*"))
-// Always a binary op (may also include .+=, .-=, ./=, ...).
-bin_op = true;
-else if (! isdigit (c1) && c1 != ' ' && c1 != '\t' && c1 != '.')
-// A structure element reference is a binary op.
-bin_op = true;
-xunput (c1, yytext);
-}
-break;
-// = == & && | || * **
-case '=':
-case '&':
-case '|':
-case '*':
-// Always a binary op (may also include ==, &&, ||, **).
-bin_op = true;
-break;
-// < <= <> > >=
-case '<':
-case '>':
-// Always a binary op (may also include <=, <>, >=).
-bin_op = true;
-break;
-// ~= !=
-case '~':
-case '!':
-{
-int c1 = text_yyinput ();
-// ~ and ! can be unary ops, so require following =.
-if (c1 == '=')
-bin_op = true;
-xunput (c1, yytext);
-}
-break;
-default:
-break;
-}
-xunput (c0, yytext);
-return bin_op;
-}
-// FIXME -- we need to handle block comments here.
-void
-lexical_feedback::scan_for_comments (const char *text)
-{
-std::string comment_buf;
-bool in_comment = false;
-bool beginning_of_comment = false;
-int len = strlen (text);
-int i = 0;
-while (i < len)
-{
-char c = text[i++];
-switch (c)
-{
-case '%':
-case '#':
-if (in_comment)
-{
-if (! beginning_of_comment)
-comment_buf += static_cast<char> (c);
-}
-else
-{
-maybe_gripe_matlab_incompatible_comment (c);
-in_comment = true;
-beginning_of_comment = true;
-}
-break;
-case '\n':
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-octave_comment_buffer::append (comment_buf);
-comment_buf.resize (0);
-in_comment = false;
-beginning_of_comment = false;
-}
-break;
-default:
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-beginning_of_comment = false;
-}
-break;
-}
-}
-if (! comment_buf.empty ())
-octave_comment_buffer::append (comment_buf);
-}
-// Discard whitespace, including comments and continuations.
-// FIXME -- we need to handle block comments here.
-int
-lexical_feedback::eat_whitespace (void)
-{
-int retval = lexical_feedback::NO_WHITESPACE;
-std::string comment_buf;
-bool in_comment = false;
-bool beginning_of_comment = false;
-int c = 0;
-while ((c = text_yyinput ()) != EOF)
-{
-current_input_column++;
-switch (c)
-{
-case ' ':
-case '\t':
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-beginning_of_comment = false;
-}
-retval |= lexical_feedback::SPACE_OR_TAB;
-break;
-case '\n':
-retval |= lexical_feedback::NEWLINE;
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-octave_comment_buffer::append (comment_buf);
-comment_buf.resize (0);
-in_comment = false;
-beginning_of_comment = false;
-}
-current_input_column = 0;
-break;
-case '#':
-case '%':
-if (in_comment)
-{
-if (! beginning_of_comment)
-comment_buf += static_cast<char> (c);
-}
-else
-{
-maybe_gripe_matlab_incompatible_comment (c);
-in_comment = true;
-beginning_of_comment = true;
-}
-break;
-case '.':
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-beginning_of_comment = false;
-break;
-}
-else
-{
-if (have_ellipsis_continuation ())
-break;
-else
-goto done;
-}
-case '\\':
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-beginning_of_comment = false;
-break;
-}
-else
-{
-if (have_continuation ())
-break;
-else
-goto done;
-}
-default:
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-beginning_of_comment = false;
-break;
-}
-else
-goto done;
-}
-}
-if (! comment_buf.empty ())
-octave_comment_buffer::append (comment_buf);
-done:
-xunput (c, yytext);
-current_input_column--;
-return retval;
-}
-static inline bool
-looks_like_hex (const char *s, int len)
-{
-return (len > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X'));
-}
-void
-lexical_feedback::handle_number (void)
-{
-double value = 0.0;
-int nread = 0;
-if (looks_like_hex (yytext, strlen (yytext)))
-{
-unsigned long ival;
-nread = sscanf (yytext, "%lx", &ival);
-value = static_cast<double> (ival);
-}
-else
-{
-char *tmp = strsave (yytext);
-char *idx = strpbrk (tmp, "Dd");
-if (idx)
-*idx = 'e';
-nread = sscanf (tmp, "%lf", &value);
-delete [] tmp;
-}
-// If yytext doesn't contain a valid number, we are in deep doo doo.
-assert (nread == 1);
-quote_is_transpose = true;
-convert_spaces_to_comma = true;
-looking_for_object_index = false;
-at_beginning_of_statement = false;
-yylval.tok_val = new token (value, yytext, input_line_number,
-current_input_column);
-token_stack.push (yylval.tok_val);
-current_input_column += yyleng;
-do_comma_insert_check ();
-}
-// We have seen a backslash and need to find out if it should be
-// treated as a continuation character.  If so, this eats it, up to
-// and including the new line character.
-//
-// Match whitespace only, followed by a comment character or newline.
-// Once a comment character is found, discard all input until newline.
-// If non-whitespace characters are found before comment
-// characters, return 0.  Otherwise, return 1.
-// FIXME -- we need to handle block comments here.
-bool
-lexical_feedback::have_continuation (bool trailing_comments_ok)
-{
-std::ostringstream buf;
-std::string comment_buf;
-bool in_comment = false;
-bool beginning_of_comment = false;
-int c = 0;
-while ((c = text_yyinput ()) != EOF)
-{
-buf << static_cast<char> (c);
-switch (c)
-{
-case ' ':
-case '\t':
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-beginning_of_comment = false;
-}
-break;
-case '%':
-case '#':
-if (trailing_comments_ok)
-{
-if (in_comment)
-{
-if (! beginning_of_comment)
-comment_buf += static_cast<char> (c);
-}
-else
-{
-maybe_gripe_matlab_incompatible_comment (c);
-in_comment = true;
-beginning_of_comment = true;
-}
-}
-else
-goto cleanup;
-break;
-case '\n':
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-octave_comment_buffer::append (comment_buf);
-}
-current_input_column = 0;
-promptflag--;
-gripe_matlab_incompatible_continuation ();
-return true;
-default:
-if (in_comment)
-{
-comment_buf += static_cast<char> (c);
-beginning_of_comment = false;
-}
-else
-goto cleanup;
-break;
-}
-}
-xunput (c, yytext);
-return false;
-cleanup:
-std::string s = buf.str ();
-int len = s.length ();
-while (len--)
-xunput (s[len], yytext);
-return false;
-}
-// We have seen a '.' and need to see if it is the start of a
-// continuation.  If so, this eats it, up to and including the new
-// line character.
-bool
-lexical_feedback::have_ellipsis_continuation (bool trailing_comments_ok)
-{
-char c1 = text_yyinput ();
-if (c1 == '.')
-{
-char c2 = text_yyinput ();
-if (c2 == '.' && have_continuation (trailing_comments_ok))
-return true;
-else
-{
-xunput (c2, yytext);
-xunput (c1, yytext);
-}
-}
-else
-xunput (c1, yytext);
-return false;
-}
-// See if we have a continuation line.  If so, eat it and the leading
-// whitespace on the next line.
-int
-lexical_feedback::eat_continuation (void)
-{
-int retval = lexical_feedback::NO_WHITESPACE;
-int c = text_yyinput ();
-if ((c == '.' && have_ellipsis_continuation ())
-|| (c == '\\' && have_continuation ()))
-retval = eat_whitespace ();
-else
-xunput (c, yytext);
-return retval;
-}
-int
-lexical_feedback::handle_string (char delim)
-{
-std::ostringstream buf;
-int bos_line = input_line_number;
-int bos_col = current_input_column;
-int c;
-int escape_pending = 0;
-while ((c = text_yyinput ()) != EOF)
-{
-current_input_column++;
-if (c == '\\')
-{
-if (delim == '\'' || escape_pending)
-{
-buf << static_cast<char> (c);
-escape_pending = 0;
-}
-else
-{
-if (have_continuation (false))
-escape_pending = 0;
-else
-{
-buf << static_cast<char> (c);
-escape_pending = 1;
-}
-}
-continue;
-}
-else if (c == '.')
-{
-if (delim == '\'' || ! have_ellipsis_continuation (false))
-buf << static_cast<char> (c);
-}
-else if (c == '\n')
-{
-error ("unterminated string constant");
-break;
-}
-else if (c == delim)
-{
-if (escape_pending)
-buf << static_cast<char> (c);
-else
-{
-c = text_yyinput ();
-if (c == delim)
-{
-buf << static_cast<char> (c);
-}
-else
-{
-std::string s;
-xunput (c, yytext);
-if (delim == '\'')
-s = buf.str ();
-else
-s = do_string_escapes (buf.str ());
-quote_is_transpose = true;
-convert_spaces_to_comma = true;
-yylval.tok_val = new token (s, bos_line, bos_col);
-token_stack.push (yylval.tok_val);
-if (delim == '"')
-gripe_matlab_incompatible ("\" used as string delimiter");
-else if (delim == '\'')
-gripe_single_quote_string ();
-looking_for_object_index = true;
-at_beginning_of_statement = false;
-return delim == '"' ? DQ_STRING : SQ_STRING;
-}
-}
-}
-else
-{
-buf << static_cast<char> (c);
-}
-escape_pending = 0;
-}
-return LEXICAL_ERROR;
-}
-bool
-lexical_feedback::next_token_is_assign_op (void)
-{
-bool retval = false;
-int c0 = text_yyinput ();
-switch (c0)
-{
-case '=':
-{
-int c1 = text_yyinput ();
-xunput (c1, yytext);
-if (c1 != '=')
-retval = true;
-}
-break;
-case '+':
-case '-':
-case '*':
-case '/':
-case '\\':
-case '&':
-case '|':
-{
-int c1 = text_yyinput ();
-xunput (c1, yytext);
-if (c1 == '=')
-retval = true;
-}
-break;
-case '.':
-{
-int c1 = text_yyinput ();
-if (match_any (c1, "+-*/\\"))
-{
-int c2 = text_yyinput ();
-xunput (c2, yytext);
-if (c2 == '=')
-retval = true;
-}
-xunput (c1, yytext);
-}
-break;
-case '>':
-{
-int c1 = text_yyinput ();
-if (c1 == '>')
-{
-int c2 = text_yyinput ();
-xunput (c2, yytext);
-if (c2 == '=')
-retval = true;
-}
-xunput (c1, yytext);
-}
-break;
-case '<':
-{
-int c1 = text_yyinput ();
-if (c1 == '<')
-{
-int c2 = text_yyinput ();
-xunput (c2, yytext);
-if (c2 == '=')
-retval = true;
-}
-xunput (c1, yytext);
-}
-break;
-default:
-break;
-}
-xunput (c0, yytext);
-return retval;
-}
-bool
-lexical_feedback::next_token_is_index_op (void)
-{
-int c = text_yyinput ();
-xunput (c, yytext);
-return c == '(' || c == '{';
-}
-int
-lexical_feedback::handle_close_bracket (bool spc_gobbled, int bracket_type)
-{
-int retval = bracket_type;
-if (! nesting_level.none ())
-{
-nesting_level.remove ();
-if (bracket_type == ']')
-bracketflag--;
-else if (bracket_type == '}')
-braceflag--;
-else
-panic_impossible ();
-}
-if (bracketflag == 0 && braceflag == 0)
-BEGIN (INITIAL);
-if (bracket_type == ']'
-&& next_token_is_assign_op ()
-&& ! looking_at_return_list)
-{
-retval = CLOSE_BRACE;
-}
-else if ((bracketflag || braceflag)
-&& convert_spaces_to_comma
-&& (nesting_level.is_bracket ()
-|| (nesting_level.is_brace ()
-&& ! looking_at_object_index.front ())))
-{
-bool index_op = next_token_is_index_op ();
-// Don't insert comma if we are looking at something like
-//
-//   [x{i}{j}] or [x{i}(j)]
-//
-// but do if we are looking at
-//
-//   [x{i} {j}] or [x{i} (j)]
-if (spc_gobbled || ! (bracket_type == '}' && index_op))
-{
-bool bin_op = next_token_is_bin_op (spc_gobbled);
-bool postfix_un_op = next_token_is_postfix_unary_op (spc_gobbled);
-bool sep_op = next_token_is_sep_op ();
-if (! (postfix_un_op || bin_op || sep_op))
-{
-maybe_warn_separator_insert (',');
-xunput (',', yytext);
-return retval;
-}
-}
-}
-quote_is_transpose = true;
-convert_spaces_to_comma = true;
-return retval;
-}
-void
-lexical_feedback::maybe_unput_comma (int spc_gobbled)
-{
-if (nesting_level.is_bracket ()
-|| (nesting_level.is_brace ()
-&& ! looking_at_object_index.front ()))
-{
-int bin_op = next_token_is_bin_op (spc_gobbled);
-int postfix_un_op = next_token_is_postfix_unary_op (spc_gobbled);
-int c1 = text_yyinput ();
-int c2 = text_yyinput ();
-xunput (c2, yytext);
-xunput (c1, yytext);
-int sep_op = next_token_is_sep_op ();
-int dot_op = (c1 == '.'
-&& (isalpha (c2) || isspace (c2) || c2 == '_'));
-if (postfix_un_op || bin_op || sep_op || dot_op)
-return;
-int index_op = (c1 == '(' || c1 == '{');
-// If there is no space before the indexing op, we don't insert
-// a comma.
-if (index_op && ! spc_gobbled)
-return;
-maybe_warn_separator_insert (',');
-xunput (',', yytext);
-}
-}
-bool
-lexical_feedback::next_token_can_follow_bin_op (void)
-{
-std::stack<char> buf;
-int c = EOF;
-// Skip whitespace in current statement on current line
-while (true)
-{
-c = text_yyinput ();
-buf.push (c);
-if (match_any (c, ",;\n") || (c != ' ' && c != '\t'))
-break;
-}
-// Restore input.
-while (! buf.empty ())
-{
-xunput (buf.top (), yytext);
-buf.pop ();
-}
-return (isalnum (c) || match_any (c, "!\"'(-[_{~"));
-}
-static bool
-can_be_command (const std::string& tok)
-{
-// Don't allow these names to be treated as commands to avoid
-// surprises when parsing things like "NaN ^2".
-return ! (tok == "e"
-|| tok == "I" || tok == "i"
-|| tok == "J" || tok == "j"
-|| tok == "Inf" || tok == "inf"
-|| tok == "NaN" || tok == "nan");
-}
-bool
-lexical_feedback::looks_like_command_arg (void)
-{
-bool retval = true;
-int c0 = text_yyinput ();
-switch (c0)
-{
-// = ==
-case '=':
-{
-int c1 = text_yyinput ();
-if (c1 == '=')
-{
-int c2 = text_yyinput ();
-if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c2, yytext);
-}
-else
-retval = false;
-xunput (c1, yytext);
-}
-break;
-case '(':
-case '{':
-// Indexing.
-retval = false;
-break;
-case '\n':
-// EOL.
-break;
-case '\'':
-case '"':
-// Beginning of a character string.
-break;
-// + - ++ -- += -=
-case '+':
-case '-':
-{
-int c1 = text_yyinput ();
-switch (c1)
-{
-case '\n':
-// EOL.
-case '+':
-case '-':
-// Unary ops, spacing doesn't matter.
-break;
-case '\t':
-case ' ':
-{
-if (next_token_can_follow_bin_op ())
-retval = false;
-}
-break;
-case '=':
-{
-int c2 = text_yyinput ();
-if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c2, yytext);
-}
-break;
-}
-xunput (c1, yytext);
-}
-break;
-case ':':
-case '/':
-case '\\':
-case '^':
-{
-int c1 = text_yyinput ();
-if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c1, yytext);
-}
-break;
-// .+ .- ./ .\ .^ .* .**
-case '.':
-{
-int c1 = text_yyinput ();
-if (match_any (c1, "+-/\\^*"))
-{
-int c2 = text_yyinput ();
-if (c2 == '=')
-{
-int c3 = text_yyinput ();
-if (! match_any (c3, ",;\n") && (c3 == ' ' || c3 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c3, yytext);
-}
-else if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c2, yytext);
-}
-else if (! match_any (c1, ",;\n")
-&& (! isdigit (c1) && c1 != ' ' && c1 != '\t'
-&& c1 != '.'))
-{
-// Structure reference.  FIXME -- is this a complete check?
-retval = false;
-}
-xunput (c1, yytext);
-}
-break;
-// & && | || * **
-case '&':
-case '|':
-case '*':
-{
-int c1 = text_yyinput ();
-if (c1 == c0)
-{
-int c2 = text_yyinput ();
-if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c2, yytext);
-}
-else if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c1, yytext);
-}
-break;
-// < <= > >=
-case '<':
-case '>':
-{
-int c1 = text_yyinput ();
-if (c1 == '=')
-{
-int c2 = text_yyinput ();
-if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c2, yytext);
-}
-else if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c1, yytext);
-}
-break;
-// ~= !=
-case '~':
-case '!':
-{
-int c1 = text_yyinput ();
-// ~ and ! can be unary ops, so require following =.
-if (c1 == '=')
-{
-int c2 = text_yyinput ();
-if (! match_any (c2, ",;\n") && (c2 == ' ' || c2 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c2, yytext);
-}
-else if (! match_any (c1, ",;\n") && (c1 == ' ' || c1 == '\t')
-&& next_token_can_follow_bin_op ())
-retval = false;
-xunput (c1, yytext);
-}
-break;
-default:
-break;
-}
-xunput (c0, yytext);
-return retval;
-}
-int
-lexical_feedback::handle_superclass_identifier (void)
-{
-eat_continuation ();
-std::string pkg;
-std::string meth = strip_trailing_whitespace (yytext);
-size_t pos = meth.find ("@");
-std::string cls = meth.substr (pos).substr (1);
-meth = meth.substr (0, pos - 1);
-pos = cls.find (".");
-if (pos != std::string::npos)
-{
-pkg = cls.substr (pos).substr (1);
-cls = cls.substr (0, pos - 1);
-}
-int kw_token = (is_keyword_token (meth) || is_keyword_token (cls)
-|| is_keyword_token (pkg));
-if (kw_token)
-{
-error ("method, class and package names may not be keywords");
-return LEXICAL_ERROR;
-}
-yylval.tok_val
-= new token (meth.empty () ? 0 : &(symbol_table::insert (meth)),
-cls.empty () ? 0 : &(symbol_table::insert (cls)),
-pkg.empty () ? 0 : &(symbol_table::insert (pkg)),
-input_line_number,
-current_input_column);
-token_stack.push (yylval.tok_val);
-convert_spaces_to_comma = true;
-current_input_column += yyleng;
-return SUPERCLASSREF;
-}
-int
-lexical_feedback::handle_meta_identifier (void)
-{
-eat_continuation ();
-std::string pkg;
-std::string cls = strip_trailing_whitespace (yytext).substr (1);
-size_t pos = cls.find (".");
-if (pos != std::string::npos)
-{
-pkg = cls.substr (pos).substr (1);
-cls = cls.substr (0, pos - 1);
-}
-int kw_token = is_keyword_token (cls) || is_keyword_token (pkg);
-if (kw_token)
-{
-error ("class and package names may not be keywords");
-return LEXICAL_ERROR;
-}
-yylval.tok_val
-= new token (cls.empty () ? 0 : &(symbol_table::insert (cls)),
-pkg.empty () ? 0 : &(symbol_table::insert (pkg)),
-input_line_number,
-current_input_column);
-token_stack.push (yylval.tok_val);
-convert_spaces_to_comma = true;
-current_input_column += yyleng;
-return METAQUERY;
-}
-// Figure out exactly what kind of token to return when we have seen
-// an identifier.  Handles keywords.  Return -1 if the identifier
-// should be ignored.
-int
-lexical_feedback::handle_identifier (void)
-{
-bool at_bos = at_beginning_of_statement;
-std::string tok = strip_trailing_whitespace (yytext);
-int c = yytext[yyleng-1];
-bool cont_is_spc = (eat_continuation () != lexical_feedback::NO_WHITESPACE);
-int spc_gobbled = (cont_is_spc || c == ' ' || c == '\t');
-// If we are expecting a structure element, avoid recognizing
-// keywords and other special names and return STRUCT_ELT, which is
-// a string that is also a valid identifier.  But first, we have to
-// decide whether to insert a comma.
-if (looking_at_indirect_ref)
-{
-do_comma_insert_check ();
-maybe_unput_comma (spc_gobbled);
-yylval.tok_val = new token (tok, input_line_number,
-current_input_column);
-token_stack.push (yylval.tok_val);
-quote_is_transpose = true;
-convert_spaces_to_comma = true;
-looking_for_object_index = true;
-current_input_column += yyleng;
-return STRUCT_ELT;
-}
-at_beginning_of_statement = false;
-// The is_keyword_token may reset
-// at_beginning_of_statement.  For example, if it sees
-// an else token, then the next token is at the beginning of a
-// statement.
-int kw_token = is_keyword_token (tok);
-// If we found a keyword token, then the beginning_of_statement flag
-// is already set.  Otherwise, we won't be at the beginning of a
-// statement.
-if (looking_at_function_handle)
-{
-if (kw_token)
-{
-error ("function handles may not refer to keywords");
-return LEXICAL_ERROR;
-}
-else
-{
-yylval.tok_val = new token (tok, input_line_number,
-current_input_column);
-token_stack.push (yylval.tok_val);
-current_input_column += yyleng;
-quote_is_transpose = false;
-convert_spaces_to_comma = true;
-looking_for_object_index = true;
-return FCN_HANDLE;
-}
-}
-// If we have a regular keyword, return it.
-// Keywords can be followed by identifiers.
-if (kw_token)
-{
-if (kw_token >= 0)
-{
-current_input_column += yyleng;
-quote_is_transpose = false;
-convert_spaces_to_comma = true;
-looking_for_object_index = false;
-}
-return kw_token;
-}
-// See if we have a plot keyword (title, using, with, or clear).
-int c1 = text_yyinput ();
-bool next_tok_is_eq = false;
-if (c1 == '=')
-{
-int c2 = text_yyinput ();
-xunput (c2, yytext);
-if (c2 != '=')
-next_tok_is_eq = true;
-}
-xunput (c1, yytext);
-// Kluge alert.
-//
-// If we are looking at a text style function, set up to gobble its
-// arguments.
-//
-// If the following token is '=', or if we are parsing a function
-// return list or function parameter list, or if we are looking at
-// something like [ab,cd] = foo (), force the symbol to be inserted
-// as a variable in the current symbol table.
-if (! is_variable (tok))
-{
-if (at_bos && spc_gobbled && can_be_command (tok)
-&& looks_like_command_arg ())
-{
-BEGIN (COMMAND_START);
-}
-else if (next_tok_is_eq
-|| looking_at_decl_list
-|| looking_at_return_list
-|| (looking_at_parameter_list
-&& ! looking_at_initializer_expression))
-{
-symbol_table::force_variable (tok);
-}
-else if (looking_at_matrix_or_assign_lhs)
-{
-pending_local_variables.insert (tok);
-}
-}
-// Find the token in the symbol table.  Beware the magic
-// transformation of the end keyword...
-if (tok == "end")
-tok = "__end__";
-yylval.tok_val = new token (&(symbol_table::insert (tok)),
-input_line_number,
-current_input_column);
-token_stack.push (yylval.tok_val);
-// After seeing an identifer, it is ok to convert spaces to a comma
-// (if needed).
-convert_spaces_to_comma = true;
-if (! (next_tok_is_eq || YY_START == COMMAND_START))
-{
-quote_is_transpose = true;
-do_comma_insert_check ();
-maybe_unput_comma (spc_gobbled);
-}
-current_input_column += yyleng;
-if (tok != "__end__")
-looking_for_object_index = true;
-return NAME;
-}
-void
-lexical_feedback::maybe_warn_separator_insert (char sep)
-{
-std::string nm = curr_fcn_file_full_name;
-if (nm.empty ())
-warning_with_id ("Octave:separator-insert",
-"potential auto-insertion of '%c' near line %d",
-sep, input_line_number);
-else
-warning_with_id ("Octave:separator-insert",
-"potential auto-insertion of '%c' near line %d of file %s",
-sep, input_line_number, nm.c_str ());
-}
-void
-lexical_feedback::gripe_single_quote_string (void)
-{
-std::string nm = curr_fcn_file_full_name;
-if (nm.empty ())
-warning_with_id ("Octave:single-quote-string",
-"single quote delimited string near line %d",
-input_line_number);
-else
-warning_with_id ("Octave:single-quote-string",
-"single quote delimited string near line %d of file %s",
-input_line_number, nm.c_str ());
-}
-void
-lexical_feedback::gripe_matlab_incompatible (const std::string& msg)
-{
-std::string nm = curr_fcn_file_full_name;
-if (nm.empty ())
-warning_with_id ("Octave:matlab-incompatible",
-"potential Matlab compatibility problem: %s",
-msg.c_str ());
-else
-warning_with_id ("Octave:matlab-incompatible",
-"potential Matlab compatibility problem: %s near line %d offile %s",
-msg.c_str (), input_line_number, nm.c_str ());
-}
-void
-lexical_feedback::maybe_gripe_matlab_incompatible_comment (char c)
-{
-if (c == '#')
-gripe_matlab_incompatible ("# used as comment character");
-}
-void
-lexical_feedback::gripe_matlab_incompatible_continuation (void)
-{
-gripe_matlab_incompatible ("\\ used as line continuation marker");
-}
-void
-lexical_feedback::gripe_matlab_incompatible_operator (const std::string& op)
-{
-std::string t = op;
-int n = t.length ();
-if (t[n-1] == '\n')
-t.resize (n-1);
-gripe_matlab_incompatible (t + " used as operator");
-}

Mercurial > octave

comparison libinterp/parse-tree/lex.ll @ 16119:b31eb56f4d84