diff libinterp/parse-tree/lex.ll @ 16253:a89cf57ba3a5

partial cleanup of continuation handling in lexer * lex.h, lex.ll (octave_lexer::handle_continuation): New function. (octave_lexer::finish_comment): Move to octave_lexer::finish_comment. Don't return token. New argument, looking_at_continuation. If not handling continuation, unput newline character. (^{S}*{CCHAR}\{{S}*{NL}): Call yyless before finishing comment. (^{S}*{CCHAR}\{{S}*{NL}, <BLOCK_COMMENT_START>^{S}*{CCHAR}\{{S}*{NL}, <LINE_COMMENT_START>{ANY_INCLUDING_NL}, <LINE_COMMENT_START>{S}*{CCHAR}.*{NL}): Don't return token. (<LINE_COMMENT_START>{S}*{CCHAR}.*{NL}): Don't give COMMAND_START start state special treatment. ({CONT}{S}*{NL}|{CONT}{S}*{COMMENT}): Call octave_lexer::handle_continuation.
author John W. Eaton <jwe@octave.org>
date Sat, 09 Mar 2013 21:44:14 -0500
parents 2b15ae55c721
children b8a2df776118 12bf6a3f8c45
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.ll	Sun Mar 10 00:37:06 2013 -0500
+++ b/libinterp/parse-tree/lex.ll	Sat Mar 09 21:44:14 2013 -0500
@@ -485,22 +485,18 @@
 ^{S}*{CCHAR}\{{S}*{NL} {
     curr_lexer->lexer_debug ("^{S}*{CCHAR}\{{S}*{NL}");
 
-    int tok = 0;
+    yyless (0);
 
     if (curr_lexer->start_state () == LINE_COMMENT_START)
       {
         if (! curr_lexer->comment_text.empty ())
-          tok = curr_lexer->finish_comment (octave_comment_elt::full_line);
+          curr_lexer->finish_comment (octave_comment_elt::full_line);
 
         curr_lexer->pop_start_state ();
       }
 
     curr_lexer->push_start_state (BLOCK_COMMENT_START);
 
-    yyless (0);
-
-    if (tok > 0)
-      return curr_lexer->count_token (tok);
   }
 
 <BLOCK_COMMENT_START>^{S}*{CCHAR}\{{S}*{NL} {
@@ -527,18 +523,13 @@
     curr_lexer->input_line_number++;
     curr_lexer->current_input_column = 1;
 
-    int tok = 0;
-
     if (curr_lexer->block_comment_nesting_level > 1)
       curr_lexer->comment_text = "\n";
     else
-      tok = curr_lexer->finish_comment (octave_comment_elt::block);
+      curr_lexer->finish_comment (octave_comment_elt::block);
 
     curr_lexer->block_comment_nesting_level--;
     curr_lexer->pop_start_state ();
-
-    if (tok > 0)
-      return curr_lexer->count_token (tok);
   }
 
 %{
@@ -584,26 +575,12 @@
       
     curr_lexer->comment_text += &yytext[i];
 
-    int tok = 0;
-
     if (! full_line_comment)
       {
-        tok = curr_lexer->finish_comment (octave_comment_elt::end_of_line);
+        curr_lexer->finish_comment (octave_comment_elt::end_of_line);
 
         curr_lexer->pop_start_state ();
-
-        if (curr_lexer->start_state () == COMMAND_START)
-          {
-            // Allow the actions for the end of a COMMAND line to be
-            // executed next.
-
-            tok = 0;
-            curr_lexer->xunput ('\n');
-          }
       }
-
-    if (tok > 0)
-      return curr_lexer->count_token (tok);
   }
 
 %{
@@ -615,12 +592,9 @@
 
     curr_lexer->xunput (yytext[0]);
 
-    int tok = curr_lexer->finish_comment (octave_comment_elt::full_line);  
+    curr_lexer->finish_comment (octave_comment_elt::full_line);  
 
     curr_lexer->pop_start_state ();
-
-    if (tok > 0)
-      return curr_lexer->count_token (tok);
   }
 
 %{
@@ -660,15 +634,10 @@
 %}
 
 {CONT}{S}*{NL} |
-{CONT}{S}*{COMMENT} {
-    curr_lexer->lexer_debug ("{CONT}{S}*{NL}|{CONT}{S}*{COMMENT}");
-
-    if (yytext[0] == '\\')
-      curr_lexer->gripe_matlab_incompatible_continuation ();
-    curr_lexer->scan_for_comments (yytext);
-    curr_lexer->decrement_promptflag ();
-    curr_lexer->input_line_number++;
-    curr_lexer->current_input_column = 1;
+{CONT}{S}*{CCHAR}.*{NL} {
+    curr_lexer->lexer_debug ("{CONT}{S}*{NL}|{CONT}{S}*{CCHAR}.*{NL}");
+
+    curr_lexer->handle_continuation ();
   }
 
 %{
@@ -1415,36 +1384,6 @@
   return retval;
 }
 
-int
-lexical_feedback::finish_comment (octave_comment_elt::comment_type typ)
-{
-  bool copyright = looks_like_copyright (comment_text);
-
-  if (nesting_level.none () && help_text.empty ()
-    && ! comment_text.empty () && ! copyright)
-    help_text = comment_text;
-
-  if (copyright)
-    typ = octave_comment_elt::copyright;
-
-  octave_comment_buffer::append (comment_text, typ);
-
-  comment_text = "";
-
-  quote_is_transpose = false;
-  convert_spaces_to_comma = true;
-  at_beginning_of_statement = true;
-
-  if (nesting_level.none ())
-    return '\n';
-  else if (nesting_level.is_bracket_or_brace ())
-    // FIXME -- this result will be different if the comment follows a
-    // continuation token.
-    return ';';
-  else
-    return 0;
-}
-
 void
 octave_lexer::input_buffer::fill (const std::string& input, bool eof_arg)
 {
@@ -2354,6 +2293,81 @@
   do_comma_insert_check ();
 }
 
+void
+octave_lexer::handle_continuation (void)
+{
+  char *yytxt = flex_yytext ();
+  int yylng = flex_yyleng ();
+
+  size_t offset = 1;
+  if (yytxt[0] == '\\')
+    gripe_matlab_incompatible_continuation ();
+  else
+    offset = 3;
+
+  bool have_space = false;
+  while (offset < yylng)
+    {
+      char c = yytxt[offset];
+      if (c == ' ' || c == '\t')
+        {
+          have_space = true;
+          offset++;
+        }
+      else
+        break;
+    }
+
+  bool have_comment = false;
+  while (offset < yylng)
+    {
+      char c = yytxt[offset];
+      if (c == '#' || c == '%')
+        {
+          have_comment = true;
+          offset++;
+        }
+      else
+        break;
+    }
+
+  if (have_comment)
+    {
+      comment_text = &yytxt[offset];
+
+      finish_comment (octave_comment_elt::end_of_line, true);
+    }
+
+  decrement_promptflag ();
+  input_line_number++;
+  current_input_column = 1;
+}
+
+void
+octave_lexer::finish_comment (octave_comment_elt::comment_type typ,
+                              bool looking_at_continuation)
+{
+  bool copyright = looks_like_copyright (comment_text);
+
+  if (nesting_level.none () && help_text.empty ()
+    && ! comment_text.empty () && ! copyright)
+    help_text = comment_text;
+
+  if (copyright)
+    typ = octave_comment_elt::copyright;
+
+  octave_comment_buffer::append (comment_text, typ);
+
+  comment_text = "";
+
+  quote_is_transpose = false;
+  convert_spaces_to_comma = true;
+  at_beginning_of_statement = true;
+
+  if (! looking_at_continuation)
+    xunput ('\n');
+}
+
 // We have seen a backslash and need to find out if it should be
 // treated as a continuation character.  If so, this eats it, up to
 // and including the new line character.