changeset 28265:6b6cb174d6bb

maint: merge stable to default.
author John W. Eaton <jwe@octave.org>
date Tue, 05 May 2020 01:10:39 -0400
parents d63e05fdf11b (current diff) d938c4d22200 (diff)
children 4ee43852f5b6
files
diffstat 2 files changed, 52 insertions(+), 15 deletions(-) [+]
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h	Mon May 04 11:51:22 2020 -0400
+++ b/libinterp/parse-tree/lex.h	Tue May 05 01:10:39 2020 -0400
@@ -697,6 +697,8 @@
 
     token * current_token (void);
 
+    size_t pending_token_count (void) const;
+
     void display_token (int tok);
 
     void fatal_error (const char *msg);
--- a/libinterp/parse-tree/lex.ll	Mon May 04 11:51:22 2020 -0400
+++ b/libinterp/parse-tree/lex.ll	Tue May 05 01:10:39 2020 -0400
@@ -820,21 +820,50 @@
 <LINE_COMMENT_START>{ANY_INCLUDING_NL} {
     curr_lexer->lexer_debug ("<LINE_COMMENT_START>{ANY_INCLUDING_NL}");
 
-    curr_lexer->finish_comment (octave::comment_elt::full_line);
-
-    curr_lexer->pop_start_state ();
-
-    // If yytext contains the special ASCII 1 marker inserted by
-    // push_lexer::fill_flex_buffer and we are at the end of the buffer,
-    // return -1 to indicate that we are expecting more input.
-
-    HANDLE_EOB_OR_EOF (yytext[0] == '\001' ? -1 : -2);
-
-    // Restore all characters except the ASCII 1 marker that was
-    // inserted by push_lexer::fill_flex_buffer.
-
-    if (yytext[0] != '\001')
-      curr_lexer->xunput (yytext[0]);
+    if (yytext[0] == '\001')
+      {
+        // We are here because we are using the push parser/lexer
+        // interface and we hit the end of the input buffer or file.
+        // The special ASCII 1 marker is added to the input by
+        // push_lexer::fill_flex_buffer.
+
+        if (curr_lexer->pending_token_count () > 0)
+          {
+            // We are in the middle of parsing a command, expresison,
+            // etc., so set the return status so that if we are at the
+            // end of the buffer we'll continue looking for more input,
+            // possibly buffering a series of line oriented comments as
+            // a single block.
+
+            HANDLE_EOB_OR_EOF (-1);
+          }
+        else
+          {
+            // We are not in the process of parsing a command,
+            // expression, etc., so end any current sequence of comments
+            // with this full line comment, pop the start state and
+            // return as if we have just finished parsing a complete
+            // statement.
+
+            curr_lexer->finish_comment (octave::comment_elt::full_line);
+
+            curr_lexer->pop_start_state ();
+
+            HANDLE_EOB_OR_EOF (-2);
+          }
+      }
+    else
+      {
+        // End any current sequence of comments, pop the start state,
+        // and unput the pending input character that ended the series
+        // of comments.
+
+        curr_lexer->finish_comment (octave::comment_elt::full_line);
+
+        curr_lexer->pop_start_state ();
+
+        curr_lexer->xunput (yytext[0]);
+      }
   }
 
 %{
@@ -3321,6 +3350,12 @@
     return lval->tok_val;
   }
 
+  size_t
+  base_lexer::pending_token_count (void) const
+  {
+    return m_tokens.size ();
+  }
+
   void
   base_lexer::display_token (int tok)
   {