changeset 28407:808e3964987b stable

accept multi-line interactive input (bug #58370) * lex.h, lex.ll (base_lexer::input_buffer::m_pos): Delete. Use offset from beginning of buffer instead. (base_lexer::input_buffer::m_offset): New data member. (base_lexer::input_buffer::copy_chunk): New argument, BY_LINES. Perform line buffering on input if BY_LINES is true. (push_lexer::fill_flex_buffer): Call copy_chunk with BY_LINES = true. * parse.h, oct-parse.yy (base_parser::statement_list): Append new statements to existing m_stmt_list. (base_parser::run (const std::string&, bool)): Also continue parsing input if lexer input buffer is not empty.
author John W. Eaton <jwe@octave.org>
date Thu, 14 May 2020 23:27:43 -0400
parents 26d69b0a3e8d
children 99ffd1058bec 1296a89b08f4
files libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll libinterp/parse-tree/oct-parse.yy libinterp/parse-tree/parse.h
diffstat 4 files changed, 58 insertions(+), 22 deletions(-) [+]
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h	Thu Jun 04 15:48:30 2020 +0200
+++ b/libinterp/parse-tree/lex.h	Thu May 14 23:27:43 2020 -0400
@@ -542,13 +542,13 @@
     public:
 
       input_buffer (void)
-        : m_buffer (), m_pos (nullptr), m_chars_left (0), m_eof (false)
+        : m_buffer (), m_offset (0), m_chars_left (0), m_eof (false)
       { }
 
       void fill (const std::string& input, bool eof_arg);
 
       // Copy at most max_size characters to buf.
-      int copy_chunk (char *buf, size_t max_size);
+      int copy_chunk (char *buf, size_t max_size, bool by_lines = false);
 
       bool empty (void) const { return m_chars_left == 0; }
 
@@ -557,7 +557,7 @@
     private:
 
       std::string m_buffer;
-      const char *m_pos;
+      size_t m_offset;
       size_t m_chars_left;
       bool m_eof;
     };
--- a/libinterp/parse-tree/lex.ll	Thu Jun 04 15:48:30 2020 +0200
+++ b/libinterp/parse-tree/lex.ll	Thu May 14 23:27:43 2020 -0400
@@ -2365,24 +2365,37 @@
   {
     m_buffer = input;
     m_chars_left = m_buffer.length ();
-    m_pos = m_buffer.c_str ();
+    m_offset = 0;
     m_eof = eof_arg;
   }
 
+  // If BY_LINES is true, return chunks to the lexer line by line.
   int
-  base_lexer::input_buffer::copy_chunk (char *buf, size_t max_size)
+  base_lexer::input_buffer::copy_chunk (char *buf, size_t max_size,
+                                        bool by_lines)
   {
     static const char * const eol = "\n";
 
-    size_t len = max_size > m_chars_left ? m_chars_left : max_size;
+    size_t len = 0;
+    if (by_lines)
+      {
+        size_t newline_pos = m_buffer.find ('\n', m_offset);
+        len = (newline_pos != std::string::npos
+               ? newline_pos - m_offset + 1
+               : (max_size > m_chars_left ? m_chars_left : max_size));
+      }
+    else
+      len = max_size > m_chars_left ? m_chars_left : max_size;
+
     assert (len > 0);
-
-    memcpy (buf, m_pos, len);
+    memcpy (buf, m_buffer.c_str () + m_offset, len);
 
     m_chars_left -= len;
-    m_pos += len;
-
-    // Make sure input ends with a new line character.
+    m_offset += len;
+
+    // Make sure the final input returned to the lexer ends with a new
+    // line character.
+
     if (m_chars_left == 0 && buf[len-1] != '\n')
       {
         if (len < max_size)
@@ -2394,10 +2407,17 @@
         else
           {
             // There isn't enough room to plug the newline character
-            // in the buffer so arrange to have it returned on the next
-            // call to base_lexer::read.
-            m_pos = eol;
+            // in BUF so arrange to have it returned on the next call
+            // to base_lexer::read.
+
+            // At this point we've exhausted the original input
+            // (m_chars_left is zero) so we can overwrite the initial
+            // buffer with a single newline character to be returned on
+            // the next call.
+
+            m_buffer = eol;
             m_chars_left = 1;
+            m_offset = 0;
           }
       }
 
@@ -3831,7 +3851,7 @@
         // character to the input.
 
         if (! m_input_buf.empty ())
-          status = m_input_buf.copy_chunk (buf, max_size);
+          status = m_input_buf.copy_chunk (buf, max_size, true);
         else
           status = YY_NULL;
       }
--- a/libinterp/parse-tree/oct-parse.yy	Thu Jun 04 15:48:30 2020 +0200
+++ b/libinterp/parse-tree/oct-parse.yy	Thu May 14 23:27:43 2020 -0400
@@ -385,7 +385,8 @@
                     YYUSE ($2);
 
                     $$ = nullptr;
-                    parser.statement_list (std::shared_ptr<octave::tree_statement_list> ($1));
+                    std::shared_ptr<octave::tree_statement_list> tmp_lst ($1);
+                    parser.statement_list (tmp_lst);
                     YYACCEPT;
                   }
                 | simple_list END_OF_INPUT
@@ -394,7 +395,8 @@
 
                     $$ = nullptr;
                     lexer.m_end_of_input = true;
-                    parser.statement_list (std::shared_ptr<octave::tree_statement_list> ($1));
+                    std::shared_ptr<octave::tree_statement_list> tmp_lst ($1);
+                    parser.statement_list (tmp_lst);
                     YYACCEPT;
                   }
                 | parse_error
@@ -2437,6 +2439,23 @@
   }
 
   void
+  base_parser::statement_list (std::shared_ptr<tree_statement_list>& lst)
+  {
+    if (m_stmt_list)
+      {
+        // Append additional code to existing statement list.
+
+        while (! lst->empty ())
+          {
+            m_stmt_list->push_back (lst->front ());
+            lst->pop_front ();
+          }
+      }
+    else
+      m_stmt_list = lst;
+  }
+
+  void
   base_parser::end_token_error (token *tok, token::end_tok_type expected)
   {
     std::string msg = ("'" + end_token_as_string (expected)
@@ -4709,7 +4728,7 @@
               error ("unexpected exception while parsing %s", file.c_str ());
           }
       }
-    while (status == YYPUSH_MORE);
+    while (status == YYPUSH_MORE || ! m_lexer.at_end_of_buffer ());
 
     if (status != 0)
       parse_error ("%s", m_parse_error_msg.c_str ());
--- a/libinterp/parse-tree/parse.h	Thu Jun 04 15:48:30 2020 +0200
+++ b/libinterp/parse-tree/parse.h	Thu May 14 23:27:43 2020 -0400
@@ -172,10 +172,7 @@
       return m_classdef_object;
     }
 
-    void statement_list (const std::shared_ptr<tree_statement_list>& lst)
-    {
-      m_stmt_list = lst;
-    }
+    void statement_list (std::shared_ptr<tree_statement_list>& lst);
 
     std::shared_ptr<tree_statement_list> statement_list (void) const
     {