jwe/octave: libinterp/corefcn/oct-stream.cc comparison

comparison libinterp/corefcn/oct-stream.cc @ 30833:cfb708de1fc9

textscan: Refresh buffer with overlay (bug #62152). * libinterp/corefcn/oct-stream.cc (delimited_stream): Retain overlap with previous buffer content when refreshing to allow "putback" after buffer refresh. (delimited_stream::remaining): Add function that returns the number of chars until end of stream if it is already buffered. (textscan): Use functions that refresh the buffer when peeking into delimited_stream. Avoid buffer refresh if end of stream is already buffered.

author	Markus Mützel <markus.muetzel@gmx.de>
date	Sun, 13 Mar 2022 11:30:55 +0100
parents	0826c503f294
children	79edd49a5a97

comparison

equal deleted inserted replaced

-:3e395f6fc03a
+:cfb708de1fc9
 #include <cstring>
 #include <algorithm>
 #include <deque>
 #include <fstream>
+#include <limits>
 #include <iomanip>
 #include <iostream>
 #include <sstream>
 #include <string>
 refresh_buf ();
 }
 // Load new data into buffer, and set eob, last, idx.
 // Return EOF at end of file, 0 otherwise.
-int refresh_buf (void);
+int refresh_buf (bool initialize = false);
 // Get a character, relying on caller to call field_done if
 // a delimiter has been reached.
 int get (void)
 {
 void seekg (char *old_idx) { m_idx = old_idx; }
 bool eof (void)
 {
-return (m_eob == m_buf && m_i_stream.eof ())
+return (m_eob == m_buf + m_overlap && m_i_stream.eof ())
 || (m_flags & std::ios_base::eofbit);
 }
 operator const void* (void)
 { return (! eof () && ! m_flags) ? this : nullptr; }
 void progress_benchmark (void) { m_progress_marker = m_idx; }
 bool no_progress (void) { return m_progress_marker == m_idx; }
+// Number of characters remaining until end of stream if it is already
+// buffered. int_max otherwise.
+std::ptrdiff_t remaining (void)
+{
+if (m_eob < m_buf + m_bufsize)
+return m_eob - m_idx;
+else
+return std::numeric_limits<std::ptrdiff_t>::max ();
+}
 private:
 // Number of characters to read from the file at once.
 int m_bufsize;
 // delimited is false).
 char *m_last;
 // Position after last character in buffer.
 char *m_eob;
+// Overlap with old content when refreshing buffer.
+std::ptrdiff_t m_overlap;
 // True if there is delimiter in the buffer after idx.
 bool m_delimited;
 // Longest lookahead required.
 {
 m_buf = new char[m_bufsize];
 m_eob = m_buf + m_bufsize;
 m_idx = m_eob;                // refresh_buf shouldn't try to copy old data
 m_progress_marker = m_idx;
-refresh_buf ();               // load the first batch of data
+refresh_buf (true);           // load the first batch of data
 }
 // Used to create a stream from a strstream from data read from a dstr.
 delimited_stream::delimited_stream (std::istream& is,
 const delimited_stream& ds)
 // Seek to the correct position in i_stream.
 if (! eof ())
 {
 m_i_stream.clear ();
 m_i_stream.seekg (m_buf_in_file);
-m_i_stream.read (m_buf, m_idx - m_buf);
+m_i_stream.read (m_buf, m_idx - m_buf - m_overlap);
 }
 delete [] m_buf;
 }
 // new data to fill it.  Return EOF if the file is at EOF before
 // reading any data and all of the data that has been read has been
 // processed.
 int
-delimited_stream::refresh_buf (void)
+delimited_stream::refresh_buf (bool initialize)
 {
 if (eof ())
 return std::istream::traits_type::eof ();
 int retval;
 if (m_eob < m_idx)
 m_idx = m_eob;
 std::size_t old_remaining = m_eob - m_idx;
+std::size_t old_overlap = 0;
+if (initialize || (m_idx - m_buf <= 0))
+m_overlap = 0;
+else
+{
+old_overlap = m_overlap;
+// Retain the last 25 bytes in the buffer.  That should be more than enough
+// to putback an entire double precision floating point number in decimal
+// including 3 digit exponent and signs.  Do we ever need to putback more
+// than that?
+m_overlap = 25;
+// Assure we don't "underflow" with the overlap
+m_overlap = std::min (m_overlap, m_idx - m_buf - 1);
+}
 octave_quit ();                       // allow ctrl-C
-if (old_remaining > 0)
+if (old_remaining + m_overlap > 0)
 {
-m_buf_in_file += (m_idx - m_buf);
+m_buf_in_file += (m_idx - old_overlap - m_buf);
-memmove (m_buf, m_idx, old_remaining);
+std::memmove (m_buf, m_idx - m_overlap, m_overlap + old_remaining);
 }
 else
 m_buf_in_file = m_i_stream.tellg ();  // record for destructor
-m_progress_marker -= m_idx - m_buf;  // where original idx would have been
+// where original idx would have been
-m_idx = m_buf;
+m_progress_marker -= m_idx - m_overlap - m_buf;
+m_idx = m_buf + m_overlap;
 int gcount;   // chars read
 if (! m_i_stream.eof ())
 {
-m_i_stream.read (m_buf + old_remaining, m_bufsize - old_remaining);
+m_i_stream.read (m_buf + m_overlap + old_remaining,
+m_bufsize - m_overlap - old_remaining);
 gcount = m_i_stream.gcount ();
 }
 else
 gcount = 0;
-m_eob = m_buf + old_remaining + gcount;
+m_eob = m_buf + m_overlap + old_remaining + gcount;
 m_last = m_eob;
 if (gcount == 0)
 {
 m_delimited = false;
-if (m_eob != m_buf)   // no more data in file, but still some to go
+if (m_eob != m_buf + m_overlap)
+// no more data in file, but still some to go
 retval = 0;
 else
 // file and buffer are both done.
 retval = std::istream::traits_type::eof ();
 }
 else
 {
 m_delimited = true;
-for (m_last = m_eob - m_longest; m_last - m_buf >= 0; m_last--)
+for (m_last = m_eob - m_longest; m_last - m_buf - m_overlap >= 0;
+m_last--)
 {
 if (m_delims.find (*m_last) != std::string::npos)
 break;
 }
-if (m_last < m_buf)
+if (m_last < m_buf + m_overlap)
 m_delimited = false;
 retval = 0;
 }
 char *
 delimited_stream::read (char *buffer, int size, char* &prior_tell)
 {
 char *retval;
-if (m_eob - m_idx > size)
+if (m_eob - m_idx >= size)
 {
 retval = m_idx;
 m_idx += size;
 if (m_idx > m_last)
 m_delimited = false;
 buf_size = std::min (buf_size, std::max (ntimes, 80 * ntimes));
 buf_size = std::max (buf_size, ntimes);
 }
 // Finally, create the stream.
 delimited_stream is (isp,
-(m_delim_table.empty () ? m_whitespace + "\r\n"
+(m_delims.empty () ? m_whitespace + "\r\n"
 : m_delims),
 max_lookahead, buf_size);
 // Grow retval dynamically.  "size" is half the initial size
 // (FIXME: Should we start smaller if ntimes is large?)
 octave_idx_type size = ((ntimes < 8 && ntimes >= 0) ? ntimes : 1);
 int sign = 1;
 unsigned int width_left = fmt.width;
 double retval = 0;
 bool valid = false;         // syntactically correct double?
-int ch = is.peek ();
+int ch = is.peek_undelim ();
 if (ch == '+')
 {
 is.get ();
-ch = is.peek ();
+ch = is.peek_undelim ();
 if (width_left)
 width_left--;
 }
 else if (ch == '-')
 {
 sign = -1;
 is.get ();
-ch = is.peek ();
+ch = is.peek_undelim ();
 if (width_left)
 width_left--;
 }
 // Read integer part
 // look for exponent part in, e.g.,  6.023E+23
 bool used_exp = false;
 if (valid && width_left > 1 && m_exp_chars.find (ch) != std::string::npos)
 {
-int ch1 = is.peek ();
+int ch1 = is.peek_undelim ();
 if (ch1 == '-' || ch1 == '+' || (ch1 >= '0' && ch1 <= '9'))
 {
 // if 1.0e+$ or some such, this will set failbit, as we want
 width_left--;                         // count "E"
 int exp = 0;
 width_left--;
 exp_sign = -1;
 is.get ();
 }
 valid = false;
-while (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9')
+while (width_left-- && is && (ch = is.get_undelim ()) >= '0' && ch <= '9')
 {
 exp = exp*10 + ch - '0';
 valid = true;
 }
 width_left++;
 is.clear ();
 if (! used_exp && ch != std::istream::traits_type::eof () && width_left)
 is.putback (ch);
 // Check for +/- inf and NaN
-if (! valid && width_left >= 3)
+if (! valid && width_left >= 3 && is.remaining () >= 3)
 {
 int i = lookahead (is, m_inf_nan, 3, false);  // false->case insensitive
 if (i == 0)
 {
 retval = numeric_limits<double>::Inf ();
 double im = 0;
 double re = 0;
 bool as_empty = false;  // did we fail but match a "treat_as_empty" string?
 bool inf = false;
-int ch = is.peek ();
+int ch = is.peek_undelim ();
 if (ch == '+' || ch == '-')   // check for [+-][ij] with no coefficients
 {
 ch = is.get ();
-int ch2 = is.peek ();
+int ch2 = is.peek_undelim ();
 if (ch2 == 'i' || ch2 == 'j')
 {
 double value = 1;
 is.get ();
 // Check not -inf
-if (is.peek () == 'n')
+if (is.peek_undelim () == 'n')
 {
 char *pos = is.tellg ();
 std::ios::iostate state = is.rdstate ();
 is.get ();
-ch2 = is.get ();
+ch2 = is.get_undelim ();
 if (ch2 == 'f')
 {
 inf = true;
 re = (ch == '+' ? numeric_limits<double>::Inf ()
 : -numeric_limits<double>::Inf ());
 value = 0;
 }
 else
 {
 is.clear (state);
+// FIXME: Buffer might have refreshed.
+//        pos might no longer be valid.
 is.seekg (pos);   // reset to position before look-ahead
 }
 }
 im = (ch == '+') ? value : -value;
 if (! im && ! inf)        // if not [+-][ij] or [+-]inf, read real normally
 {
 char *pos = is.tellg ();
 std::ios::iostate state = is.rdstate ();
 //re = read_value<double> (is);
+// FIXME: read_double might refresh the buffer.  So seekg might be off.
 re = read_double (is, fmt);
 // check for "treat as empty" string
 if (m_treat_as_empty.numel ()
 && (is.fail () || math::is_NaN_or_NA (Complex (re))
 if (! is.eof () && ! as_empty)
 {
 state = is.rdstate ();   // before tellg, since that fails at EOF
-ch = is.peek ();   // ch == EOF if read failed; no need to chk fail
+ch = is.peek_undelim ();
+// ch == EOF if read failed; no need to chk fail
 if (ch == 'i' || ch == 'j')           // pure imaginary
 {
 is.get ();
 im = re;
 re = 0;
 // save stream state in case we have to restore it
 pos   = is.tellg ();
 state = is.rdstate ();
 //im = read_value<double> (is);
+// FIXME: read_double might refresh the buffer.
+//        So seekg might be off after this.
 im = read_double (is, fmt);
 if (is.fail ())
 im = 1;
-if (is.peek () == 'i' || is.peek () == 'j')
+if (is.peek_undelim () == 'i' || is.peek_undelim () == 'j')
 is.get ();
 else
 {
 im = 0;           // no valid imaginary part.  Restore state
 is.clear (state); // eof shouldn't cause fail.
 // Grow string in an exponential fashion if necessary.
 if (i >= val.length ())
 val.append (std::max (val.length (),
 static_cast<std::size_t> (16)), '\0');
-int ch = is.get ();
+int ch = is.get_undelim ();
 if (is_delim (ch) || ch == std::istream::traits_type::eof ())
 {
 is.putback (ch);
 break;
 }
 textscan::scan_qstring (delimited_stream& is, const textscan_format_elt& fmt,
 std::string& val)
 {
 skip_whitespace (is);
-if (is.peek () != '"')
+if (is.peek_undelim () != '"')
 scan_string (is, fmt, val);
 else
 {
 is.get ();
 scan_caret (is, R"(")", val);     // read everything until "
 if (! is.eof ())
 {
 if (m_delim_list.isempty ())
 {
-if (! is_delim (is.peek ()))
+if (! is_delim (is.peek_undelim ()))
 this_conversion_failed = true;
 }
 else  // Cell array of multi-character delimiters
 {
 char *pos = is.tellg ();
 {
 if (is_delim (c1) || c1 == m_eol1 || c1 == m_eol2)
 {
 is.get ();
 if (c1 == m_eol1 && is.peek_undelim () == m_eol2)
-is.get_undelim ();          // if \r\n, skip the \n too.
+is.get ();          // if \r\n, skip the \n too.
 if (multiple_delims_as_one)
 {
 int prev = -1;
 // skip multiple delims.

Mercurial > jwe > octave

comparison libinterp/corefcn/oct-stream.cc @ 30833:cfb708de1fc9