changeset 22857:d90e0c79aa45

Backed out changeset d5b58e234c37
author John W. Eaton <jwe@octave.org>
date Fri, 02 Dec 2016 14:16:43 -0500
parents d5b58e234c37
children a183a0929653
files libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll
diffstat 2 files changed, 102 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h	Fri Dec 02 14:52:09 2016 +0100
+++ b/libinterp/parse-tree/lex.h	Fri Dec 02 14:16:43 2016 -0500
@@ -177,6 +177,88 @@
       std::stack<int> context;
     };
 
+    class token_cache
+    {
+    public:
+
+      // Store an "unlimited" number of tokens.
+      token_cache (size_t sz_arg = std::numeric_limits<size_t>::max ())
+        : buffer (), sz (sz_arg)
+      { }
+
+      void push (token *tok)
+      {
+        if (buffer.size () == sz)
+          pop ();
+
+        buffer.push_front (tok);
+      }
+
+      void pop (void)
+      {
+        if (! empty ())
+          {
+            delete buffer.back ();
+            buffer.pop_back ();
+          }
+      }
+
+      // Direct access.
+      token *at (size_t n)
+      {
+        return empty () ? 0 : buffer.at (n);
+      }
+
+      const token *at (size_t n) const
+      {
+        return empty () ? 0 : buffer.at (n);
+      }
+
+      // Most recently pushed.
+      token *front (void)
+      {
+        return empty () ? 0 : buffer.front ();
+      }
+
+      const token *front (void) const
+      {
+        return empty () ? 0 : buffer.front ();
+      }
+
+      token *back (void)
+      {
+        return empty () ? 0 : buffer.back ();
+      }
+
+      const token *back (void) const
+      {
+        return empty () ? 0 : buffer.back ();
+      }
+
+      // Number of elements currently in the buffer, max of sz.
+      size_t size (void) const { return buffer.size (); }
+
+      bool empty (void) const { return buffer.empty (); }
+
+      void clear (void)
+      {
+        while (! empty ())
+          pop ();
+      }
+
+    private:
+
+      std::deque<token *> buffer;
+
+      size_t sz;
+
+      // No copying!
+
+      token_cache (const token_cache&);
+
+      token_cache& operator = (const token_cache&);
+    };
+
     lexical_feedback (void)
       : end_of_input (false), at_beginning_of_statement (true),
         looking_at_anon_fcn_args (false), looking_at_return_list (false),
@@ -370,7 +452,7 @@
     bbp_nesting_level nesting_level;
 
     // Tokens generated by the lexer.
-    std::deque<token *> tokens;
+    token_cache tokens;
 
   private:
 
@@ -770,3 +852,4 @@
 }
 
 #endif
+
--- a/libinterp/parse-tree/lex.ll	Fri Dec 02 14:52:09 2016 +0100
+++ b/libinterp/parse-tree/lex.ll	Fri Dec 02 14:16:43 2016 -0500
@@ -2145,26 +2145,30 @@
   int
   lexical_feedback::previous_token_value (void) const
   {
-    return tokens.empty () ? 0 : tokens.front ()->token_value ();
+    const token *tok = tokens.front ();
+    return tok ? tok->token_value () : 0;
   }
 
   bool
   lexical_feedback::previous_token_value_is (int tok_val) const
   {
-    return tokens.empty () ? false : tokens.front ()->token_value_is (tok_val);
+    const token *tok = tokens.front ();
+    return tok ? tok->token_value_is (tok_val) : false;
   }
 
   void
   lexical_feedback::mark_previous_token_trailing_space (void)
   {
-    if (! tokens.empty () && ! previous_token_value_is ('\n'))
-      tokens.front ()->mark_trailing_space ();
+    token *tok = tokens.front ();
+    if (tok && ! previous_token_value_is ('\n'))
+      tok->mark_trailing_space ();
   }
 
   bool
   lexical_feedback::space_follows_previous_token (void) const
   {
-    return tokens.empty () ? false : tokens.front ()->space_follows_token ();
+    const token *tok = tokens.front ();
+    return tok ? tok->space_follows_token () : false;
   }
 
   bool
@@ -2190,25 +2194,24 @@
   bool
   lexical_feedback::previous_token_is_keyword (void) const
   {
-    return tokens.empty () ? false : tokens.front ()->is_keyword ();
+    const token *tok = tokens.front ();
+    return tok ? tok->is_keyword () : false;
   }
 
   bool
   lexical_feedback::previous_token_may_be_command (void) const
   {
-    return tokens.empty () ? false : tokens.front ()->may_be_command ();
+    const token *tok = tokens.front ();
+    return tok ? tok->may_be_command () : false;
   }
 
   void
   lexical_feedback::maybe_mark_previous_token_as_variable (void)
   {
-    if (! tokens.empty ())
-    {
-      token *tok = tokens.front ();
-
-      if (tok->is_symbol ())
-        pending_local_variables.insert (tok->symbol_name ());
-    }
+    token *tok = tokens.front ();
+
+    if (tok && tok->is_symbol ())
+      pending_local_variables.insert (tok->symbol_name ());
   }
 
   void
@@ -3200,7 +3203,7 @@
   {
     YYSTYPE *lval = yyget_lval (scanner);
     lval->tok_val = tok;
-    tokens.push_front (tok);
+    tokens.push (tok);
   }
 
   token *