changeset 16230:4bf907906134

use a queue to hold tokens in the lexer * lex.h, lex.ll (lexical_feedback::token_cache): New class. (lexical_feedback::tokens): Rename from token_stack. Declare as token_cache, not std::stack<token *>. Change all uses. (lexical_feedback::reset_token_stack): Delete.
author John W. Eaton <jwe@octave.org>
date Sat, 09 Mar 2013 01:20:23 -0500
parents 7b7b1e4968e8
children 2b15ae55c721
files libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll
diffstat 2 files changed, 70 insertions(+), 26 deletions(-) [+]
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h	Sat Mar 09 00:02:58 2013 -0500
+++ b/libinterp/parse-tree/lex.h	Sat Mar 09 01:20:23 2013 -0500
@@ -23,12 +23,15 @@
 #if !defined (octave_lex_h)
 #define octave_lex_h 1
 
+#include <deque>
+#include <limits>
 #include <list>
 #include <set>
 #include <stack>
 
 #include "comment-list.h"
 #include "input.h"
+#include "token.h"
 
 extern OCTINTERP_API void cleanup_parser (void);
 
@@ -53,9 +56,6 @@
   stream_reader& operator = (const stream_reader&);
 };
 
-// Forward decl for octave_lexer::token_stack.
-class token;
-
 // For communication between the lexer and parser.
 
 class
@@ -160,6 +160,67 @@
 
     std::stack<int> context;
   };
+
+  class token_cache
+  {
+  public:
+
+    // Store an "unlimited" number of tokens.
+    token_cache (size_t sz_arg = std::numeric_limits<size_t>::max ())
+      : buffer (), sz (sz_arg)
+    { }
+
+    void push (token *tok)
+    {
+      if (buffer.size () == sz)
+        pop ();
+
+      buffer.push_front (tok);
+    }
+
+    void pop (void)
+    {
+      if (! empty ())
+        {
+          delete buffer.back ();
+          buffer.pop_back ();
+        }
+    }
+
+    // Direct access.
+    token *at (size_t n) { return buffer.at (n); }
+    const token *at (size_t n) const { return buffer.at (n); }
+
+    // Most recently pushed.
+    token *front (void) { return buffer.front (); }
+    const token *front (void) const { return buffer.front (); }
+
+    token *back (void) { return buffer.back (); }
+    const token *back (void) const { return buffer.back (); }
+
+    // Number of elements currently in the buffer, max of sz.
+    size_t size (void) const { return buffer.size (); }
+
+    bool empty (void) const { return buffer.empty (); }
+
+    void clear (void)
+    {
+      while (! empty ())
+        pop ();
+    }
+
+  private:
+
+    std::deque<token *> buffer;
+
+    size_t sz;
+
+    // No copying!
+
+    token_cache (const token_cache&);
+
+    token_cache& operator = (const token_cache&);
+  };
   
   lexical_feedback (void)
     : end_of_input (false), convert_spaces_to_comma (true),
@@ -181,7 +242,7 @@
       current_input_line (), comment_text (), help_text (),
       fcn_file_name (), fcn_file_full_name (), looking_at_object_index (),
       parsed_function_name (), pending_local_variables (),
-      nesting_level (), token_stack ()
+      nesting_level (), tokens ()
   {
     init ();
   }
@@ -322,15 +383,11 @@
   // a paren?
   bbp_nesting_level nesting_level;
 
-  // Stack to hold tokens so that we can delete them when the parser is
-  // reset and avoid growing forever just because we are stashing some
-  // information.
-  std::stack <token*> token_stack;
+  // Tokens generated by the lexer.
+  token_cache tokens;
 
 private:
 
-  void reset_token_stack (void);
-
   // No copying!
 
   lexical_feedback (const lexical_feedback&);
--- a/libinterp/parse-tree/lex.ll	Sat Mar 09 00:02:58 2013 -0500
+++ b/libinterp/parse-tree/lex.ll	Sat Mar 09 01:20:23 2013 -0500
@@ -1339,7 +1339,7 @@
 
 lexical_feedback::~lexical_feedback (void)
 {
-  reset_token_stack ();
+  tokens.clear ();
 }
 
 void
@@ -1397,7 +1397,7 @@
 
   nesting_level.reset ();
 
-  reset_token_stack ();
+  tokens.clear ();
 }
 
 static bool
@@ -1446,19 +1446,6 @@
 }
 
 void
-lexical_feedback::reset_token_stack (void)
-{
-  // Clear out the stack of token info used to track line and
-  // column numbers.
-
-  while (! token_stack.empty ())
-    {
-      delete token_stack.top ();
-      token_stack.pop ();
-    }
-}
-
-void
 octave_lexer::input_buffer::fill (const std::string& input, bool eof_arg)
 {
   buffer = input;
@@ -3374,7 +3361,7 @@
 {
   YYSTYPE *lval = yyget_lval (scanner);
   lval->tok_val = tok;
-  token_stack.push (tok);
+  tokens.push (tok);
 }
 
 token *