Mercurial > octave-nkf

changeset 16293:57e87ddfee14

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
create base class for lexer * lex.h, lex.ll, parse.h, oct-parse.in.yy: (octave_base_lexer): New base class for lexer class. Move most of previous octave_lexer class here. Change all uses.
author John W. Eaton <jwe@octave.org>
date Wed, 13 Mar 2013 02:46:56 -0400
parents 6ce905b89cee
children 0925d1f6875e
files libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll libinterp/parse-tree/oct-parse.in.yy libinterp/parse-tree/parse.h
diffstat 4 files changed, 174 insertions(+), 138 deletions(-) [+]
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h	Wed Mar 13 02:25:50 2013 -0400
+++ b/libinterp/parse-tree/lex.h	Wed Mar 13 02:46:56 2013 -0400
@@ -391,13 +391,13 @@
   lexical_feedback& operator = (const lexical_feedback&);
 };
 
-// octave_lexer inherits from lexical_feedback because we will
+// octave_base_lexer inherits from lexical_feedback because we will
 // eventually have several different constructors and it is easier to
 // intialize if everything is grouped in a parent class rather than
-// listing all the members in the octave_lexer class.
+// listing all the members in the octave_base_lexer class.
 
 class
-octave_lexer : public lexical_feedback
+octave_base_lexer : public lexical_feedback
 {
 public:
 
@@ -428,35 +428,21 @@
     bool eof;
   };
 
-  octave_lexer (void)
-    : lexical_feedback (), scanner (0), input_buf (), input_reader ()
-  {
-    init ();
-  }
-
-  octave_lexer (FILE *file)
-    : lexical_feedback (), scanner (0), input_buf (),
-      input_reader (file)
+  octave_base_lexer (void)
+    : lexical_feedback (), scanner (0), input_buf ()
   {
     init ();
   }
 
-  octave_lexer (const std::string& eval_string)
-    : lexical_feedback (), scanner (0), input_buf (),
-      input_reader (eval_string)
-  {
-    init ();
-  }
-
-  ~octave_lexer (void);
+  virtual ~octave_base_lexer (void);
 
   void init (void);
 
-  void reset (void);
+  virtual void reset (void);
 
   void prep_for_file (void);
 
-  int read (char *buf, unsigned int max_size);
+  virtual int fill_flex_buffer (char *buf, unsigned int max_size) = 0;
 
   int handle_end_of_input (void);
 
@@ -529,35 +515,21 @@
   // Object that reads and buffers input.
   input_buffer input_buf;
 
-  octave_input_reader input_reader;
+  virtual void increment_promptflag (void) = 0;
 
-  void increment_promptflag (void) { input_reader.increment_promptflag (); }
-
-  void decrement_promptflag (void) { input_reader.decrement_promptflag (); }
+  virtual void decrement_promptflag (void) = 0;
 
-  int promptflag (void) const { return input_reader.promptflag (); }
-
-  int promptflag (int n) { return input_reader.promptflag (n); }
+  virtual int promptflag (void) const = 0;
 
-  std::string input_source (void) const
-  {
-    return input_reader.input_source ();
-  }
+  virtual int promptflag (int) = 0;
 
-  bool input_from_terminal (void) const
-  {
-    return input_source () == "terminal";
-  }
+  virtual std::string input_source (void) const { return "unknown"; }
+
+  virtual bool input_from_terminal (void) const { return false; }
 
-  bool input_from_file (void) const
-  {
-    return input_source () == "file";
-  }
+  virtual bool input_from_file (void) const { return false; }
 
-  bool input_from_eval_string (void) const
-  {
-    return input_source () == "eval_string";
-  }
+  virtual bool input_from_eval_string (void) const { return false; }
 
   void push_start_state (int state);
 
@@ -596,14 +568,79 @@
   int show_token (int tok);
 
   // For unwind protect.
-  static void cleanup (octave_lexer *lexer) { delete lexer; }
+  static void cleanup (octave_base_lexer *lexer) { delete lexer; }
 
-private:
+protected:
 
   std::stack<int> start_state_stack;
 
   // No copying!
 
+  octave_base_lexer (const octave_base_lexer&);
+
+  octave_base_lexer& operator = (const octave_base_lexer&);
+};
+
+class
+octave_lexer : public octave_base_lexer
+{
+public:
+
+  octave_lexer (void)
+    : octave_base_lexer (), input_reader ()
+  { }
+
+  octave_lexer (FILE *file)
+    : octave_base_lexer (), input_reader (file)
+  { }
+
+  octave_lexer (const std::string& eval_string)
+    : octave_base_lexer (), input_reader (eval_string)
+  { }
+
+  void reset (void)
+  {
+    input_reader.reset ();
+
+    octave_base_lexer::reset ();
+  }
+
+  void increment_promptflag (void) { input_reader.increment_promptflag (); }
+
+  void decrement_promptflag (void) { input_reader.decrement_promptflag (); }
+
+  int promptflag (void) const { return input_reader.promptflag (); }
+
+  int promptflag (int n) { return input_reader.promptflag (n); }
+
+  std::string input_source (void) const
+  {
+    return input_reader.input_source ();
+  }
+
+  bool input_from_terminal (void) const
+  {
+    return input_source () == "terminal";
+  }
+
+  bool input_from_file (void) const
+  {
+    return input_source () == "file";
+  }
+
+  bool input_from_eval_string (void) const
+  {
+    return input_source () == "eval_string";
+  }
+
+  int fill_flex_buffer (char *buf, unsigned int max_size);
+
+  octave_input_reader input_reader;
+
+protected:
+
+  // No copying!
+
   octave_lexer (const octave_lexer&);
 
   octave_lexer& operator = (const octave_lexer&);
--- a/libinterp/parse-tree/lex.ll	Wed Mar 13 02:25:50 2013 -0400
+++ b/libinterp/parse-tree/lex.ll	Wed Mar 13 02:46:56 2013 -0400
@@ -111,7 +111,7 @@
 #error lex.l requires flex version 2.5.4 or later
 #endif
 
-#define YY_EXTRA_TYPE octave_lexer *
+#define YY_EXTRA_TYPE octave_base_lexer *
 #define curr_lexer yyextra
 
 // Arrange to get input via readline.
@@ -120,7 +120,7 @@
 #undef YY_INPUT
 #endif
 #define YY_INPUT(buf, result, max_size) \
-  result = curr_lexer->read (buf, max_size)
+  result = curr_lexer->fill_flex_buffer (buf, max_size)
 
 // Try to avoid crashing out completely on fatal scanner errors.
 
@@ -1547,7 +1547,7 @@
 }
 
 void
-octave_lexer::input_buffer::fill (const std::string& input, bool eof_arg)
+octave_base_lexer::input_buffer::fill (const std::string& input, bool eof_arg)
 {
   buffer = input;
   chars_left = buffer.length ();
@@ -1556,7 +1556,7 @@
 }
 
 int
-octave_lexer::input_buffer::copy_chunk (char *buf, size_t max_size)
+octave_base_lexer::input_buffer::copy_chunk (char *buf, size_t max_size)
 {
   static const char * const eol = "\n";
 
@@ -1581,7 +1581,7 @@
         {
           // There isn't enough room to plug the newline character
           // in the buffer so arrange to have it returned on the next
-          // call to octave_lexer::read.
+          // call to octave_base_lexer::read.
           pos = eol;
           chars_left = 1;
         }
@@ -1590,17 +1590,17 @@
   return len;
 }
 
-octave_lexer::~octave_lexer (void)
+octave_base_lexer::~octave_base_lexer (void)
 {
   yylex_destroy (scanner);
 }
 
 void
-octave_lexer::init (void)
+octave_base_lexer::init (void)
 {
   yylex_init (&scanner);
 
-  // Make octave_lexer object available through yyextra in
+  // Make octave_base_lexer object available through yyextra in
   // flex-generated lexer.
   yyset_extra (this, scanner);
 
@@ -1608,7 +1608,7 @@
 }
 
 // Inside Flex-generated functions, yyg is the scanner cast to its real
-// type.  Some flex macros that we use in octave_lexer member functions
+// type.  Some flex macros that we use in octave_base_lexer member functions
 // (for example, BEGIN) use yyg.  If we could perform the actions of
 // these macros with functions instead, we could eliminate the
 // OCTAVE_YYG macro.
@@ -1617,7 +1617,7 @@
   struct yyguts_t *yyg = static_cast<struct yyguts_t*> (scanner)
 
 void
-octave_lexer::reset (void)
+octave_base_lexer::reset (void)
 {
   // Start off on the right foot.
   clear_start_state ();
@@ -1638,13 +1638,11 @@
             || input_from_eval_string ()))
     yyrestart (stdin, scanner);
 
-  input_reader.reset ();
-
   lexical_feedback::reset ();
 }
 
 void
-octave_lexer::prep_for_file (void)
+octave_base_lexer::prep_for_file (void)
 {
   reading_script_file = true;
 
@@ -1652,32 +1650,7 @@
 }
 
 int
-octave_lexer::read (char *buf, unsigned max_size)
-{
-  int status = 0;
-
-  if (input_buf.empty ())
-    {
-      bool eof = false;
-      current_input_line = input_reader.get_input (eof);
-      input_buf.fill (current_input_line, eof);
-    }
-
-  if (! input_buf.empty ())
-    status = input_buf.copy_chunk (buf, max_size);
-  else
-    {
-      status = YY_NULL;
-
-      if (! input_buf.at_eof ())
-        fatal_error ("octave_lexer::read () in flex scanner failed");
-    }
-
-  return status;
-}
-
-int
-octave_lexer::handle_end_of_input (void)
+octave_base_lexer::handle_end_of_input (void)
 {
   lexer_debug ("<<EOF>>");
 
@@ -1695,19 +1668,19 @@
 }
 
 char *
-octave_lexer::flex_yytext (void)
+octave_base_lexer::flex_yytext (void)
 {
   return yyget_text (scanner);
 }
 
 int
-octave_lexer::flex_yyleng (void)
+octave_base_lexer::flex_yyleng (void)
 {
   return yyget_leng (scanner);
 }
 
 int
-octave_lexer::text_yyinput (void)
+octave_base_lexer::text_yyinput (void)
 {
   int c = yyinput (scanner);
 
@@ -1742,7 +1715,7 @@
 }
 
 void
-octave_lexer::xunput (char c, char *buf)
+octave_base_lexer::xunput (char c, char *buf)
 {
   if (c != EOF)
     {
@@ -1758,7 +1731,7 @@
 }
 
 void
-octave_lexer::xunput (char c)
+octave_base_lexer::xunput (char c)
 {
   char *yytxt = flex_yytext ();
 
@@ -1766,7 +1739,7 @@
 }
 
 bool
-octave_lexer::looking_at_space (void)
+octave_base_lexer::looking_at_space (void)
 {
   int c = text_yyinput ();
   xunput (c);
@@ -1774,7 +1747,7 @@
 }
 
 bool
-octave_lexer::inside_any_object_index (void)
+octave_base_lexer::inside_any_object_index (void)
 {
   bool retval = false;
 
@@ -1794,7 +1767,7 @@
 // Handle keywords.  Return -1 if the keyword should be ignored.
 
 int
-octave_lexer::is_keyword_token (const std::string& s)
+octave_base_lexer::is_keyword_token (const std::string& s)
 {
   int l = input_line_number;
   int c = current_input_column;
@@ -2025,7 +1998,7 @@
 }
 
 bool
-octave_lexer::whitespace_is_significant (void)
+octave_base_lexer::whitespace_is_significant (void)
 {
   return (nesting_level.is_bracket ()
           || (nesting_level.is_brace ()
@@ -2039,7 +2012,7 @@
 }
 
 void
-octave_lexer::handle_number (void)
+octave_base_lexer::handle_number (void)
 {
   double value = 0.0;
   int nread = 0;
@@ -2082,7 +2055,7 @@
 }
 
 void
-octave_lexer::handle_continuation (void)
+octave_base_lexer::handle_continuation (void)
 {
   char *yytxt = flex_yytext ();
   int yylng = flex_yyleng ();
@@ -2135,8 +2108,8 @@
 }
 
 void
-octave_lexer::finish_comment (octave_comment_elt::comment_type typ,
-                              bool looking_at_continuation)
+octave_base_lexer::finish_comment (octave_comment_elt::comment_type typ,
+                                   bool looking_at_continuation)
 {
   bool copyright = looks_like_copyright (comment_text);
 
@@ -2173,7 +2146,7 @@
 // FIXME -- we need to handle block comments here.
 
 bool
-octave_lexer::have_continuation (bool trailing_comments_ok)
+octave_base_lexer::have_continuation (bool trailing_comments_ok)
 {
   std::ostringstream buf;
 
@@ -2261,7 +2234,7 @@
 // line character.
 
 bool
-octave_lexer::have_ellipsis_continuation (bool trailing_comments_ok)
+octave_base_lexer::have_ellipsis_continuation (bool trailing_comments_ok)
 {
   char c1 = text_yyinput ();
   if (c1 == '.')
@@ -2282,7 +2255,7 @@
 }
 
 int
-octave_lexer::handle_string (char delim)
+octave_base_lexer::handle_string (char delim)
 {
   std::ostringstream buf;
 
@@ -2374,7 +2347,7 @@
 }
 
 int
-octave_lexer::handle_close_bracket (int bracket_type)
+octave_base_lexer::handle_close_bracket (int bracket_type)
 {
   int retval = bracket_type;
 
@@ -2396,7 +2369,7 @@
 }
 
 bool
-octave_lexer::looks_like_command_arg (void)
+octave_base_lexer::looks_like_command_arg (void)
 {
   bool space_before = space_follows_previous_token ();
   bool space_after = looking_at_space ();
@@ -2406,7 +2379,7 @@
 }
 
 int
-octave_lexer::handle_superclass_identifier (void)
+octave_base_lexer::handle_superclass_identifier (void)
 {
   std::string pkg;
   char *yytxt = flex_yytext ();
@@ -2442,7 +2415,7 @@
 }
 
 int
-octave_lexer::handle_meta_identifier (void)
+octave_base_lexer::handle_meta_identifier (void)
 {
   std::string pkg;
   char *yytxt = flex_yytext ();
@@ -2477,7 +2450,7 @@
 // should be ignored.
 
 int
-octave_lexer::handle_identifier (void)
+octave_base_lexer::handle_identifier (void)
 {
   char *yytxt = flex_yytext ();
 
@@ -2578,7 +2551,7 @@
 }
 
 void
-octave_lexer::maybe_warn_separator_insert (char sep)
+octave_base_lexer::maybe_warn_separator_insert (char sep)
 {
   std::string nm = fcn_file_full_name;
 
@@ -2593,7 +2566,7 @@
 }
 
 void
-octave_lexer::gripe_single_quote_string (void)
+octave_base_lexer::gripe_single_quote_string (void)
 {
   std::string nm = fcn_file_full_name;
 
@@ -2608,7 +2581,7 @@
 }
 
 void
-octave_lexer::gripe_matlab_incompatible (const std::string& msg)
+octave_base_lexer::gripe_matlab_incompatible (const std::string& msg)
 {
   std::string nm = fcn_file_full_name;
 
@@ -2623,20 +2596,20 @@
 }
 
 void
-octave_lexer::maybe_gripe_matlab_incompatible_comment (char c)
+octave_base_lexer::maybe_gripe_matlab_incompatible_comment (char c)
 {
   if (c == '#')
     gripe_matlab_incompatible ("# used as comment character");
 }
 
 void
-octave_lexer::gripe_matlab_incompatible_continuation (void)
+octave_base_lexer::gripe_matlab_incompatible_continuation (void)
 {
   gripe_matlab_incompatible ("\\ used as line continuation marker");
 }
 
 void
-octave_lexer::gripe_matlab_incompatible_operator (const std::string& op)
+octave_base_lexer::gripe_matlab_incompatible_operator (const std::string& op)
 {
   std::string t = op;
   int n = t.length ();
@@ -2646,7 +2619,7 @@
 }
 
 void
-octave_lexer::push_token (token *tok)
+octave_base_lexer::push_token (token *tok)
 {
   YYSTYPE *lval = yyget_lval (scanner);
   lval->tok_val = tok;
@@ -2654,14 +2627,14 @@
 }
 
 token *
-octave_lexer::current_token (void)
+octave_base_lexer::current_token (void)
 {
   YYSTYPE *lval = yyget_lval (scanner);
   return lval->tok_val;
 }
 
 void
-octave_lexer::display_token (int tok)
+octave_base_lexer::display_token (int tok)
 {
   switch (tok)
     {
@@ -2797,7 +2770,7 @@
 }
 
 void
-octave_lexer::fatal_error (const char *msg)
+octave_base_lexer::fatal_error (const char *msg)
 {
   error (msg);
 
@@ -2807,7 +2780,7 @@
 }
 
 void
-octave_lexer::lexer_debug (const char *pattern)
+octave_base_lexer::lexer_debug (const char *pattern)
 {
   if (lexer_debug_flag)
     {
@@ -2821,7 +2794,7 @@
 }
 
 void
-octave_lexer::push_start_state (int state)
+octave_base_lexer::push_start_state (int state)
 {
   OCTAVE_YYG;
 
@@ -2831,7 +2804,7 @@
 }
 
 void
-octave_lexer::pop_start_state (void)
+octave_base_lexer::pop_start_state (void)
 {
   OCTAVE_YYG;
 
@@ -2841,7 +2814,7 @@
 }
 
 void
-octave_lexer::clear_start_state (void)
+octave_base_lexer::clear_start_state (void)
 {
   while (! start_state_stack.empty ())
     start_state_stack.pop ();
@@ -2850,7 +2823,7 @@
 }
 
 void
-octave_lexer::display_start_state (void) const
+octave_base_lexer::display_start_state (void) const
 {
   std::cerr << "S: ";
 
@@ -2887,7 +2860,7 @@
 }
 
 int
-octave_lexer::handle_op (const char *pattern, int tok, bool bos)
+octave_base_lexer::handle_op (const char *pattern, int tok, bool bos)
 {
   lexer_debug (pattern);
 
@@ -2895,7 +2868,8 @@
 }
 
 int
-octave_lexer::handle_incompatible_op (const char *pattern, int tok, bool bos)
+octave_base_lexer::handle_incompatible_op (const char *pattern, int tok,
+                                           bool bos)
 {
   lexer_debug (pattern);
 
@@ -2903,7 +2877,7 @@
 }
 
 bool
-octave_lexer::maybe_unput_comma_before_unary_op (int tok)
+octave_base_lexer::maybe_unput_comma_before_unary_op (int tok)
 {
   int prev_tok = previous_token_value ();
 
@@ -2926,21 +2900,21 @@
 }
 
 int
-octave_lexer::handle_unary_op (int tok, bool bos)
+octave_base_lexer::handle_unary_op (int tok, bool bos)
 {
   return maybe_unput_comma_before_unary_op (tok)
     ? -1 : handle_op_internal (tok, bos, true);
 }
 
 int
-octave_lexer::handle_incompatible_unary_op (int tok, bool bos)
+octave_base_lexer::handle_incompatible_unary_op (int tok, bool bos)
 {
   return maybe_unput_comma_before_unary_op (tok)
     ? -1 : handle_op_internal (tok, bos, false);
 }
 
 int
-octave_lexer::handle_op_internal (int tok, bool bos, bool compat)
+octave_base_lexer::handle_op_internal (int tok, bool bos, bool compat)
 {
   if (! compat)
     gripe_matlab_incompatible_operator (flex_yytext ());
@@ -2955,7 +2929,7 @@
 }
 
 int
-octave_lexer::handle_token (const std::string& name, int tok)
+octave_base_lexer::handle_token (const std::string& name, int tok)
 {
   token *tok_val = new token (tok, name, input_line_number,
                               current_input_column);
@@ -2964,7 +2938,7 @@
 }
 
 int
-octave_lexer::handle_token (int tok, token *tok_val)
+octave_base_lexer::handle_token (int tok, token *tok_val)
 {
   if (! tok_val)
     tok_val = new token (tok, input_line_number, current_input_column);
@@ -2977,7 +2951,7 @@
 }
 
 int
-octave_lexer::count_token (int tok)
+octave_base_lexer::count_token (int tok)
 {
   token *tok_val = new token (tok, input_line_number, current_input_column);
 
@@ -2987,7 +2961,7 @@
 }
 
 int
-octave_lexer::count_token_internal (int tok)
+octave_base_lexer::count_token_internal (int tok)
 {
   if (tok != '\n')
     {
@@ -2999,7 +2973,7 @@
 }
 
 int
-octave_lexer::show_token (int tok)
+octave_base_lexer::show_token (int tok)
 {
   if (Vdisplay_tokens)
     display_token (tok);
@@ -3013,3 +2987,28 @@
 
   return tok;
 }
+
+int
+octave_lexer::fill_flex_buffer (char *buf, unsigned max_size)
+{
+  int status = 0;
+
+  if (input_buf.empty ())
+    {
+      bool eof = false;
+      current_input_line = input_reader.get_input (eof);
+      input_buf.fill (current_input_line, eof);
+    }
+
+  if (! input_buf.empty ())
+    status = input_buf.copy_chunk (buf, max_size);
+  else
+    {
+      status = YY_NULL;
+
+      if (! input_buf.at_eof ())
+        fatal_error ("octave_base_lexer::fill_flex_buffer failed");
+    }
+
+  return status;
+}
--- a/libinterp/parse-tree/oct-parse.in.yy	Wed Mar 13 02:25:50 2013 -0400
+++ b/libinterp/parse-tree/oct-parse.in.yy	Wed Mar 13 02:46:56 2013 -0400
@@ -84,7 +84,7 @@
 
 // Global access to currently active lexer.
 // FIXME -- to be removed after more parser+lexer refactoring.
-octave_lexer *LEXER = 0;
+octave_base_lexer *LEXER = 0;
 
 #if defined (GNULIB_NAMESPACE)
 // Calls to the following functions appear in the generated output from
--- a/libinterp/parse-tree/parse.h	Wed Mar 13 02:25:50 2013 -0400
+++ b/libinterp/parse-tree/parse.h	Wed Mar 13 02:46:56 2013 -0400
@@ -123,14 +123,14 @@
 
 // Global access to currently active lexer.
 // FIXME -- to be removed after more parser+lexer refactoring.
-extern octave_lexer *LEXER;
+extern octave_base_lexer *LEXER;
 
 class
 octave_base_parser
 {
 public:
 
-  octave_base_parser (octave_lexer& lxr)
+  octave_base_parser (octave_base_lexer& lxr)
     : endfunction_found (false),
       autoloading (false), fcn_file_from_relative_lookup (false),
       parsing_subfunctions (false), max_fcn_depth (0),
@@ -142,7 +142,7 @@
     init ();
   }
 
-  virtual ~octave_base_parser (void);
+  ~octave_base_parser (void);
 
   void init (void);
 
@@ -381,7 +381,7 @@
   tree_statement_list *stmt_list;
 
   // State of the lexer.
-  octave_lexer& lexer;
+  octave_base_lexer& lexer;
 
   // Internal state of the parser.  Only used if USE_PUSH_PARSER is
   // defined.