diff libinterp/parse-tree/lex.ll @ 16273:c5e5f6ccac5d

9/10 commits reworking the lexer
author John W. Eaton <jwe@octave.org>
date Mon, 11 Mar 2013 14:32:53 -0400
parents 87069bd38107
children 3c265e4dba6d dd7de0345124
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.ll	Mon Mar 11 14:32:15 2013 -0400
+++ b/libinterp/parse-tree/lex.ll	Mon Mar 11 14:32:53 2013 -0400
@@ -51,8 +51,6 @@
 %x BLOCK_COMMENT_START
 %x LINE_COMMENT_START
 
-%x KLUGE
-
 %{
 
 #include <cctype>
@@ -253,12 +251,6 @@
       }
   }
 
-<KLUGE>@ {
-    curr_lexer->lexer_debug ("<KLUGE>@");
-    curr_lexer->pop_start_state ();
-    return curr_lexer->count_token (CHOOSE_ASSIGNMENT);
-  }
-
 %{
 // For this and the next two rules, we're looking at ']', and we
 // need to know if the next token is '=' or '=='.
@@ -802,24 +794,24 @@
 // Other operators.
 %}
 
-":"     { return curr_lexer->handle_op (":", ':'); }
-".+"    { return curr_lexer->handle_incompatible_op (".+", EPLUS); }
-".-"    { return curr_lexer->handle_incompatible_op (".-", EMINUS); }
-".*"    { return curr_lexer->handle_op (".*", EMUL); }
-"./"    { return curr_lexer->handle_op ("./", EDIV); }
-".\\"   { return curr_lexer->handle_op (".\\", ELEFTDIV); }
-".^"    { return curr_lexer->handle_op (".^", EPOW); }
-".**"   { return curr_lexer->handle_incompatible_op (".**", EPOW); }
-"<="    { return curr_lexer->handle_op ("<=", EXPR_LE); }
-"=="    { return curr_lexer->handle_op ("==", EXPR_EQ); }
-"~="    { return curr_lexer->handle_op ("~=", EXPR_NE); }
-"!="    { return curr_lexer->handle_incompatible_op ("!=", EXPR_NE); }
-">="    { return curr_lexer->handle_op (">=", EXPR_GE); }
-"&"     { return curr_lexer->handle_op ("&", EXPR_AND); }
-"|"     { return curr_lexer->handle_op ("|", EXPR_OR); }
-"<"     { return curr_lexer->handle_op ("<", EXPR_LT); }
-">"     { return curr_lexer->handle_op (">", EXPR_GT); }
-"*"     { return curr_lexer->handle_op ("*", '*'); }
+":"   { return curr_lexer->handle_op (":", ':'); }
+".+"  { return curr_lexer->handle_incompatible_op (".+", EPLUS); }
+".-"  { return curr_lexer->handle_incompatible_op (".-", EMINUS); }
+".*"  { return curr_lexer->handle_op (".*", EMUL); }
+"./"  { return curr_lexer->handle_op ("./", EDIV); }
+".\\" { return curr_lexer->handle_op (".\\", ELEFTDIV); }
+".^"  { return curr_lexer->handle_op (".^", EPOW); }
+".**" { return curr_lexer->handle_incompatible_op (".**", EPOW); }
+"<="  { return curr_lexer->handle_op ("<=", EXPR_LE); }
+"=="  { return curr_lexer->handle_op ("==", EXPR_EQ); }
+"~="  { return curr_lexer->handle_op ("~=", EXPR_NE); }
+"!="  { return curr_lexer->handle_incompatible_op ("!=", EXPR_NE); }
+">="  { return curr_lexer->handle_op (">=", EXPR_GE); }
+"&"   { return curr_lexer->handle_op ("&", EXPR_AND); }
+"|"   { return curr_lexer->handle_op ("|", EXPR_OR); }
+"<"   { return curr_lexer->handle_op ("<", EXPR_LT); }
+">"   { return curr_lexer->handle_op (">", EXPR_GT); }
+"*"   { return curr_lexer->handle_op ("*", '*'); }
 
 "/" {
     int prev_tok = curr_lexer->previous_token_value ();
@@ -838,13 +830,13 @@
       return curr_lexer->handle_op ("/", '/');
   }
 
-"\\"    { return curr_lexer->handle_op ("\\", LEFTDIV); }
-"^"     { return curr_lexer->handle_op ("^", POW); }
-"**"    { return curr_lexer->handle_incompatible_op ("**", POW); }
-"&&"    { return curr_lexer->handle_op ("&&", EXPR_AND_AND); }
-"||"    { return curr_lexer->handle_op ("||", EXPR_OR_OR); }
-"<<"    { return curr_lexer->handle_incompatible_op ("<<", LSHIFT); }
-">>"    { return curr_lexer->handle_incompatible_op (">>", RSHIFT); }
+"\\" { return curr_lexer->handle_op ("\\", LEFTDIV); }
+"^"  { return curr_lexer->handle_op ("^", POW); }
+"**" { return curr_lexer->handle_incompatible_op ("**", POW); }
+"&&" { return curr_lexer->handle_op ("&&", EXPR_AND_AND); }
+"||" { return curr_lexer->handle_op ("||", EXPR_OR_OR); }
+"<<" { return curr_lexer->handle_incompatible_op ("<<", LSHIFT); }
+">>" { return curr_lexer->handle_incompatible_op (">>", RSHIFT); }
 
 ";" {
     bool at_beginning_of_statement
@@ -1030,233 +1022,25 @@
 // = and op= operators.
 %}
 
-"=" {
-    int tok = curr_lexer->handle_assign_op ("=", '=');
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"+=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("+=", ADD_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"-=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("-=", SUB_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"*=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("*=", MUL_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"/=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("/=", DIV_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"\\=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("\\=", LEFTDIV_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-".+=" {
-    int tok = curr_lexer->handle_incompatible_assign_op (".+=", ADD_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-".-=" {
-    int tok = curr_lexer->handle_incompatible_assign_op (".-=", SUB_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-".*=" {
-    int tok = curr_lexer->handle_incompatible_assign_op (".*=", EMUL_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"./=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("./=", EDIV_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-".\\=" {
-    int tok = curr_lexer->handle_incompatible_assign_op (".\\=", ELEFTDIV_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"^=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("^=", POW_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"**=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("^=", POW_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-".^=" {
-    int tok = curr_lexer->handle_incompatible_assign_op (".^=", EPOW_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-".**=" {
-    int tok = curr_lexer->handle_incompatible_assign_op (".^=", EPOW_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"&=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("&=", AND_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"|=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("|=", OR_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-"<<=" {
-    int tok = curr_lexer->handle_incompatible_assign_op ("<<=", LSHIFT_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
-
-">>=" {
-    int tok = curr_lexer->handle_incompatible_assign_op (">>=", RSHIFT_EQ);
-    if (tok < 0)
-      {
-        yyless (0);
-        curr_lexer->xunput ('@');
-        curr_lexer->push_start_state (KLUGE);
-      }
-    else
-      return tok;
-  }
+"="    { return curr_lexer->handle_op ("=", '='); }
+"+="   { return curr_lexer->handle_incompatible_op ("+=", ADD_EQ); }
+"-="   { return curr_lexer->handle_incompatible_op ("-=", SUB_EQ); }
+"*="   { return curr_lexer->handle_incompatible_op ("*=", MUL_EQ); }
+"/="   { return curr_lexer->handle_incompatible_op ("/=", DIV_EQ); }
+"\\="  { return curr_lexer->handle_incompatible_op ("\\=", LEFTDIV_EQ); }
+".+="  { return curr_lexer->handle_incompatible_op (".+=", ADD_EQ); }
+".-="  { return curr_lexer->handle_incompatible_op (".-=", SUB_EQ); }
+".*="  { return curr_lexer->handle_incompatible_op (".*=", EMUL_EQ); }
+"./="  { return curr_lexer->handle_incompatible_op ("./=", EDIV_EQ); }
+".\\=" { return curr_lexer->handle_incompatible_op (".\\=", ELEFTDIV_EQ); }
+"^="   { return curr_lexer->handle_incompatible_op ("^=", POW_EQ); }
+"**="  { return curr_lexer->handle_incompatible_op ("^=", POW_EQ); }
+".^="  { return curr_lexer->handle_incompatible_op (".^=", EPOW_EQ); }
+".**=" { return curr_lexer->handle_incompatible_op (".^=", EPOW_EQ); }
+"&="   { return curr_lexer->handle_incompatible_op ("&=", AND_EQ); }
+"|="   { return curr_lexer->handle_incompatible_op ("|=", OR_EQ); }
+"<<="  { return curr_lexer->handle_incompatible_op ("<<=", LSHIFT_EQ); }
+">>="  { return curr_lexer->handle_incompatible_op (">>=", RSHIFT_EQ); }
 
 "{" {
     curr_lexer->lexer_debug ("{");
@@ -3031,48 +2815,6 @@
       return kw_token;
     }
 
-  // See if we have a plot keyword (title, using, with, or clear).
-
-  int c1 = text_yyinput ();
-
-  bool next_tok_is_eq = false;
-  if (c1 == '=')
-    {
-      int c2 = text_yyinput ();
-      xunput (c2);
-
-      if (c2 != '=')
-        next_tok_is_eq = true;
-    }
-
-  xunput (c1);
-
-  // Kluge alert.
-  //
-  // If we are looking at a text style function, set up to gobble its
-  // arguments.
-  //
-  // If the following token is '=', or if we are parsing a function
-  // return list or function parameter list, or if we are looking at
-  // something like [ab,cd] = foo (), force the symbol to be inserted
-  // as a variable in the current symbol table.
-
-  if (! is_variable (tok))
-    {
-      if (next_tok_is_eq
-          || looking_at_decl_list
-          || looking_at_return_list
-          || (looking_at_parameter_list
-              && ! looking_at_initializer_expression))
-        {
-          symbol_table::force_variable (tok);
-        }
-      else if (looking_at_matrix_or_assign_lhs)
-        {
-          pending_local_variables.insert (tok);
-        }
-    }
-
   // Find the token in the symbol table.  Beware the magic
   // transformation of the end keyword...
 
@@ -3298,7 +3040,6 @@
     case END_OF_INPUT: std::cerr << "END_OF_INPUT\n\n"; break;
     case LEXICAL_ERROR: std::cerr << "LEXICAL_ERROR\n\n"; break;
     case FCN: std::cerr << "FCN\n"; break;
-    case CHOOSE_ASSIGNMENT: std::cerr << "CHOOSE_ASSIGNMENT\n"; break;
     case INPUT_FILE: std::cerr << "INPUT_FILE\n"; break;
     case SUPERCLASSREF: std::cerr << "SUPERCLASSREF\n"; break;
     case METAQUERY: std::cerr << "METAQUERY\n"; break;
@@ -3406,10 +3147,6 @@
       std::cerr << "LINE_COMMENT_START" << std::endl;
       break;
 
-    case KLUGE:
-      std::cerr << "KLUGE" << std::endl;
-      break;
-
     default:
       std::cerr << "UNKNOWN START STATE!" << std::endl;
       break;
@@ -3478,24 +3215,6 @@
 }
 
 int
-octave_lexer::handle_assign_op (const char *pattern, int tok)
-{
-  lexer_debug (pattern);
-
-  return (previous_token_value_is (']') && looking_at_matrix_or_assign_lhs)
-    ? -1 : handle_op_internal (pattern, tok, false, false, false, true);
-}
-
-int
-octave_lexer::handle_incompatible_assign_op (const char *pattern, int tok)
-{
-  lexer_debug (pattern);
-
-  return (previous_token_value_is (']') && looking_at_matrix_or_assign_lhs)
-    ? -1 : handle_op_internal (pattern, tok, false, false, false, false);
-}
-
-int
 octave_lexer::handle_op_internal (const char *pattern, int tok, bool convert,
                                   bool bos, bool qit, bool compat)
 {