Mercurial > octave

comparison src/lex.l @ 7722:c3bb0b7a4261

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lex.l: allow tokens to be displayed when parsed
author John W. Eaton <jwe@octave.org>
date Fri, 18 Apr 2008 13:29:50 -0400
parents 4e2eafef689c
children 74f5e0c7de9e
comparison
equal deleted inserted replaced
7721:9369589f2ba5 7722:c3bb0b7a4261
109 OCTAVE_QUIT; \ 109 OCTAVE_QUIT; \
110 yy_fatal_error (msg); \ 110 yy_fatal_error (msg); \
111 } \ 111 } \
112 while (0) 112 while (0)
113 113
114 #define DISPLAY_TOK_AND_RETURN(tok) \
115 do \
116 { \
117 int tok_val = tok; \
118 if (Vdisplay_tokens) \
119 display_token (tok_val); \
120 return tok_val; \
121 } \
122 while (0)
123
114 #define COUNT_TOK_AND_RETURN(tok) \ 124 #define COUNT_TOK_AND_RETURN(tok) \
115 do \ 125 do \
116 { \ 126 { \
117 Vtoken_count++; \ 127 Vtoken_count++; \
118 return tok; \ 128 DISPLAY_TOK_AND_RETURN (tok); \
119 } \ 129 } \
120 while (0) 130 while (0)
121 131
122 #define TOK_RETURN(tok) \ 132 #define TOK_RETURN(tok) \
123 do \ 133 do \
231 const int bracket_brace_paren_nesting_level::BRACKET = 1; 241 const int bracket_brace_paren_nesting_level::BRACKET = 1;
232 const int bracket_brace_paren_nesting_level::BRACE = 2; 242 const int bracket_brace_paren_nesting_level::BRACE = 2;
233 const int bracket_brace_paren_nesting_level::PAREN = 3; 243 const int bracket_brace_paren_nesting_level::PAREN = 3;
234 244
235 static bracket_brace_paren_nesting_level nesting_level; 245 static bracket_brace_paren_nesting_level nesting_level;
246
247 static bool Vdisplay_tokens = false;
236 248
237 static unsigned int Vtoken_count = 0; 249 static unsigned int Vtoken_count = 0;
238 250
239 // Forward declarations for functions defined at the bottom of this 251 // Forward declarations for functions defined at the bottom of this
240 // file. 252 // file.
263 static void gripe_single_quote_string (void); 275 static void gripe_single_quote_string (void);
264 static void gripe_matlab_incompatible (const std::string& msg); 276 static void gripe_matlab_incompatible (const std::string& msg);
265 static void maybe_gripe_matlab_incompatible_comment (char c); 277 static void maybe_gripe_matlab_incompatible_comment (char c);
266 static void gripe_matlab_incompatible_continuation (void); 278 static void gripe_matlab_incompatible_continuation (void);
267 static void gripe_matlab_incompatible_operator (const std::string& op); 279 static void gripe_matlab_incompatible_operator (const std::string& op);
280 static void display_token (int tok);
268 281
269 %} 282 %}
270 283
271 D [0-9] 284 D [0-9]
272 S [ \t] 285 S [ \t]
333 TOK_RETURN (';'); 346 TOK_RETURN (';');
334 } 347 }
335 348
336 <COMMAND_START>[\"\'] { 349 <COMMAND_START>[\"\'] {
337 current_input_column++; 350 current_input_column++;
338 COUNT_TOK_AND_RETURN (handle_string (yytext[0], true)); 351 int tok = handle_string (yytext[0], true);
352 COUNT_TOK_AND_RETURN (tok);
339 } 353 }
340 354
341 <COMMAND_START>[^#% \t\r\n\;\,\"\'][^ \t\r\n\;\,]*{S}* { 355 <COMMAND_START>[^#% \t\r\n\;\,\"\'][^ \t\r\n\;\,]*{S}* {
342 std::string tok = strip_trailing_whitespace (yytext); 356 std::string tok = strip_trailing_whitespace (yytext);
343 TOK_PUSH_AND_RETURN (tok, SQ_STRING); 357 TOK_PUSH_AND_RETURN (tok, SQ_STRING);
609 { 623 {
610 do_comma_insert_check (); 624 do_comma_insert_check ();
611 COUNT_TOK_AND_RETURN (QUOTE); 625 COUNT_TOK_AND_RETURN (QUOTE);
612 } 626 }
613 else 627 else
614 COUNT_TOK_AND_RETURN (handle_string ('\'')); 628 {
629 int tok = handle_string ('\'');
630 COUNT_TOK_AND_RETURN (tok);
631 }
615 } 632 }
616 633
617 %{ 634 %{
618 // Double quotes always begin strings. 635 // Double quotes always begin strings.
619 %} 636 %}
620 637
621 \" { 638 \" {
622 current_input_column++; 639 current_input_column++;
623 COUNT_TOK_AND_RETURN (handle_string ('"')); 640 int tok = handle_string ('"');
641 COUNT_TOK_AND_RETURN (tok);
624 } 642 }
625 643
626 %{ 644 %{
627 // Gobble comments. If closest nesting is inside parentheses, don't 645 // Gobble comments. If closest nesting is inside parentheses, don't
628 // return a new line. 646 // return a new line.
2258 error ("function handles may not refer to keywords"); 2276 error ("function handles may not refer to keywords");
2259 2277
2260 return LEXICAL_ERROR; 2278 return LEXICAL_ERROR;
2261 } 2279 }
2262 else 2280 else
2263 TOK_PUSH_AND_RETURN (tok, FCN_HANDLE); 2281 {
2282 yylval.tok_val = new token (tok, input_line_number,
2283 current_input_column);
2284
2285 token_stack.push (yylval.tok_val);
2286
2287 current_input_column += yyleng;
2288 lexer_flags.quote_is_transpose = false;
2289 lexer_flags.convert_spaces_to_comma = true;
2290
2291 return FCN_HANDLE;
2292 }
2264 } 2293 }
2265 2294
2266 // If we have a regular keyword, return it. 2295 // If we have a regular keyword, return it.
2267 // Keywords can be followed by identifiers (TOK_RETURN handles 2296 // Keywords can be followed by identifiers.
2268 // that).
2269 2297
2270 if (kw_token) 2298 if (kw_token)
2271 { 2299 {
2272 if (kw_token < 0) 2300 if (kw_token >= 0)
2273 return kw_token; 2301 {
2274 else 2302 current_input_column += yyleng;
2275 TOK_RETURN (kw_token); 2303 lexer_flags.quote_is_transpose = false;
2304 lexer_flags.convert_spaces_to_comma = true;
2305 }
2306
2307 return kw_token;
2276 } 2308 }
2277 2309
2278 // See if we have a plot keyword (title, using, with, or clear). 2310 // See if we have a plot keyword (title, using, with, or clear).
2279 2311
2280 int c1 = yyinput (); 2312 int c1 = yyinput ();
2511 if (t[n-1] == '\n') 2543 if (t[n-1] == '\n')
2512 t.resize (n-1); 2544 t.resize (n-1);
2513 gripe_matlab_incompatible (t + " used as operator"); 2545 gripe_matlab_incompatible (t + " used as operator");
2514 } 2546 }
2515 2547
2548 static void
2549 display_token (int tok)
2550 {
2551 switch (tok)
2552 {
2553 case '=': std::cerr << "'='\n"; break;
2554 case ':': std::cerr << "':'\n"; break;
2555 case '-': std::cerr << "'-'\n"; break;
2556 case '+': std::cerr << "'+'\n"; break;
2557 case '*': std::cerr << "'*'\n"; break;
2558 case '/': std::cerr << "'/'\n"; break;
2559 case ADD_EQ: std::cerr << "ADD_EQ\n"; break;
2560 case SUB_EQ: std::cerr << "SUB_EQ\n"; break;
2561 case MUL_EQ: std::cerr << "MUL_EQ\n"; break;
2562 case DIV_EQ: std::cerr << "DIV_EQ\n"; break;
2563 case LEFTDIV_EQ: std::cerr << "LEFTDIV_EQ\n"; break;
2564 case POW_EQ: std::cerr << "POW_EQ\n"; break;
2565 case EMUL_EQ: std::cerr << "EMUL_EQ\n"; break;
2566 case EDIV_EQ: std::cerr << "EDIV_EQ\n"; break;
2567 case ELEFTDIV_EQ: std::cerr << "ELEFTDIV_EQ\n"; break;
2568 case EPOW_EQ: std::cerr << "EPOW_EQ\n"; break;
2569 case AND_EQ: std::cerr << "AND_EQ\n"; break;
2570 case OR_EQ: std::cerr << "OR_EQ\n"; break;
2571 case LSHIFT_EQ: std::cerr << "LSHIFT_EQ\n"; break;
2572 case RSHIFT_EQ: std::cerr << "RSHIFT_EQ\n"; break;
2573 case LSHIFT: std::cerr << "LSHIFT\n"; break;
2574 case RSHIFT: std::cerr << "RSHIFT\n"; break;
2575 case EXPR_AND_AND: std::cerr << "EXPR_AND_AND\n"; break;
2576 case EXPR_OR_OR: std::cerr << "EXPR_OR_OR\n"; break;
2577 case EXPR_AND: std::cerr << "EXPR_AND\n"; break;
2578 case EXPR_OR: std::cerr << "EXPR_OR\n"; break;
2579 case EXPR_NOT: std::cerr << "EXPR_NOT\n"; break;
2580 case EXPR_LT: std::cerr << "EXPR_LT\n"; break;
2581 case EXPR_LE: std::cerr << "EXPR_LE\n"; break;
2582 case EXPR_EQ: std::cerr << "EXPR_EQ\n"; break;
2583 case EXPR_NE: std::cerr << "EXPR_NE\n"; break;
2584 case EXPR_GE: std::cerr << "EXPR_GE\n"; break;
2585 case EXPR_GT: std::cerr << "EXPR_GT\n"; break;
2586 case LEFTDIV: std::cerr << "LEFTDIV\n"; break;
2587 case EMUL: std::cerr << "EMUL\n"; break;
2588 case EDIV: std::cerr << "EDIV\n"; break;
2589 case ELEFTDIV: std::cerr << "ELEFTDIV\n"; break;
2590 case EPLUS: std::cerr << "EPLUS\n"; break;
2591 case EMINUS: std::cerr << "EMINUS\n"; break;
2592 case QUOTE: std::cerr << "QUOTE\n"; break;
2593 case TRANSPOSE: std::cerr << "TRANSPOSE\n"; break;
2594 case PLUS_PLUS: std::cerr << "PLUS_PLUS\n"; break;
2595 case MINUS_MINUS: std::cerr << "MINUS_MINUS\n"; break;
2596 case POW: std::cerr << "POW\n"; break;
2597 case EPOW: std::cerr << "EPOW\n"; break;
2598 case NUM: std::cerr << "NUM\n"; break;
2599 case IMAG_NUM: std::cerr << "IMAG_NUM\n"; break;
2600 case STRUCT_ELT: std::cerr << "STRUCT_ELT\n"; break;
2601 case NAME: std::cerr << "NAME\n"; break;
2602 case END: std::cerr << "END\n"; break;
2603 case DQ_STRING: std::cerr << "DQ_STRING\n"; break;
2604 case SQ_STRING: std::cerr << "SQ_STRING\n"; break;
2605 case FOR: std::cerr << "FOR\n"; break;
2606 case WHILE: std::cerr << "WHILE\n"; break;
2607 case DO: std::cerr << "DO\n"; break;
2608 case UNTIL: std::cerr << "UNTIL\n"; break;
2609 case IF: std::cerr << "IF\n"; break;
2610 case ELSEIF: std::cerr << "ELSEIF\n"; break;
2611 case ELSE: std::cerr << "ELSE\n"; break;
2612 case SWITCH: std::cerr << "SWITCH\n"; break;
2613 case CASE: std::cerr << "CASE\n"; break;
2614 case OTHERWISE: std::cerr << "OTHERWISE\n"; break;
2615 case BREAK: std::cerr << "BREAK\n"; break;
2616 case CONTINUE: std::cerr << "CONTINUE\n"; break;
2617 case FUNC_RET: std::cerr << "FUNC_RET\n"; break;
2618 case UNWIND: std::cerr << "UNWIND\n"; break;
2619 case CLEANUP: std::cerr << "CLEANUP\n"; break;
2620 case TRY: std::cerr << "TRY\n"; break;
2621 case CATCH: std::cerr << "CATCH\n"; break;
2622 case GLOBAL: std::cerr << "GLOBAL\n"; break;
2623 case STATIC: std::cerr << "STATIC\n"; break;
2624 case FCN_HANDLE: std::cerr << "FCN_HANDLE\n"; break;
2625 case END_OF_INPUT: std::cerr << "END_OF_INPUT\n\n"; break;
2626 case LEXICAL_ERROR: std::cerr << "LEXICAL_ERROR\n\n"; break;
2627 case FCN: std::cerr << "FCN\n"; break;
2628 case CLOSE_BRACE: std::cerr << "CLOSE_BRACE\n"; break;
2629 case '\n': std::cerr << "\\n\n"; break;
2630 case '\r': std::cerr << "\\r\n"; break;
2631 case '\t': std::cerr << "TAB\n"; break;
2632 default:
2633 {
2634 if (tok < 256)
2635 std::cerr << static_cast<char> (tok) << "\n";
2636 else
2637 std::cerr << "UNKNOWN(" << tok << ")\n";
2638 }
2639 break;
2640 }
2641 }
2642
2643 DEFUN (__display_tokens__, args, nargout,
2644 "-*- texinfo -*-\n\
2645 @deftypefn {Built-in Function} {} __display_tokens__\n\
2646 Query or set the internal variable that determines whether Octave's\n\
2647 lexer displays tokens as they are read.\n\
2648 @end deftypefn")
2649 {
2650 return SET_INTERNAL_VARIABLE (display_tokens);
2651 }
2652
2516 DEFUN (__token_count__, , , 2653 DEFUN (__token_count__, , ,
2517 "-*- texinfo -*-\n\ 2654 "-*- texinfo -*-\n\
2518 @deftypefn {Built-in Function} {} __token_count__\n\ 2655 @deftypefn {Built-in Function} {} __token_count__\n\
2519 Number of language tokens processed since Octave startup.\n\ 2656 Number of language tokens processed since Octave startup.\n\
2520 @end deftypefn") 2657 @end deftypefn")