Mercurial > octave-nkf

comparison libinterp/parse-tree/lex.ll @ 20623:e34692daf663

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Extend parser to accept '_' in numbers. * NEWS: Announce change. * lex.ll: Define NUMBER to be a NUMREAL (real number) or NUMHEX (hex number). Define NUMREAL to begin with a digit (D) followed by more digits or an '_'. Define NUMHEX to begin with 0[xX], a hex digit (a-fA-F0-9), followed by '_' or more hex digits. Define EXPON to have a digit (D) followed by more digits or an '_'. * lex.ll (octave_base_lexer::handle_number): Strip out any underscores before processing number with sscanf. * parser.tst: Add tests for new behavior.
author Rik <rik@octave.org>
date Mon, 05 Oct 2015 14:05:58 -0700
parents 52ce821a52fd
children 7c0e10f035bd
comparison
equal deleted inserted replaced
20622:128414587af2 20623:e34692daf663
318 static bool lexer_debug_flag = false; 318 static bool lexer_debug_flag = false;
319 319
320 %} 320 %}
321 321
322 D [0-9] 322 D [0-9]
323 D_ [0-9_]
323 S [ \t] 324 S [ \t]
324 NL ((\n)|(\r)|(\r\n)) 325 NL ((\n)|(\r)|(\r\n))
325 Im [iIjJ] 326 Im [iIjJ]
326 CCHAR [#%] 327 CCHAR [#%]
327 IDENT ([_$a-zA-Z][_$a-zA-Z0-9]*) 328 IDENT ([_$a-zA-Z][_$a-zA-Z0-9]*)
328 FQIDENT ({IDENT}(\.{IDENT})*) 329 FQIDENT ({IDENT}(\.{IDENT})*)
329 EXPON ([DdEe][+-]?{D}+) 330 EXPON ([DdEe][+-]?{D}{D_}*)
330 NUMBER (({D}+\.?{D}*{EXPON}?)|(\.{D}+{EXPON}?)|(0[xX][0-9a-fA-F]+)) 331 NUMREAL (({D}{D_}*\.?{D_}*{EXPON}?)|(\.{D}{D_}*{EXPON}?))
332 NUMHEX (0[xX][0-9a-fA-F][0-9a-fA-F_]*)
333 NUMBER ({NUMREAL}|{NUMHEX})
331 334
332 ANY_EXCEPT_NL [^\r\n] 335 ANY_EXCEPT_NL [^\r\n]
333 ANY_INCLUDING_NL (.|{NL}) 336 ANY_INCLUDING_NL (.|{NL})
334 337
335 %% 338 %%
1122 %{ 1125 %{
1123 // Real numbers. Don't grab the '.' part of a dot operator as part of 1126 // Real numbers. Don't grab the '.' part of a dot operator as part of
1124 // the constant. 1127 // the constant.
1125 %} 1128 %}
1126 1129
1127 {D}+/\.[\*/\\^\'] | 1130 {D}{D_}*/\.[\*/\\^\'] |
1128 {NUMBER} { 1131 {NUMBER} {
1129 curr_lexer->lexer_debug ("{D}+/\\.[\\*/\\\\^\\']|{NUMBER}"); 1132 curr_lexer->lexer_debug ("{D}{D_}*/\\.[\\*/\\\\^\\']|{NUMBER}");
1130 1133
1131 if (curr_lexer->previous_token_may_be_command () 1134 if (curr_lexer->previous_token_may_be_command ()
1132 && curr_lexer->space_follows_previous_token ()) 1135 && curr_lexer->space_follows_previous_token ())
1133 { 1136 {
1134 yyless (0); 1137 yyless (0);
2666 double value = 0.0; 2669 double value = 0.0;
2667 int nread = 0; 2670 int nread = 0;
2668 2671
2669 char *yytxt = flex_yytext (); 2672 char *yytxt = flex_yytext ();
2670 2673
2671 if (looks_like_hex (yytxt, strlen (yytxt))) 2674 // Strip any underscores
2675 char *tmptxt = strsave (yytxt);
2676 char *rptr = tmptxt;
2677 char *wptr = tmptxt;
2678 while (*rptr)
2679 {
2680 *wptr = *rptr++;
2681 wptr += (*wptr != '_');
2682 }
2683 *wptr = '\0';
2684
2685 if (looks_like_hex (tmptxt, strlen (tmptxt)))
2672 { 2686 {
2673 unsigned long ival; 2687 unsigned long ival;
2674 2688
2675 nread = sscanf (yytxt, "%lx", &ival); 2689 nread = sscanf (tmptxt, "%lx", &ival);
2676 2690
2677 value = static_cast<double> (ival); 2691 value = static_cast<double> (ival);
2678 } 2692 }
2679 else 2693 else
2680 { 2694 {
2681 char *tmp = strsave (yytxt); 2695 char *idx = strpbrk (tmptxt, "Dd");
2682
2683 char *idx = strpbrk (tmp, "Dd");
2684 2696
2685 if (idx) 2697 if (idx)
2686 *idx = 'e'; 2698 *idx = 'e';
2687 2699
2688 nread = sscanf (tmp, "%lf", &value); 2700 nread = sscanf (tmptxt, "%lf", &value);
2689
2690 delete [] tmp;
2691 } 2701 }
2702
2703 delete [] tmptxt;
2692 2704
2693 // If yytext doesn't contain a valid number, we are in deep doo doo. 2705 // If yytext doesn't contain a valid number, we are in deep doo doo.
2694 2706
2695 assert (nread == 1); 2707 assert (nread == 1);
2696 2708