Mercurial > octave-nkf

comparison scripts/testfun/test.m @ 19478:3f29b433bd5d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
do a better job of preserving warning state when running tests * error.cc (vwarning): Always set Vlast_warning and Vlast_warning_message. * __run_test_suite__.m: Restore warning state to initial value before exiting. * test.m: Restore warning state to initial value after each test block.
author John W. Eaton <jwe@octave.org>
date Tue, 30 Dec 2014 11:33:33 -0500
parents 5cd83b466a3e
children 0e1f5a750d00
comparison
equal deleted inserted replaced
19477:0f79fa9b3a8c 19478:3f29b433bd5d
299 __shared = " "; 299 __shared = " ";
300 __shared_r = " "; 300 __shared_r = " ";
301 __clearfcn = ""; 301 __clearfcn = "";
302 for __i = 1:numel (__blockidx)-1 302 for __i = 1:numel (__blockidx)-1
303 303
304 ## Extract the block. 304 ## FIXME: Should other global settings be similarly saved and restored?
305 __block = __body(__blockidx(__i):__blockidx(__i+1)-2); 305 orig_wstate = warning ();
306 306 unwind_protect
307 ## Print the code block before execution if in verbose mode. 307
308 if (__verbose > 0) 308 ## Extract the block.
309 fprintf (__fid, "%s%s\n", __signal_block, __block); 309 __block = __body(__blockidx(__i):__blockidx(__i+1)-2);
310 fflush (__fid); 310
311 endif 311 ## Print the code block before execution if in verbose mode.
312 312 if (__verbose > 0)
313 ## Split __block into __type and __code. 313 fprintf (__fid, "%s%s\n", __signal_block, __block);
314 __idx = find (! isletter (__block)); 314 fflush (__fid);
315 if (isempty (__idx)) 315 endif
316 __type = __block; 316
317 __code = ""; 317 ## Split __block into __type and __code.
318 else 318 __idx = find (! isletter (__block));
319 __type = __block(1:__idx(1)-1);
320 __code = __block(__idx(1):length (__block));
321 endif
322
323 ## Assume the block will succeed.
324 __success = true;
325 __msg = [];
326 __isxtest = false;
327
328 ### DEMO
329
330 ## If in __grabdemo mode, then don't process any other block type.
331 ## So that the other block types don't have to worry about
332 ## this __grabdemo mode, the demo block processor grabs all block
333 ## types and skips those which aren't demo blocks.
334
335 __isdemo = strcmp (__type, "demo");
336 if (__grabdemo || __isdemo)
337 __istest = false;
338
339 if (__grabdemo && __isdemo)
340 if (isempty (__demo_code))
341 __demo_code = __code;
342 __demo_idx = [1, length(__demo_code)+1];
343 else
344 __demo_code = [__demo_code, __code];
345 __demo_idx = [__demo_idx, length(__demo_code)+1];
346 endif
347
348 elseif (__rundemo && __isdemo)
349 try
350 ## process the code in an environment without variables
351 eval (sprintf ("function __test__ ()\n%s\nendfunction", __code));
352 __test__;
353 input ("Press <enter> to continue: ", "s");
354 catch
355 __success = false;
356 __msg = [__signal_fail "demo failed\n" lasterr()];
357 end_try_catch
358 clear __test__;
359
360 endif
361 ## Code already processed.
362 __code = "";
363
364 ### SHARED
365
366 elseif (strcmp (__type, "shared"))
367 __istest = false;
368
369 ## Separate initialization code from variables.
370 __idx = find (__code == "\n");
371 if (isempty (__idx)) 319 if (isempty (__idx))
372 __vars = __code; 320 __type = __block;
373 __code = ""; 321 __code = "";
374 else 322 else
375 __vars = __code (1:__idx(1)-1); 323 __type = __block(1:__idx(1)-1);
376 __code = __code (__idx(1):length (__code)); 324 __code = __block(__idx(1):length (__block));
377 endif 325 endif
378 326
379 ## Strip comments off the variables. 327 ## Assume the block will succeed.
380 __idx = find (__vars == "%" | __vars == "#"); 328 __success = true;
381 if (! isempty (__idx)) 329 __msg = [];
382 __vars = __vars(1:__idx(1)-1); 330 __isxtest = false;
383 endif 331
384 332 ### DEMO
385 ## Assign default values to variables. 333
386 try 334 ## If in __grabdemo mode, then don't process any other block type.
387 __vars = deblank (__vars); 335 ## So that the other block types don't have to worry about
388 if (! isempty (__vars)) 336 ## this __grabdemo mode, the demo block processor grabs all block
389 eval ([strrep(__vars, ",", "=[];"), "=[];"]); 337 ## types and skips those which aren't demo blocks.
390 __shared = __vars; 338
391 __shared_r = ["[ " __vars "] = "]; 339 __isdemo = strcmp (__type, "demo");
340 if (__grabdemo || __isdemo)
341 __istest = false;
342
343 if (__grabdemo && __isdemo)
344 if (isempty (__demo_code))
345 __demo_code = __code;
346 __demo_idx = [1, length(__demo_code)+1];
347 else
348 __demo_code = [__demo_code, __code];
349 __demo_idx = [__demo_idx, length(__demo_code)+1];
350 endif
351
352 elseif (__rundemo && __isdemo)
353 try
354 ## process the code in an environment without variables
355 eval (sprintf ("function __test__ ()\n%s\nendfunction", __code));
356 __test__;
357 input ("Press <enter> to continue: ", "s");
358 catch
359 __success = false;
360 __msg = [__signal_fail "demo failed\n" lasterr()];
361 end_try_catch
362 clear __test__;
363
364 endif
365 ## Code already processed.
366 __code = "";
367
368 ### SHARED
369
370 elseif (strcmp (__type, "shared"))
371 __istest = false;
372
373 ## Separate initialization code from variables.
374 __idx = find (__code == "\n");
375 if (isempty (__idx))
376 __vars = __code;
377 __code = "";
392 else 378 else
393 __shared = " "; 379 __vars = __code (1:__idx(1)-1);
394 __shared_r = " "; 380 __code = __code (__idx(1):length (__code));
395 endif 381 endif
396 catch 382
397 ## Couldn't declare, so don't initialize. 383 ## Strip comments off the variables.
384 __idx = find (__vars == "%" | __vars == "#");
385 if (! isempty (__idx))
386 __vars = __vars(1:__idx(1)-1);
387 endif
388
389 ## Assign default values to variables.
390 try
391 __vars = deblank (__vars);
392 if (! isempty (__vars))
393 eval ([strrep(__vars, ",", "=[];"), "=[];"]);
394 __shared = __vars;
395 __shared_r = ["[ " __vars "] = "];
396 else
397 __shared = " ";
398 __shared_r = " ";
399 endif
400 catch
401 ## Couldn't declare, so don't initialize.
402 __code = "";
403 __success = false;
404 __msg = [__signal_fail "shared variable initialization failed\n"];
405 end_try_catch
406
407 ## Initialization code will be evaluated below.
408
409 ### FUNCTION
410
411 elseif (strcmp (__type, "function"))
412 __istest = false;
413 persistent __fn = 0;
414 __name_position = function_name (__block);
415 if (isempty (__name_position))
416 __success = false;
417 __msg = [__signal_fail "test failed: missing function name\n"];
418 else
419 __name = __block(__name_position(1):__name_position(2));
420 __code = __block;
421 try
422 eval (__code); # Define the function
423 __clearfcn = sprintf ("%sclear %s;\n", __clearfcn, __name);
424 catch
425 __success = false;
426 __msg = [__signal_fail "test failed: syntax error\n" lasterr()];
427 end_try_catch
428 endif
398 __code = ""; 429 __code = "";
399 __success = false; 430
400 __msg = [__signal_fail "shared variable initialization failed\n"]; 431 ### ENDFUNCTION
401 end_try_catch 432
402 433 elseif (strcmp (__type, "endfunction"))
403 ## Initialization code will be evaluated below. 434 ## endfunction simply declares the end of a previous function block.
404 435 ## There is no processing to be done here, just skip to next block.
405 ### FUNCTION 436 __istest = false;
406 437 __code = "";
407 elseif (strcmp (__type, "function")) 438
408 __istest = false; 439 ### ASSERT/FAIL
409 persistent __fn = 0; 440
410 __name_position = function_name (__block); 441 elseif (strcmp (__type, "assert") || strcmp (__type, "fail"))
411 if (isempty (__name_position)) 442 __istest = true;
412 __success = false; 443 ## Put the keyword back on the code.
413 __msg = [__signal_fail "test failed: missing function name\n"];
414 else
415 __name = __block(__name_position(1):__name_position(2));
416 __code = __block; 444 __code = __block;
445 ## The code will be evaluated below as a test block.
446
447 ### ERROR/WARNING
448
449 elseif (strcmp (__type, "error") || strcmp (__type, "warning"))
450 __istest = true;
451 __iswarning = strcmp (__type, "warning");
452 [__pattern, __id, __code] = getpattern (__code);
453 if (__id)
454 __patstr = ["id=" __id];
455 else
456 if (! strcmp (__pattern, '.'))
457 __patstr = ["<" __pattern ">"];
458 else
459 __patstr = ifelse (__iswarning, "a warning", "an error");
460 endif
461 endif
417 try 462 try
418 eval (__code); # Define the function 463 eval (sprintf ("function __test__(%s)\n%s\nendfunction",
419 __clearfcn = sprintf ("%sclear %s;\n", __clearfcn, __name); 464 __shared, __code));
420 catch 465 catch
421 __success = false; 466 __success = false;
422 __msg = [__signal_fail "test failed: syntax error\n" lasterr()]; 467 __msg = [__signal_fail "test failed: syntax error\n" lasterr()];
423 end_try_catch 468 end_try_catch
424 endif 469
425 __code = ""; 470 if (__success)
426 471 __success = false;
427 ### ENDFUNCTION 472 __warnstate = warning ("query", "quiet");
428 473 warning ("on", "quiet");
429 elseif (strcmp (__type, "endfunction")) 474 ## Clear error and warning strings before starting
430 ## endfunction simply declares the end of a previous function block. 475 lasterr ("");
431 ## There is no processing to be done here, just skip to next block. 476 lastwarn ("");
432 __istest = false; 477 try
433 __code = ""; 478 eval (sprintf ("__test__(%s);", __shared));
434 479 if (! __iswarning)
435 ### ASSERT/FAIL 480 __msg = [__signal_fail "error failed.\n" ...
436 481 "Expected " __patstr ", but got no error\n"];
437 elseif (strcmp (__type, "assert") || strcmp (__type, "fail")) 482 else
438 __istest = true; 483 if (! isempty (__id))
439 ## Put the keyword back on the code. 484 [~, __err] = lastwarn ();
440 __code = __block; 485 __mismatch = ! strcmp (__err, __id);
441 ## The code will be evaluated below as a test block. 486 else
442 487 __err = trimerr (lastwarn (), "warning");
443 ### ERROR/WARNING 488 __mismatch = isempty (regexp (__err, __pattern, "once"));
444 489 endif
445 elseif (strcmp (__type, "error") || strcmp (__type, "warning")) 490 warning (__warnstate.state, "quiet");
446 __istest = true; 491 if (isempty (__err))
447 __iswarning = strcmp (__type, "warning"); 492 __msg = [__signal_fail "warning failed.\n" ...
448 [__pattern, __id, __code] = getpattern (__code); 493 "Expected " __patstr ", but got no warning\n"];
449 if (__id) 494 elseif (__mismatch)
450 __patstr = ["id=" __id]; 495 __msg = [__signal_fail "warning failed.\n" ...
451 else 496 "Expected " __patstr ", but got <" __err ">\n"];
452 if (! strcmp (__pattern, '.')) 497 else
453 __patstr = ["<" __pattern ">"]; 498 __success = true;
454 else 499 endif
455 __patstr = ifelse (__iswarning, "a warning", "an error"); 500 endif
456 endif 501
457 endif 502 catch
458 try
459 eval (sprintf ("function __test__(%s)\n%s\nendfunction",
460 __shared, __code));
461 catch
462 __success = false;
463 __msg = [__signal_fail "test failed: syntax error\n" lasterr()];
464 end_try_catch
465
466 if (__success)
467 __success = false;
468 __warnstate = warning ("query", "quiet");
469 warning ("on", "quiet");
470 ## Clear error and warning strings before starting
471 lasterr ("");
472 lastwarn ("");
473 try
474 ## FIXME: lastwarn () must be called once from *WITHIN* the try block
475 ## or subsequent warning/lastwarn statements may fail.
476 ## Likely this is something to do with the specialness of
477 ## the try block which is disabling normal errors.
478 lastwarn ();
479 eval (sprintf ("__test__(%s);", __shared));
480 if (! __iswarning)
481 __msg = [__signal_fail "error failed.\n" ...
482 "Expected " __patstr ", but got no error\n"];
483 else
484 if (! isempty (__id)) 503 if (! isempty (__id))
485 [~, __err] = lastwarn (); 504 [~, __err] = lasterr ();
486 __mismatch = ! strcmp (__err, __id); 505 __mismatch = ! strcmp (__err, __id);
487 else 506 else
488 __err = trimerr (lastwarn (), "warning"); 507 __err = trimerr (lasterr (), "error");
489 __mismatch = isempty (regexp (__err, __pattern, "once")); 508 __mismatch = isempty (regexp (__err, __pattern, "once"));
490 endif 509 endif
491 warning (__warnstate.state, "quiet"); 510 warning (__warnstate.state, "quiet");
492 if (isempty (__err)) 511 if (__iswarning)
493 __msg = [__signal_fail "warning failed.\n" ... 512 __msg = [__signal_fail "warning failed.\n" ...
494 "Expected " __patstr ", but got no warning\n"]; 513 "Expected warning " __patstr ...
514 ", but got error <" __err ">\n"];
495 elseif (__mismatch) 515 elseif (__mismatch)
496 __msg = [__signal_fail "warning failed.\n" ... 516 __msg = [__signal_fail "error failed.\n" ...
497 "Expected " __patstr ", but got <" __err ">\n"]; 517 "Expected " __patstr ", but got <" __err ">\n"];
498 else 518 else
499 __success = true; 519 __success = true;
500 endif 520 endif
521 end_try_catch
522 clear __test__;
523 endif
524 ## Code already processed.
525 __code = "";
526
527 ### TESTIF
528
529 elseif (strcmp (__type, "testif"))
530 __e = regexp (__code, '.$', 'lineanchors', 'once');
531 ## Strip any comment from testif line before looking for features
532 __feat_line = strtok (__code(1:__e), '#%');
533 __feat = regexp (__feat_line, '\w+', 'match');
534 __feat = strrep (__feat, "HAVE_", "");
535 __have_feat = __have_feature__ (__feat);
536 if (__have_feat)
537 __istest = true;
538 __code = __code(__e + 1 : end);
539 else
540 __xskip++;
541 __istest = false;
542 __code = ""; # Skip the code.
543 __msg = [__signal_skip "skipped test\n"];
544 endif
545
546 ### TEST
547
548 elseif (strcmp (__type, "test"))
549 __istest = true;
550 ## Code will be evaluated below.
551
552 ### XTEST
553
554 elseif (strcmp (__type, "xtest"))
555 __istest = false;
556 __isxtest = true;
557 ## Code will be evaluated below.
558
559 ### Comment block.
560
561 elseif (strcmp (__block(1:1), "#"))
562 __istest = false;
563 __code = ""; # skip the code
564
565 ### Unknown block.
566
567 else
568 __istest = true;
569 __success = false;
570 __msg = [__signal_fail "unknown test type!\n"];
571 __code = ""; # skip the code
572 endif
573
574 ## evaluate code for test, shared, and assert.
575 if (! isempty(__code))
576 try
577 ## FIXME: Must check for embedded test functions, which cause
578 ## segfaults, until issues with subfunctions in functions are resolved.
579 embed_func = regexp (__code, '^\s*function ', 'once', 'lineanchors');
580 if (isempty (embed_func))
581 eval (sprintf ("function %s__test__(%s)\n%s\nendfunction",
582 __shared_r, __shared, __code));
583 eval (sprintf ("%s__test__(%s);", __shared_r, __shared));
584 else
585 error (["Functions embedded in %!test blocks are not allowed.\n", ...
586 "Use the %!function/%!endfunction syntax instead to define shared functions for testing.\n"]);
501 endif 587 endif
502
503 catch 588 catch
504 if (! isempty (__id)) 589 if (strcmp (__type, "xtest"))
505 [~, __err] = lasterr (); 590 __msg = [__signal_fail "known failure\n" lasterr()];
506 __mismatch = ! strcmp (__err, __id); 591 __xfail++;
592 __success = false;
507 else 593 else
508 __err = trimerr (lasterr (), "error"); 594 __msg = [__signal_fail "test failed\n" lasterr()];
509 __mismatch = isempty (regexp (__err, __pattern, "once")); 595 __success = false;
510 endif 596 endif
511 warning (__warnstate.state, "quiet"); 597 if (isempty (lasterr ()))
512 if (__iswarning) 598 error ("empty error text, probably Ctrl-C --- aborting");
513 __msg = [__signal_fail "warning failed.\n" ...
514 "Expected warning " __patstr ...
515 ", but got error <" __err ">\n"];
516 elseif (__mismatch)
517 __msg = [__signal_fail "error failed.\n" ...
518 "Expected " __patstr ", but got <" __err ">\n"];
519 else
520 __success = true;
521 endif 599 endif
522 end_try_catch 600 end_try_catch
523 clear __test__; 601 clear __test__;
524 endif 602 endif
525 ## Code already processed. 603
526 __code = ""; 604 ## All done. Remember if we were successful and print any messages.
527 605 if (! isempty (__msg) && (__verbose >= 0 || __logfile))
528 ### TESTIF 606 ## Make sure the user knows what caused the error.
529 607 if (__verbose < 1)
530 elseif (strcmp (__type, "testif")) 608 fprintf (__fid, "%s%s\n", __signal_block, __block);
531 __e = regexp (__code, '.$', 'lineanchors', 'once'); 609 fflush (__fid);
532 ## Strip any comment from testif line before looking for features 610 endif
533 __feat_line = strtok (__code(1:__e), '#%'); 611 fprintf (__fid, "%s\n", __msg);
534 __feat = regexp (__feat_line, '\w+', 'match');
535 __feat = strrep (__feat, "HAVE_", "");
536 __have_feat = __have_feature__ (__feat);
537 if (__have_feat)
538 __istest = true;
539 __code = __code(__e + 1 : end);
540 else
541 __xskip++;
542 __istest = false;
543 __code = ""; # Skip the code.
544 __msg = [__signal_skip "skipped test\n"];
545 endif
546
547 ### TEST
548
549 elseif (strcmp (__type, "test"))
550 __istest = true;
551 ## Code will be evaluated below.
552
553 ### XTEST
554
555 elseif (strcmp (__type, "xtest"))
556 __istest = false;
557 __isxtest = true;
558 ## Code will be evaluated below.
559
560 ### Comment block.
561
562 elseif (strcmp (__block(1:1), "#"))
563 __istest = false;
564 __code = ""; # skip the code
565
566 ### Unknown block.
567
568 else
569 __istest = true;
570 __success = false;
571 __msg = [__signal_fail "unknown test type!\n"];
572 __code = ""; # skip the code
573 endif
574
575 ## evaluate code for test, shared, and assert.
576 if (! isempty(__code))
577 try
578 ## FIXME: Must check for embedded test functions, which cause
579 ## segfaults, until issues with subfunctions in functions are resolved.
580 embed_func = regexp (__code, '^\s*function ', 'once', 'lineanchors');
581 if (isempty (embed_func))
582 eval (sprintf ("function %s__test__(%s)\n%s\nendfunction",
583 __shared_r, __shared, __code));
584 eval (sprintf ("%s__test__(%s);", __shared_r, __shared));
585 else
586 error (["Functions embedded in %!test blocks are not allowed.\n", ...
587 "Use the %!function/%!endfunction syntax instead to define shared functions for testing.\n"]);
588 endif
589 catch
590 if (strcmp (__type, "xtest"))
591 __msg = [__signal_fail "known failure\n" lasterr()];
592 __xfail++;
593 __success = false;
594 else
595 __msg = [__signal_fail "test failed\n" lasterr()];
596 __success = false;
597 endif
598 if (isempty (lasterr ()))
599 error ("empty error text, probably Ctrl-C --- aborting");
600 endif
601 end_try_catch
602 clear __test__;
603 endif
604
605 ## All done. Remember if we were successful and print any messages.
606 if (! isempty (__msg) && (__verbose >= 0 || __logfile))
607 ## Make sure the user knows what caused the error.
608 if (__verbose < 1)
609 fprintf (__fid, "%s%s\n", __signal_block, __block);
610 fflush (__fid); 612 fflush (__fid);
611 endif 613 ## Show the variable context.
612 fprintf (__fid, "%s\n", __msg); 614 if (! strcmp (__type, "error") && ! strcmp (__type, "testif")
613 fflush (__fid); 615 && ! all (__shared == " "))
614 ## Show the variable context. 616 fputs (__fid, "shared variables ");
615 if (! strcmp (__type, "error") && ! strcmp (__type, "testif") 617 eval (sprintf ("fdisp(__fid,var2struct(%s));", __shared));
616 && ! all (__shared == " ")) 618 fflush (__fid);
617 fputs (__fid, "shared variables "); 619 endif
618 eval (sprintf ("fdisp(__fid,var2struct(%s));", __shared)); 620 endif
619 fflush (__fid); 621 if (! __success && ! __isxtest)
620 endif 622 __all_success = false;
621 endif 623 ## Stop after 1 error if not in batch mode or only pass/fail requested.
622 if (! __success && ! __isxtest) 624 if (! __batch || nargout == 1)
623 __all_success = false; 625 if (nargout > 0)
624 ## Stop after 1 error if not in batch mode or only pass/fail requested. 626 if (nargout == 1)
625 if (! __batch || nargout == 1) 627 __n = false;
626 if (nargout > 0) 628 else
627 if (nargout == 1) 629 __n = __nmax = 0;
628 __n = false; 630 endif
629 else
630 __n = __nmax = 0;
631 endif 631 endif
632 endif 632 if (__close_fid)
633 if (__close_fid) 633 fclose (__fid);
634 fclose (__fid); 634 endif
635 endif 635 return;
636 return; 636 endif
637 endif 637 endif
638 endif 638 __tests += (__istest || __isxtest);
639 __tests += (__istest || __isxtest); 639 __successes += __success && (__istest || __isxtest);
640 __successes += __success && (__istest || __isxtest); 640
641 unwind_protect_cleanup
642 warning ("off", "all");
643 warning (orig_wstate);
644 end_unwind_protect
641 endfor 645 endfor
642 646
643 ## Clear any functions created during test run 647 ## Clear any functions created during test run
644 eval (__clearfcn, ""); 648 eval (__clearfcn, "");
645 649