Mercurial > octave

changeset 22376:3473246a824e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
allow %!test blocks to be tagged with messages or bug ids * test.m: Handle <MESSAGE> option for assert, fail, test, testif, and xtest blocks. * doc/interpreter/testfun.txi: Update docs.
author John W. Eaton <jwe@octave.org>
date Wed, 24 Aug 2016 20:04:52 -0400
parents 179d088a6375
children c0f446d657bf
files doc/interpreter/testfun.txi scripts/testfun/test.m
diffstat 2 files changed, 115 insertions(+), 40 deletions(-) [+]
line wrap: on
line diff
--- a/doc/interpreter/testfun.txi	Wed Aug 24 15:06:30 2016 +0100
+++ b/doc/interpreter/testfun.txi	Wed Aug 24 20:04:52 2016 -0400
@@ -332,43 +332,74 @@
 
 @table @code
 @item %!test
-check that entire block is correct
+@itemx %!test <MESSAGE>
+Check that entire block is correct.  If @code{<MESSAGE>} is present, the
+test block is interpreted as for @code{xtest}.
 
 @item %!testif HAVE_XXX
-check block only if Octave was compiled with feature HAVE_XXX.
+@itemx %!testif HAVE_XXX, HAVE_YYY, @dots{}
+@itemx %!testif @dots{} <MESSAGE>
+Check block only if Octave was compiled with feature HAVE_XXX.  If
+@code{<MESSAGE>} is present, the test block is interpreted as for
+@code{xtest}.
 
 @item %!xtest
-check block, report a test failure but do not abort testing.
+@itemx %!xtest <MESSAGE>
+Check block, report a test failure but do not abort testing.  If
+@code{<MESSAGE>} is present, then the text of the message is displayed
+if the test fails, like this:
+
+@example
+!!!!! Known bug:  MESSAGE
+@end example
+
+@noindent
+If the message is an integer, it is interpreted as a bug ID for the
+Octave bug tracker and reported as
+
+@example
+!!!!! Known bug: http://octave.org/testfailure/?BUG-ID
+@end example
+
+@noindent
+in which BUG-ID is the integer bug number.  The intent is to allow
+clearer documentation of known problems.
 
 @item %!error
-check for correct error message
-
-@item %!warning
-check for correct warning message
+@itemx %!error <MESSAGE>
+@itemx %!warning
+@itemx %!warning <MESSAGE>
+Check for correct error or warning message.  If @code{<MESSAGE>} is
+supplied it is interpreted as a regular expression pattern that is
+expected to match the error or warning message.
 
 @item %!demo
-demo only executes in interactive mode
+Demo only executes in interactive mode.
 
 @item %!#
-comment: ignore everything within the block
+Comment.  Ignore everything within the block
 
 @item %!shared x,y,z
-declare variables for use in multiple tests
+Declare variables for use in multiple tests.
 
 @item %!function
-define a function for use in multiple tests
+Define a function for use in multiple tests.
 
 @item %!endfunction
-close a function definition
+Close a function definition.
 
 @item %!assert (x, y, tol)
-shorthand for @code{%!test assert (x, y, tol)}
-
+@item %!assert <MESSAGE> (x, y, tol)
 @item %!fail (CODE, PATTERN)
-shorthand for @code{%!test fail (CODE, PATTERN)}
+@item %!fail <MESSAGE> (CODE, PATTERN)
+Shorthand for @code{%!test assert (x, y, tol)} or
+@code{%!test fail (CODE, PATTERN)}.  If @code{<MESSAGE>} is present, the
+test block is interpreted as for @code{xtest}.
 
 @end table
 
+@anchor{test-message-anchor}
+
 When coding tests the Octave convention is that lines that begin with a block
 type do not have a semicolon at the end.  Any code that is within a block,
 however, is normal Octave code and usually will have a trailing semicolon.
--- a/scripts/testfun/test.m	Wed Aug 24 15:06:30 2016 +0100
+++ b/scripts/testfun/test.m	Wed Aug 24 20:04:52 2016 -0400
@@ -327,7 +327,9 @@
       ## Assume the block will succeed.
       __success = true;
       __msg = [];
+      __istest = false;
       __isxtest = false;
+      __bug_id = "";
 
 ### DEMO
 
@@ -338,8 +340,6 @@
 
       __isdemo = strcmp (__type, "demo");
       if (__grabdemo || __isdemo)
-        __istest = false;
-
         if (__grabdemo && __isdemo)
           if (isempty (__demo_code))
             __demo_code = __code;
@@ -368,8 +368,6 @@
 ### SHARED
 
       elseif (strcmp (__type, "shared"))
-        __istest = false;
-
         ## Separate initialization code from variables.
         __idx = find (__code == "\n");
         if (isempty (__idx))
@@ -409,7 +407,6 @@
 ### FUNCTION
 
       elseif (strcmp (__type, "function"))
-        __istest = false;
         persistent __fn = 0;
         __name_position = function_name (__block);
         if (isempty (__name_position))
@@ -433,15 +430,19 @@
       elseif (strcmp (__type, "endfunction"))
         ## endfunction simply declares the end of a previous function block.
         ## There is no processing to be done here, just skip to next block.
-        __istest = false;
         __code = "";
 
 ### ASSERT/FAIL
 
       elseif (strcmp (__type, "assert") || strcmp (__type, "fail"))
-        __istest = true;
+        [__bug_id, __code] = getbugid (__code);
+        if (isempty (__bug_id))
+          __istest = true;
+        else
+          __isxtest = true;
+        endif
         ## Put the keyword back on the code.
-        __code = __block;
+        __code = [__type __code];
         ## The code will be evaluated below as a test block.
 
 ### ERROR/WARNING
@@ -528,17 +529,30 @@
 
       elseif (strcmp (__type, "testif"))
         __e = regexp (__code, '.$', 'lineanchors', 'once');
-        ## Strip any comment from testif line before looking for features
+        ## Strip any comment and bug-id from testif line before
+        ## looking for features
         __feat_line = strtok (__code(1:__e), '#%');
+        __idx1 = index (__feat_line, "<");
+        if (__idx1)
+          __tmp = __feat_line(__idx1+1:end);
+          __idx2 = index (__tmp, ">");
+          if (__idx2)
+            __bug_id = __tmp(1:__idx2-1);
+            __feat_line = __feat_line(1:__idx1-1);
+          endif
+        endif
         __feat = regexp (__feat_line, '\w+', 'match');
         __feat = strrep (__feat, "HAVE_", "");
         __have_feat = __have_feature__ (__feat);
         if (__have_feat)
-          __istest = true;
+          if (isempty (__bug_id))
+            __istest = true;
+          else
+            __isxtest = true;
+          endif
           __code = __code(__e + 1 : end);
         else
           __xskip += 1;
-          __istest = false;
           __code = ""; # Skip the code.
           __msg = [__signal_skip "skipped test\n"];
         endif
@@ -546,20 +560,24 @@
 ### TEST
 
       elseif (strcmp (__type, "test"))
-        __istest = true;
+        [__bug_id, __code] = getbugid (__code);
+        if (! isempty (__bug_id))
+          __isxtest = true;
+        else
+          __istest = true;
+        endif
         ## Code will be evaluated below.
 
 ### XTEST
 
       elseif (strcmp (__type, "xtest"))
-        __istest = false;
         __isxtest = true;
+        [__bug_id, __code] = getbugid (__code);
         ## Code will be evaluated below.
 
 ### Comment block.
 
       elseif (strcmp (__block(1:1), "#"))
-        __istest = false;
         __code = ""; # skip the code
 
 ### Unknown block.
@@ -586,16 +604,24 @@
                     "Use the %!function/%!endfunction syntax instead to define shared functions for testing.\n"]);
           endif
         catch
-          if (strcmp (__type, "xtest"))
-            __msg = [__signal_fail "known failure\n"];
-            __xfail += 1;
-            __success = false;
-          else
-            __msg = [__signal_fail "test failed\n" lasterr()];
-            __success = false;
-          endif
           if (isempty (lasterr ()))
             error ("test: empty error text, probably Ctrl-C --- aborting");
+          else
+            __success = false;
+            if (__isxtest)
+              __xfail += 1;
+              if (isempty (__bug_id))
+                __msg = [__signal_fail "known failure\n"];
+              else
+                if (all (isdigit (__bug_id)))
+                  __bug_id = ["http://octave.org/testfailure/?" __bug_id];
+                endif
+                __msg = ["known bug: " __bug_id "\n"];
+              endif
+            else
+              __msg = "test failed\n";
+            endif
+            __msg = [__signal_fail __msg lasterr()];
           endif
         end_try_catch
         clear __test__;
@@ -651,7 +677,7 @@
   if (nargout == 0)
     if (__tests || __xfail || __xskip)
       if (__xfail)
-        printf ("PASSES %d out of %d test%s (%d expected failure%s)\n",
+        printf ("PASSES %d out of %d test%s (%d known failure%s)\n",
                 __successes, __tests, ifelse (__tests > 1, "s", ""),
                 __xfail, ifelse (__xfail > 1, "s", ""));
       else
@@ -713,7 +739,7 @@
 endfunction
 
 ## Strip <pattern> from '<pattern> code'.
-## Also handles 'id=ID code'
+## Optionally also handles 'id=ID code'
 function [pattern, id, rest] = getpattern (str)
 
   pattern = ".";
@@ -732,6 +758,24 @@
 
 endfunction
 
+## Strip <bug-id> from '<pattern> code'.
+function [bug_id, rest] = getbugid (str)
+
+  bug_id = "";
+  id = [];
+  rest = str;
+  str = trimleft (str);
+  if (! isempty (str) && str(1) == "<")
+    close = index (str, ">");
+    if (close)
+      bug_id = str(2:close-1);
+      rest = str(close+1:end);
+    endif
+  endif
+
+endfunction
+
+
 ## Strip '.*prefix:' from '.*prefix: msg\n' and strip trailing blanks.
 function msg = trimerr (msg, prefix)
   idx = index (msg, [prefix ":"]);
@@ -901,7 +945,7 @@
 
 ## All of the following tests should fail.  These tests should
 ## be disabled unless you are developing test() since users don't
-## like to be presented with expected failures.
+## like to be presented with known failures.
 ## %!test   error("---------Failure tests.  Use test('test','verbose',1)");
 ## %!test   assert([a,b,c],[1,3,6]);   # variables have wrong values
 ## %!invalid                   # unknown block type