diff scripts/set/intersect.m @ 29190:7f11d59e3af8 stable

Fix incorrect results for set functions with "legacy" option (bug #59708). * intersect.m: Don't transpose output in "legacy" mode if by_rows is true. Add BIST test for "legacy" and "rows" options present at same time. * setdiff.m: In "legacy" BIST test, also verify setdiff() result when "legacy" is not present. * setxor.m: Don't transpose output in "legacy" mode if by_rows is true. Add 4 BIST tests for "legacy" output orientation for vectors. Add BIST test for "legacy" and "rows" options present at same time. * union.m: Add 4 BIST tests for "legacy" output orientation for vectors. Add BIST test for "legacy" and "rows" options present at same time.
author Rik <rik@octave.org>
date Thu, 17 Dec 2020 14:02:18 -0800
parents bd51beb6205e
children 0a5b15007766
line wrap: on
line diff
--- a/scripts/set/intersect.m	Thu Dec 17 07:23:49 2020 +0100
+++ b/scripts/set/intersect.m	Thu Dec 17 14:02:18 2020 -0800
@@ -134,11 +134,11 @@
       ## FIXME: Is there a way to avoid a call to sort?
       c = c(sort (ic(match)));
     endif
-  endif
 
-  ## Adjust output orientation for Matlab compatibility
-  if (isrowvec)
-    c = c.';
+    ## Adjust output orientation for Matlab compatibility
+    if (isrowvec)
+      c = c.';
+    endif
   endif
 
   if (nargout > 1)
@@ -150,7 +150,7 @@
       [~, idx] = min (ib);
       ib = [ib(idx:end); ib(1:idx-1)];
     endif
-    if (optlegacy && isrowvec)
+    if (optlegacy && isrowvec && ! by_rows)
       ia = ia.';
       ib = ib.';
     endif
@@ -252,6 +252,18 @@
 %! assert (ia, [5, 4]);
 %! assert (ib, [4, 1]);
 
+%!test  # "legacy" + "rows"
+%! A = [ 1 2; 3 4; 5 6; 3 4; 7 8 ];
+%! B = [ 3 4; 7 8; 9 10 ];
+%! [c, ia, ib] = intersect (A, B, "rows");
+%! assert (c, [3, 4; 7, 8]);
+%! assert (ia, [2; 5]);
+%! assert (ib, [1; 2]);
+%! [c, ia, ib] = intersect (A, B, "rows", "legacy");
+%! assert (c, [3, 4; 7, 8]);
+%! assert (ia, [4; 5]);
+%! assert (ib, [1; 2]);
+
 ## Test orientation of output
 %!shared a,b
 %! a = 1:4;