changeset 31842:79abf4cdcd95

mat2cell: Next attempt to avoid heap buffer overflow (bug #63682). * libinterp/corefcn/cellfun.cc (do_mat2cell_nd): Use sufficiently large size for all temporary buffers. Use correct values for 2nd dimension if split-values for it are omitted in the input.
author Markus Mützel <markus.muetzel@gmx.de>
date Sun, 19 Feb 2023 17:19:24 +0100
parents 0e046fefbffa
children 0199ff6cc997
files libinterp/corefcn/cellfun.cc
diffstat 1 files changed, 28 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/libinterp/corefcn/cellfun.cc	Sun Feb 19 12:45:22 2023 +0100
+++ b/libinterp/corefcn/cellfun.cc	Sun Feb 19 17:19:24 2023 +0100
@@ -2052,35 +2052,57 @@
   if (mat2cell_mismatch (a.dims (), d, nd))
     return retval;
 
+  bool is_1d = false;
+  if (nd < 2)
+    {
+      // work with 2d matrix
+      is_1d = true;
+      nd = 2;
+    }
+
+  // vector with number of splits in respective dimension
   dim_vector rdv = dim_vector::alloc (nd);
   OCTAVE_LOCAL_BUFFER (octave_idx_type, nidx, nd);
   octave_idx_type idxtot = 0;
-  for (int i = 0; i < nd; i++)
+  for (int i = 0; i < (is_1d ? 1 : nd); i++)
     {
       rdv(i) = nidx[i] = d[i].numel ();
       idxtot += nidx[i];
     }
 
-  if (nd == 1)
-    rdv(1) = 1;
+  if (is_1d)
+    {
+      // If input was 1d, don't split 2nd dimension.
+      rdv(1) = nidx[1] = 1;
+      idxtot++;
+    }
   retval.clear (rdv);
 
+  // buffer for "linearized" number of dimensions
   OCTAVE_LOCAL_BUFFER (idx_vector, xidx, idxtot);
+  // buffer with size of respective dimensions after split
   OCTAVE_LOCAL_BUFFER (idx_vector *, idx, nd);
 
   idxtot = 0;
-  for (int i = 0; i < nd; i++)
+  for (int i = 0; i < (is_1d ? 1 : nd); i++)
     {
       idx[i] = xidx + idxtot;
       prepare_idx (idx[i], i, nd, d);
       idxtot += nidx[i];
     }
 
-  OCTAVE_LOCAL_BUFFER_INIT (octave_idx_type, ridx, rdv.numel (), 0);
+  if (is_1d)
+    {
+      // If input was 1d, take complete second dimension.
+      idx[1] = xidx + idxtot;
+      idx[1][0] = idx_vector::colon;
+    }
+
+  OCTAVE_LOCAL_BUFFER_INIT (octave_idx_type, ridx, nd, 0);
   Array<idx_vector> ra_idx
   (dim_vector (1, std::max (nd, a.ndims ())), idx_vector::colon);
 
-  for (octave_idx_type j = 0; j < retval.numel (); j++)
+  for (octave_idx_type j = 0; j < nd; j++)
     {
       octave_quit ();