Mercurial > octave

diff libinterp/dldfcn/convhulln.cc @ 18077:ac74b0c4c564 stable

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
avoid overflow when passing problem dimensions to qhull with --enable-64 * __delaunayn__.cc, __voronoi__.cc, convhulln.cc (octave_qhull_dims_ok): New function. Use it to avoid overflowing the range of integer values used in Qhull when Octave uses 64-bit integer indexing.
author John W. Eaton <jwe@octave.org>
date Tue, 03 Dec 2013 21:39:39 -0500
parents 175b392e91fe
children 65554f5847ac
line wrap: on
line diff
--- a/libinterp/dldfcn/convhulln.cc	Wed Dec 04 08:41:22 2013 -0800
+++ b/libinterp/dldfcn/convhulln.cc	Tue Dec 03 21:39:39 2013 -0500
@@ -55,6 +55,23 @@
   gnulib::fclose (f);
 }
 
+static bool
+octave_qhull_dims_ok (octave_idx_type dim, octave_idx_type n, const char *who)
+{
+  if (sizeof (octave_idx_type) > sizeof (int))
+    {
+      int maxval = std::numeric_limits<int>::max ();
+
+      if (dim > maxval || n > maxval)
+        {
+          error ("%s: dimension too large for Qhull", who);
+          return false;
+        }
+    }
+
+  return true;
+}
+
 DEFUN_DLD (convhulln, args, nargout,
            "-*- texinfo -*-\n\
 @deftypefn  {Loadable Function} {@var{h} =} convhulln (@var{pts})\n\
@@ -102,6 +119,9 @@
   const octave_idx_type dim = points.columns ();
   const octave_idx_type num_points = points.rows ();
 
+  if (! octave_qhull_dims_ok (dim, num_points, "convhulln"))
+    return retval;
+
   points = points.transpose ();
 
   std::string options;