# HG changeset patch
# User John W. Eaton <jwe@octave.org>
# Date 1386124779 18000
# Node ID ac74b0c4c564ebd786fb61ed6587cc331595c446
# Parent  24759ac2b8cb735e02583d2dc9c316711b7ade06
avoid overflow when passing problem dimensions to qhull with --enable-64

* __delaunayn__.cc, __voronoi__.cc, convhulln.cc
(octave_qhull_dims_ok): New function.  Use it to avoid overflowing
the range of integer values used in Qhull when Octave uses 64-bit
integer indexing.

diff -r 24759ac2b8cb -r ac74b0c4c564 libinterp/dldfcn/__delaunayn__.cc
--- a/libinterp/dldfcn/__delaunayn__.cc	Wed Dec 04 08:41:22 2013 -0800
+++ b/libinterp/dldfcn/__delaunayn__.cc	Tue Dec 03 21:39:39 2013 -0500
@@ -64,6 +64,23 @@
   gnulib::fclose (f);
 }
 
+static bool
+octave_qhull_dims_ok (octave_idx_type dim, octave_idx_type n, const char *who)
+{
+  if (sizeof (octave_idx_type) > sizeof (int))
+    {
+      int maxval = std::numeric_limits<int>::max ();
+
+      if (dim > maxval || n > maxval)
+        {
+          error ("%s: dimension too large for Qhull", who);
+          return false;
+        }
+    }
+
+  return true;
+}
+
 DEFUN_DLD (__delaunayn__, args, ,
            "-*- texinfo -*-\n\
 @deftypefn  {Loadable Function} {@var{T} =} __delaunayn__ (@var{pts})\n\
@@ -89,6 +106,9 @@
   const octave_idx_type dim = p.columns ();
   const octave_idx_type n = p.rows ();
 
+  if (! octave_qhull_dims_ok (dim, n, "__delaynayn__"))
+    return retval;
+
   // Default options
   std::string options;
   if (dim <= 3)
diff -r 24759ac2b8cb -r ac74b0c4c564 libinterp/dldfcn/__voronoi__.cc
--- a/libinterp/dldfcn/__voronoi__.cc	Wed Dec 04 08:41:22 2013 -0800
+++ b/libinterp/dldfcn/__voronoi__.cc	Tue Dec 03 21:39:39 2013 -0500
@@ -59,6 +59,23 @@
   gnulib::fclose (f);
 }
 
+static bool
+octave_qhull_dims_ok (octave_idx_type dim, octave_idx_type n, const char *who)
+{
+  if (sizeof (octave_idx_type) > sizeof (int))
+    {
+      int maxval = std::numeric_limits<int>::max ();
+
+      if (dim > maxval || n > maxval)
+        {
+          error ("%s: dimension too large for Qhull", who);
+          return false;
+        }
+    }
+
+  return true;
+}
+
 DEFUN_DLD (__voronoi__, args, ,
            "-*- texinfo -*-\n\
 @deftypefn  {Loadable Function} {@var{C}, @var{F} =} __voronoi__ (@var{caller}, @var{pts})\n\
@@ -86,6 +103,9 @@
   const octave_idx_type dim = points.columns ();
   const octave_idx_type num_points = points.rows ();
 
+  if (! octave_qhull_dims_ok (dim, num_points, "__voronoi__"))
+    return retval;
+
   points = points.transpose ();
 
   std::string options;
diff -r 24759ac2b8cb -r ac74b0c4c564 libinterp/dldfcn/convhulln.cc
--- a/libinterp/dldfcn/convhulln.cc	Wed Dec 04 08:41:22 2013 -0800
+++ b/libinterp/dldfcn/convhulln.cc	Tue Dec 03 21:39:39 2013 -0500
@@ -55,6 +55,23 @@
   gnulib::fclose (f);
 }
 
+static bool
+octave_qhull_dims_ok (octave_idx_type dim, octave_idx_type n, const char *who)
+{
+  if (sizeof (octave_idx_type) > sizeof (int))
+    {
+      int maxval = std::numeric_limits<int>::max ();
+
+      if (dim > maxval || n > maxval)
+        {
+          error ("%s: dimension too large for Qhull", who);
+          return false;
+        }
+    }
+
+  return true;
+}
+
 DEFUN_DLD (convhulln, args, nargout,
            "-*- texinfo -*-\n\
 @deftypefn  {Loadable Function} {@var{h} =} convhulln (@var{pts})\n\
@@ -102,6 +119,9 @@
   const octave_idx_type dim = points.columns ();
   const octave_idx_type num_points = points.rows ();
 
+  if (! octave_qhull_dims_ok (dim, num_points, "convhulln"))
+    return retval;
+
   points = points.transpose ();
 
   std::string options;