comparison libinterp/dldfcn/__delaunayn__.cc @ 18077:ac74b0c4c564 stable

avoid overflow when passing problem dimensions to qhull with --enable-64 * __delaunayn__.cc, __voronoi__.cc, convhulln.cc (octave_qhull_dims_ok): New function. Use it to avoid overflowing the range of integer values used in Qhull when Octave uses 64-bit integer indexing.
author John W. Eaton <jwe@octave.org>
date Tue, 03 Dec 2013 21:39:39 -0500
parents 175b392e91fe
children 6a71e5030df5
comparison
equal deleted inserted replaced
18075:24759ac2b8cb 18077:ac74b0c4c564
60 60
61 static void 61 static void
62 close_fcn (FILE *f) 62 close_fcn (FILE *f)
63 { 63 {
64 gnulib::fclose (f); 64 gnulib::fclose (f);
65 }
66
67 static bool
68 octave_qhull_dims_ok (octave_idx_type dim, octave_idx_type n, const char *who)
69 {
70 if (sizeof (octave_idx_type) > sizeof (int))
71 {
72 int maxval = std::numeric_limits<int>::max ();
73
74 if (dim > maxval || n > maxval)
75 {
76 error ("%s: dimension too large for Qhull", who);
77 return false;
78 }
79 }
80
81 return true;
65 } 82 }
66 83
67 DEFUN_DLD (__delaunayn__, args, , 84 DEFUN_DLD (__delaunayn__, args, ,
68 "-*- texinfo -*-\n\ 85 "-*- texinfo -*-\n\
69 @deftypefn {Loadable Function} {@var{T} =} __delaunayn__ (@var{pts})\n\ 86 @deftypefn {Loadable Function} {@var{T} =} __delaunayn__ (@var{pts})\n\
86 } 103 }
87 104
88 Matrix p (args(0).matrix_value ()); 105 Matrix p (args(0).matrix_value ());
89 const octave_idx_type dim = p.columns (); 106 const octave_idx_type dim = p.columns ();
90 const octave_idx_type n = p.rows (); 107 const octave_idx_type n = p.rows ();
108
109 if (! octave_qhull_dims_ok (dim, n, "__delaynayn__"))
110 return retval;
91 111
92 // Default options 112 // Default options
93 std::string options; 113 std::string options;
94 if (dim <= 3) 114 if (dim <= 3)
95 options = "Qt Qbb Qc Qz"; 115 options = "Qt Qbb Qc Qz";