# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1648488455 -7200
# Node ID 014030798d5eb40be293e5784a469f46bf720fbd
# Parent  5fa3d8f0dcb3db894ebc808dfd554b562384c465
Avoid issues when converting large integers to floating point (bug #62212).

* libinterp/corefcn/oct-stream.cc (get_size),
libinterp/corefcn/xpow.cc (xisint),
libinterp/octave-value/ov-base.cc (INT_CONV_METHOD),
libinterp/octave-value/ov.cc (check_colon_operand, range_numel (T, double, T)),
liboctave/numeric/lo-mappers.cc (nint_big, nint),
liboctave/util/oct-inttypes.cc (emulate_mop,
operator - (const double, const octave_uint64)),
liboctave/util/oct-string.cc (rational_approx): Take into account that the
maximum value of (signed or unsigned) integers might change its value if
converted to floating point.  In comparisons, check against the first value
*outside* the range of the integer type instead of the last value *inside* its
range.

diff -r 5fa3d8f0dcb3 -r 014030798d5e libinterp/corefcn/oct-stream.cc
--- a/libinterp/corefcn/oct-stream.cc	Thu Mar 24 12:09:12 2022 -0400
+++ b/libinterp/corefcn/oct-stream.cc	Mon Mar 28 19:27:35 2022 +0200
@@ -140,7 +140,10 @@
           ::error ("%s: negative value invalid as size specification",
                    who.c_str ());
 
-        if (d > std::numeric_limits<octave_idx_type>::max ())
+        static const double out_of_range_top
+          = static_cast<double> (std::numeric_limits<octave_idx_type>::max ())
+            + 1.;
+        if (d >= out_of_range_top)
           ::error ("%s: dimension too large for Octave's index type",
                    who.c_str ());
 
diff -r 5fa3d8f0dcb3 -r 014030798d5e libinterp/corefcn/xpow.cc
--- a/libinterp/corefcn/xpow.cc	Thu Mar 24 12:09:12 2022 -0400
+++ b/libinterp/corefcn/xpow.cc	Mon Mar 28 19:27:35 2022 +0200
@@ -76,8 +76,18 @@
 xisint (T x)
 {
   return (octave::math::x_nint (x) == x
-          && ((x >= 0 && x < std::numeric_limits<int>::max ())
-              || (x <= 0 && x > std::numeric_limits<int>::min ())));
+          && x <= std::numeric_limits<int>::max ()
+          && x >= std::numeric_limits<int>::min ());
+}
+
+static inline bool
+xisint (float x)
+{
+  static const float out_of_range_top
+    = static_cast<float>(std::numeric_limits<int>::max ()) + 1.;
+  return (octave::math::x_nint (x) == x
+          && x < out_of_range_top
+          && x >= std::numeric_limits<int>::min ());
 }
 
 // Safer pow functions.
diff -r 5fa3d8f0dcb3 -r 014030798d5e libinterp/octave-value/ov-base.cc
--- a/libinterp/octave-value/ov-base.cc	Thu Mar 24 12:09:12 2022 -0400
+++ b/libinterp/octave-value/ov-base.cc	Mon Mar 28 19:27:35 2022 +0200
@@ -475,11 +475,13 @@
         err_wrong_type_arg (ee, "octave_base_value::" #F "_value ()", type_name ()); \
       }                                                                 \
                                                                         \
+    static const double out_of_range_top                                \
+      = static_cast<double>(std::numeric_limits<T>::max ()) + 1.;       \
     if (require_int && octave::math::x_nint (d) != d)                   \
       error_with_cfn ("conversion of %g to " #T " value failed", d);    \
     else if (d < std::numeric_limits<T>::min ())                        \
       retval = std::numeric_limits<T>::min ();                          \
-    else if (d > std::numeric_limits<T>::max ())                        \
+    else if (d >= out_of_range_top)                                     \
       retval = std::numeric_limits<T>::max ();                          \
     else                                                                \
       retval = static_cast<T> (octave::math::fix (d));                  \
diff -r 5fa3d8f0dcb3 -r 014030798d5e libinterp/octave-value/ov.cc
--- a/libinterp/octave-value/ov.cc	Thu Mar 24 12:09:12 2022 -0400
+++ b/libinterp/octave-value/ov.cc	Mon Mar 28 19:27:35 2022 +0200
@@ -3073,8 +3073,11 @@
 
     double dval = val.double_value ();
     double intpart;
-
-    if (dval > std::numeric_limits<typename T::val_type>::max ()
+    static const double out_of_range_top
+      = static_cast<double> (std::numeric_limits<typename T::val_type>::max ())
+        + 1.;
+
+    if (dval >= out_of_range_top
         || dval < std::numeric_limits<typename T::val_type>::min ()
         || std::modf (dval, &intpart) != 0.0)
       error ("colon operator %s invalid (not an integer or out of range for given integer type)", op_str);
@@ -3246,16 +3249,18 @@
         || (increment < 0 && base < limit))
       return 0;
 
-    static const UT max_val = std::numeric_limits<UT>::max ();
+    static const double out_of_range_top
+      = static_cast<double> (std::numeric_limits<UT>::max ()) + 1.;
 
     double abs_increment = std::abs (increment);
 
-    // Technically, this condition should be `abs_increment > max_val`.
+    // Technically, this condition should be
+    // `abs_increment > std::numeric_limits<UT>::max ()`.
     // But intmax('uint64') is not representable exactly as floating point
     // number.  Instead, it "rounds" up by 1 to 2^64.  To account for
     // this, use the following expression which works for all unsigned
     // integer types.
-    if ((abs_increment-1.) >= max_val)
+    if (abs_increment >= out_of_range_top)
       return 1;
 
     UT unsigned_increment = range_increment<T> (increment);
diff -r 5fa3d8f0dcb3 -r 014030798d5e liboctave/numeric/lo-mappers.cc
--- a/liboctave/numeric/lo-mappers.cc	Thu Mar 24 12:09:12 2022 -0400
+++ b/liboctave/numeric/lo-mappers.cc	Mon Mar 28 19:27:35 2022 +0200
@@ -183,7 +183,9 @@
     octave_idx_type
     nint_big (double x)
     {
-      if (x > std::numeric_limits<octave_idx_type>::max ())
+      static const double out_of_range_top
+        = static_cast<double>(std::numeric_limits<octave_idx_type>::max ())+1.;
+      if (x >= out_of_range_top)
         return std::numeric_limits<octave_idx_type>::max ();
       else if (x < std::numeric_limits<octave_idx_type>::min ())
         return std::numeric_limits<octave_idx_type>::min ();
@@ -195,7 +197,9 @@
     octave_idx_type
     nint_big (float x)
     {
-      if (x > std::numeric_limits<octave_idx_type>::max ())
+      static const float out_of_range_top
+        = static_cast<float>(std::numeric_limits<octave_idx_type>::max ())+1.;
+      if (x >= out_of_range_top)
         return std::numeric_limits<octave_idx_type>::max ();
       else if (x < std::numeric_limits<octave_idx_type>::min ())
         return std::numeric_limits<octave_idx_type>::min ();
@@ -218,7 +222,9 @@
     int
     nint (float x)
     {
-      if (x > std::numeric_limits<int>::max ())
+      static const float out_of_range_top
+        = static_cast<float>(std::numeric_limits<int>::max ()) + 1.;
+      if (x >= out_of_range_top)
         return std::numeric_limits<int>::max ();
       else if (x < std::numeric_limits<int>::min ())
         return std::numeric_limits<int>::min ();
diff -r 5fa3d8f0dcb3 -r 014030798d5e liboctave/util/oct-inttypes.cc
--- a/liboctave/util/oct-inttypes.cc	Thu Mar 24 12:09:12 2022 -0400
+++ b/liboctave/util/oct-inttypes.cc	Mon Mar 28 19:27:35 2022 +0200
@@ -228,7 +228,11 @@
 bool
 octave_int_cmp_op::emulate_mop (uint64_t x, double y)
 {
-  static const double xxup = std::numeric_limits<uint64_t>::max ();
+  // The following cast changes the value to 2^64 (which is outside the range
+  // of `uint64_t`).  Take care to handle this correctly (e.g., don't cast back
+  // to `uint64_t`)!
+  static const double xxup
+    = static_cast<double> (std::numeric_limits<uint64_t>::max ());
   // This converts to the nearest double.  Unless there's an equality, the
   // result is clear.
   double xx = x;
@@ -248,8 +252,14 @@
 bool
 octave_int_cmp_op::emulate_mop (int64_t x, double y)
 {
-  static const double xxup = std::numeric_limits<int64_t>::max ();
-  static const double xxlo = std::numeric_limits<int64_t>::min ();
+  // The following cast changes the value to 2^63 (which is outside the range
+  // of `int64_t`).  Take care to handle this correctly (e.g., don't cast back
+  // to `int64_t`)!  The same applies to the lower limit on systems using one's
+  // complement.
+  static const double xxup
+    = static_cast<double> (std::numeric_limits<int64_t>::max ());
+  static const double xxlo
+    = static_cast<double> (std::numeric_limits<int64_t>::min ());
   // This converts to the nearest double.  Unless there's an equality, the
   // result is clear.
   double xx = x;
@@ -452,6 +462,8 @@
 OCTAVE_API octave_int64
 operator + (const octave_int64& x, const double& y)
 {
+  // The following cast changes the value to 2^63 (which is outside the range
+  // of `int64_t`).
   if (fabs (y) < static_cast<double> (octave_int64::max ()))
     return x + octave_int64 (y);
   else
@@ -487,7 +499,9 @@
 OCTAVE_API octave_uint64
 operator - (const double& x, const octave_uint64& y)
 {
-  if (x <= static_cast<double> (octave_uint64::max ()))
+  // The following cast changes the value to 2^64 (which is outside the range
+  // of `uint64_t`).
+  if (x < static_cast<double> (octave_uint64::max ()))
     return octave_uint64 (x) - y;
   else
     {
@@ -581,7 +595,10 @@
 OCTAVE_API octave_uint64
 operator * (const octave_uint64& x, const double& y)
 {
-  if (y >= 0 && y < octave_uint64::max () && y == octave::math::fix (y))
+  // The following cast changes the value to 2^64 (which is outside the range
+  // of `uint64_t`).
+  if (y >= 0 && y < static_cast<double> (octave_uint64::max ())
+      && y == octave::math::fix (y))
     return x * octave_uint64 (static_cast<uint64_t> (y));
   else if (y == 0.5)
     return x / octave_uint64 (static_cast<uint64_t> (2));
@@ -616,7 +633,10 @@
 OCTAVE_API octave_int64
 operator * (const octave_int64& x, const double& y)
 {
-  if (fabs (y) < octave_int64::max () && y == octave::math::fix (y))
+  // The following cast changes the value to 2^63 (which is outside the range
+  // of `int64_t`).
+  if (fabs (y) < static_cast<double> (octave_int64::max ())
+      && y == octave::math::fix (y))
     return x * octave_int64 (static_cast<int64_t> (y));
   else if (fabs (y) == 0.5)
     return x / octave_int64 (static_cast<uint64_t> (4*y));
@@ -666,7 +686,10 @@
 OCTAVE_API octave_uint64
 operator / (const octave_uint64& x, const double& y)
 {
-  if (y >= 0 && y < octave_uint64::max () && y == octave::math::fix (y))
+  // The following cast changes the value to 2^64 (which is outside the range
+  // of `uint64_t`).
+  if (y >= 0 && y < static_cast<double> (octave_uint64::max ())
+      && y == octave::math::fix (y))
     return x / octave_uint64 (y);
   else
     return x * (1.0/y);
@@ -676,7 +699,10 @@
 OCTAVE_API octave_int64
 operator / (const octave_int64& x, const double& y)
 {
-  if (fabs (y) < octave_int64::max () && y == octave::math::fix (y))
+  // The following cast changes the value to 2^63 (which is outside the range
+  // of `int64_t`).
+  if (fabs (y) < static_cast<double> (octave_int64::max ())
+      && y == octave::math::fix (y))
     return x / octave_int64 (y);
   else
     return x * (1.0/y);
diff -r 5fa3d8f0dcb3 -r 014030798d5e liboctave/util/oct-string.cc
--- a/liboctave/util/oct-string.cc	Thu Mar 24 12:09:12 2022 -0400
+++ b/liboctave/util/oct-string.cc	Mon Mar 28 19:27:35 2022 +0200
@@ -616,6 +616,10 @@
   if (len <= 0)
     len = 10;
 
+  static const T out_of_range_top
+    = static_cast<T>(std::numeric_limits<int>::max ()) + 1.;
+  static const T out_of_range_bottom
+    = static_cast<T>(std::numeric_limits<int>::min ()) - 1.;
   if (octave::math::isinf (val))
     {
       if (val > 0)
@@ -625,8 +629,7 @@
     }
   else if (octave::math::isnan (val))
     s = "0/0";
-  else if (val < std::numeric_limits<int>::min ()
-           || val > std::numeric_limits<int>::max ()
+  else if (val <= out_of_range_bottom || val >= out_of_range_top
            || octave::math::x_nint (val) == val)
     {
       std::ostringstream buf;
@@ -656,7 +659,7 @@
           T nextd = d;
 
           // Have we converged to 1/intmax ?
-          if (std::abs (flip) > static_cast<T> (std::numeric_limits<int>::max ()))
+          if (std::abs (flip) > out_of_range_top)
             {
               lastn = n;
               lastd = d;
@@ -687,8 +690,8 @@
                 break;
             }
 
-          if (std::abs (n) > std::numeric_limits<int>::max ()
-              || std::abs (d) > std::numeric_limits<int>::max ())
+          if (std::abs (n) >= out_of_range_top
+              || std::abs (d) >= out_of_range_top)
             break;
 
           s = buf.str ();