Mercurial > octave

comparison libinterp/corefcn/pr-output.cc @ 21747:61f3575250e4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rats: Fix round-off corner case for 32-bit systems (bug #47964) * pr-output.cc (rational_approx): Invert convergence test to prevent aborting early due to round-off error in 32-bit environments. Use std::abs instead of fabs.
author Mike Miller <mtmiller@octave.org>
date Thu, 19 May 2016 15:47:24 -0700
parents f4d7d0eb5b0c
children 176536b15d68
comparison
equal deleted inserted replaced
21746:ac8b8bdae98f 21747:61f3575250e4
387 double step = xround (flip); 387 double step = xround (flip);
388 double nextn = n; 388 double nextn = n;
389 double nextd = d; 389 double nextd = d;
390 390
391 // Have we converged to 1/intmax ? 391 // Have we converged to 1/intmax ?
392 if (fabs (frac) < 1 / static_cast<double> (std::numeric_limits<int>::max ())) 392 if (std::abs (flip) > static_cast<double> (std::numeric_limits<int>::max ()))
393 { 393 {
394 lastn = n; 394 lastn = n;
395 lastd = d; 395 lastd = d;
396 break; 396 break;
397 } 397 }
415 break; 415 break;
416 } 416 }
417 else if (buf.str ().length () > static_cast<unsigned int>(len)) 417 else if (buf.str ().length () > static_cast<unsigned int>(len))
418 break; 418 break;
419 419
420 if (fabs (n) > std::numeric_limits<int>::max () 420 if (std::abs (n) > std::numeric_limits<int>::max ()
421 || fabs (d) > std::numeric_limits<int>::max ()) 421 || std::abs (d) > std::numeric_limits<int>::max ())
422 break; 422 break;
423 423
424 s = buf.str (); 424 s = buf.str ();
425 } 425 }
426 426