changeset 24855:41f80b9af274

prevent stack overflow crash on deeply nested function calls (bug #47620) * call-stack.h, call-stack.cc (call_stack::m_max_stack_depth): New member variable. (call_stack::max_stack_depth): New function. (Fmax_stack_depth): New function. (call_stack::push): Error if call_stack size exceeds m_max_stack_depth. * pt-eval.cc (Fmax_recursion_depth): Add @seealso to doc string. * expr.txi: Document max_stack_depth.
author John W. Eaton <jwe@octave.org>
date Fri, 09 Mar 2018 17:14:52 -0500
parents 32671b14ed7b
children 8bb0251fcfde
files doc/interpreter/expr.txi libinterp/corefcn/call-stack.cc libinterp/corefcn/call-stack.h libinterp/parse-tree/pt-eval.cc
diffstat 4 files changed, 66 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/doc/interpreter/expr.txi	Fri Mar 09 11:42:57 2018 -0500
+++ b/doc/interpreter/expr.txi	Fri Mar 09 17:14:52 2018 -0500
@@ -493,9 +493,16 @@
 
 The built-in variable @code{max_recursion_depth} specifies a limit to
 the recursion depth and prevents Octave from recursing infinitely.
+Similarly, the variable @code{max_stack_depth} specifies a limit to the
+depth of function calls, whether recursive or not.  These limits help
+prevent stack overflow on the computer Octave is running on, so that
+instead of exiting with a signal, the interpreter will throw an error
+and return to the command prompt.
 
 @DOCSTRING(max_recursion_depth)
 
+@DOCSTRING(max_stack_depth)
+
 @node Access via Handle
 @subsection Access via Handle
 @cindex function handle
--- a/libinterp/corefcn/call-stack.cc	Fri Mar 09 11:42:57 2018 -0500
+++ b/libinterp/corefcn/call-stack.cc	Fri Mar 09 17:14:52 2018 -0500
@@ -25,6 +25,7 @@
 #endif
 
 #include "call-stack.h"
+#include "defun.h"
 #include "interpreter.h"
 #include "oct-map.h"
 #include "ov.h"
@@ -32,6 +33,7 @@
 #include "ov-fcn-handle.h"
 #include "ov-usr-fcn.h"
 #include "pager.h"
+#include "variables.h"
 
 // Use static fields for the best efficiency.
 // NOTE: C++0x will allow these two to be merged into one.
@@ -86,7 +88,7 @@
   }
 
   call_stack::call_stack (interpreter& interp)
-    : cs (), curr_frame (0), m_interpreter (interp)
+    : cs (), curr_frame (0), m_max_stack_depth (1024), m_interpreter (interp)
   {
     symbol_table& symtab = m_interpreter.get_symbol_table ();
 
@@ -365,6 +367,11 @@
   {
     size_t prev_frame = curr_frame;
     curr_frame = cs.size ();
+
+    // m_max_stack_depth should never be less than zero.
+    if (curr_frame > static_cast<size_t> (m_max_stack_depth))
+      error ("max_stack_depth exceeded");
+
     cs.push_back (stack_frame (fcn, scope, context, prev_frame));
 
     symbol_table& symtab = m_interpreter.get_symbol_table ();
@@ -624,4 +631,47 @@
         symtab.set_scope_and_context (new_elt.m_scope, new_elt.m_context);
       }
   }
+
+  octave_value
+  call_stack::max_stack_depth (const octave_value_list& args, int nargout)
+  {
+    return set_internal_variable (m_max_stack_depth, args, nargout,
+                                  "max_stack_depth", 0);
+  }
 }
+
+DEFMETHOD (max_stack_depth, interp, args, nargout,
+           doc: /* -*- texinfo -*-
+@deftypefn  {} {@var{val} =} max_stack_depth ()
+@deftypefnx {} {@var{old_val} =} max_stack_depth (@var{new_val})
+@deftypefnx {} {} max_stack_depth (@var{new_val}, "local")
+Query or set the internal limit on the number of times a function may
+be called recursively.
+
+If the limit is exceeded, an error message is printed and control returns to
+the top level.
+
+When called from inside a function with the @qcode{"local"} option, the
+variable is changed locally for the function and any subroutines it calls.
+The original variable value is restored when exiting the function.
+
+@seealso{max_recursion_depth}
+@end deftypefn */)
+{
+  octave::call_stack& cs = interp.get_call_stack ();
+
+  return cs.max_stack_depth (args, nargout);
+}
+
+/*
+%!test
+%! orig_val = max_stack_depth ();
+%! old_val = max_stack_depth (2*orig_val);
+%! assert (orig_val, old_val);
+%! assert (max_stack_depth (), 2*orig_val);
+%! max_stack_depth (orig_val);
+%! assert (max_stack_depth (), orig_val);
+
+%!error (max_stack_depth (1, 2))
+*/
+
--- a/libinterp/corefcn/call-stack.h	Fri Mar 09 11:42:57 2018 -0500
+++ b/libinterp/corefcn/call-stack.h	Fri Mar 09 17:14:52 2018 -0500
@@ -32,6 +32,8 @@
 class octave_map;
 class octave_user_code;
 class octave_user_script;
+class octave_value;
+class octave_value_list;
 
 #include "symscope.h"
 
@@ -252,6 +254,8 @@
 
     void clear (void) { cs.clear (); }
 
+    octave_value max_stack_depth (const octave_value_list& args, int nargout);
+
   private:
 
     // The current call stack.
@@ -259,6 +263,8 @@
 
     size_t curr_frame;
 
+    int m_max_stack_depth;
+
     interpreter& m_interpreter;
   };
 }
--- a/libinterp/parse-tree/pt-eval.cc	Fri Mar 09 11:42:57 2018 -0500
+++ b/libinterp/parse-tree/pt-eval.cc	Fri Mar 09 17:14:52 2018 -0500
@@ -3242,6 +3242,8 @@
 When called from inside a function with the @qcode{"local"} option, the
 variable is changed locally for the function and any subroutines it calls.
 The original variable value is restored when exiting the function.
+
+@seealso{max_stack_depth}
 @end deftypefn */)
 {
   octave::tree_evaluator& tw = interp.get_evaluator ();