Kaydet (Commit) e69bfc3f authored tarafından Nick Coghlan's avatar Nick Coghlan

Issue #5765: Merge from 3.3

...@@ -30,6 +30,8 @@ struct symtable { ...@@ -30,6 +30,8 @@ struct symtable {
PyObject *st_private; /* name of current class or NULL */ PyObject *st_private; /* name of current class or NULL */
PyFutureFeatures *st_future; /* module's future features that affect PyFutureFeatures *st_future; /* module's future features that affect
the symbol table */ the symbol table */
int recursion_depth; /* current recursion depth */
int recursion_limit; /* recursion limit */
}; };
typedef struct _symtable_entry { typedef struct _symtable_entry {
......
"""
The compiler (>= 2.5) recurses happily until it blows the stack.
Recorded on the tracker as http://bugs.python.org/issue11383
"""
# The variant below blows up in compiler_call, but there are assorted
# other variations that blow up in other functions
# e.g. '1*'*10**5+'1' will die in compiler_visit_expr
# The exact limit to destroy the stack will vary by platform
# but 10M should do the trick even with huge stack allocations
compile('()'*10**7, '?', 'exec')
...@@ -474,6 +474,33 @@ if 1: ...@@ -474,6 +474,33 @@ if 1:
self.assertInvalidSingle('f()\nxy # blah\nblah()') self.assertInvalidSingle('f()\nxy # blah\nblah()')
self.assertInvalidSingle('x = 5 # comment\nx = 6\n') self.assertInvalidSingle('x = 5 # comment\nx = 6\n')
@support.cpython_only
def test_compiler_recursion_limit(self):
# Expected limit is sys.getrecursionlimit() * the scaling factor
# in symtable.c (currently 3)
# We expect to fail *at* that limit, because we use up some of
# the stack depth limit in the test suite code
# So we check the expected limit and 75% of that
# XXX (ncoghlan): duplicating the scaling factor here is a little
# ugly. Perhaps it should be exposed somewhere...
fail_depth = sys.getrecursionlimit() * 3
success_depth = int(fail_depth * 0.75)
def check_limit(prefix, repeated):
expect_ok = prefix + repeated * success_depth
self.compile_single(expect_ok)
broken = prefix + repeated * fail_depth
details = "Compiling ({!r} + {!r} * {})".format(
prefix, repeated, fail_depth)
with self.assertRaises(RuntimeError, msg=details):
self.compile_single(broken)
check_limit("a", "()")
check_limit("a", ".b")
check_limit("a", "[0]")
check_limit("a", "*a")
def test_main(): def test_main():
support.run_unittest(TestSpecifics) support.run_unittest(TestSpecifics)
......
...@@ -431,6 +431,7 @@ Hans de Graaff ...@@ -431,6 +431,7 @@ Hans de Graaff
Nathaniel Gray Nathaniel Gray
Eddy De Greef Eddy De Greef
Grant Griffin Grant Griffin
Andrea Griffini
Duncan Grisby Duncan Grisby
Fabian Groffen Fabian Groffen
Eric Groo Eric Groo
......
...@@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1? ...@@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1?
Core and Builtins Core and Builtins
----------------- -----------------
- Issue #5765: Apply a hard recursion limit in the compiler instead of
blowing the stack and segfaulting.
- Issue #16402: When slicing a range, fix shadowing of exceptions from - Issue #16402: When slicing a range, fix shadowing of exceptions from
__index__. __index__.
......
...@@ -141,6 +141,11 @@ struct compiler_unit { ...@@ -141,6 +141,11 @@ struct compiler_unit {
The u pointer points to the current compilation unit, while units The u pointer points to the current compilation unit, while units
for enclosing blocks are stored in c_stack. The u and c_stack are for enclosing blocks are stored in c_stack. The u and c_stack are
managed by compiler_enter_scope() and compiler_exit_scope(). managed by compiler_enter_scope() and compiler_exit_scope().
Note that we don't track recursion levels during compilation - the
task of detecting and rejecting excessive levels of nesting is
handled by the symbol analysis pass.
*/ */
struct compiler { struct compiler {
......
...@@ -223,17 +223,40 @@ symtable_new(void) ...@@ -223,17 +223,40 @@ symtable_new(void)
return NULL; return NULL;
} }
/* When compiling the use of C stack is probably going to be a lot
lighter than when executing Python code but still can overflow
and causing a Python crash if not checked (e.g. eval("()"*300000)).
Using the current recursion limit for the compiler seems too
restrictive (it caused at least one test to fail) so a factor is
used to allow deeper recursion when compiling an expression.
Using a scaling factor means this should automatically adjust when
the recursion limit is adjusted for small or large C stack allocations.
*/
#define COMPILER_STACK_FRAME_SCALE 3
struct symtable * struct symtable *
PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
{ {
struct symtable *st = symtable_new(); struct symtable *st = symtable_new();
asdl_seq *seq; asdl_seq *seq;
int i; int i;
PyThreadState *tstate;
if (st == NULL) if (st == NULL)
return st; return st;
st->st_filename = filename; st->st_filename = filename;
st->st_future = future; st->st_future = future;
/* Setup recursion depth check counters */
tstate = PyThreadState_GET();
if (!tstate) {
PySymtable_Free(st);
return NULL;
}
st->recursion_depth = tstate->recursion_depth * COMPILER_STACK_FRAME_SCALE;
st->recursion_limit = Py_GetRecursionLimit() * COMPILER_STACK_FRAME_SCALE;
/* Make the initial symbol information gathering pass */ /* Make the initial symbol information gathering pass */
if (!GET_IDENTIFIER(top) || if (!GET_IDENTIFIER(top) ||
!symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) { !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) {
...@@ -1031,11 +1054,17 @@ error: ...@@ -1031,11 +1054,17 @@ error:
VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is
useful if the first node in the sequence requires special treatment. useful if the first node in the sequence requires special treatment.
VISIT_QUIT macro returns the specified value exiting from the function but
first adjusts current recursion counter depth.
*/ */
#define VISIT_QUIT(ST, X) \
return --(ST)->recursion_depth,(X)
#define VISIT(ST, TYPE, V) \ #define VISIT(ST, TYPE, V) \
if (!symtable_visit_ ## TYPE((ST), (V))) \ if (!symtable_visit_ ## TYPE((ST), (V))) \
return 0; VISIT_QUIT((ST), 0);
#define VISIT_SEQ(ST, TYPE, SEQ) { \ #define VISIT_SEQ(ST, TYPE, SEQ) { \
int i; \ int i; \
...@@ -1043,7 +1072,7 @@ error: ...@@ -1043,7 +1072,7 @@ error:
for (i = 0; i < asdl_seq_LEN(seq); i++) { \ for (i = 0; i < asdl_seq_LEN(seq); i++) { \
TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
if (!symtable_visit_ ## TYPE((ST), elt)) \ if (!symtable_visit_ ## TYPE((ST), elt)) \
return 0; \ VISIT_QUIT((ST), 0); \
} \ } \
} }
...@@ -1053,7 +1082,7 @@ error: ...@@ -1053,7 +1082,7 @@ error:
for (i = (START); i < asdl_seq_LEN(seq); i++) { \ for (i = (START); i < asdl_seq_LEN(seq); i++) { \
TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
if (!symtable_visit_ ## TYPE((ST), elt)) \ if (!symtable_visit_ ## TYPE((ST), elt)) \
return 0; \ VISIT_QUIT((ST), 0); \
} \ } \
} }
...@@ -1064,7 +1093,7 @@ error: ...@@ -1064,7 +1093,7 @@ error:
expr_ty elt = (expr_ty)asdl_seq_GET(seq, i); \ expr_ty elt = (expr_ty)asdl_seq_GET(seq, i); \
if (!elt) continue; /* can be NULL */ \ if (!elt) continue; /* can be NULL */ \
if (!symtable_visit_expr((ST), elt)) \ if (!symtable_visit_expr((ST), elt)) \
return 0; \ VISIT_QUIT((ST), 0); \
} \ } \
} }
...@@ -1108,32 +1137,37 @@ symtable_record_directive(struct symtable *st, identifier name, stmt_ty s) ...@@ -1108,32 +1137,37 @@ symtable_record_directive(struct symtable *st, identifier name, stmt_ty s)
static int static int
symtable_visit_stmt(struct symtable *st, stmt_ty s) symtable_visit_stmt(struct symtable *st, stmt_ty s)
{ {
if (++st->recursion_depth > st->recursion_limit) {
PyErr_SetString(PyExc_RuntimeError,
"maximum recursion depth exceeded during compilation");
VISIT_QUIT(st, 0);
}
switch (s->kind) { switch (s->kind) {
case FunctionDef_kind: case FunctionDef_kind:
if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL)) if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL))
return 0; VISIT_QUIT(st, 0);
if (s->v.FunctionDef.args->defaults) if (s->v.FunctionDef.args->defaults)
VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
if (s->v.FunctionDef.args->kw_defaults) if (s->v.FunctionDef.args->kw_defaults)
VISIT_KWONLYDEFAULTS(st, VISIT_KWONLYDEFAULTS(st,
s->v.FunctionDef.args->kw_defaults); s->v.FunctionDef.args->kw_defaults);
if (!symtable_visit_annotations(st, s)) if (!symtable_visit_annotations(st, s))
return 0; VISIT_QUIT(st, 0);
if (s->v.FunctionDef.decorator_list) if (s->v.FunctionDef.decorator_list)
VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
if (!symtable_enter_block(st, s->v.FunctionDef.name, if (!symtable_enter_block(st, s->v.FunctionDef.name,
FunctionBlock, (void *)s, s->lineno, FunctionBlock, (void *)s, s->lineno,
s->col_offset)) s->col_offset))
return 0; VISIT_QUIT(st, 0);
VISIT(st, arguments, s->v.FunctionDef.args); VISIT(st, arguments, s->v.FunctionDef.args);
VISIT_SEQ(st, stmt, s->v.FunctionDef.body); VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
if (!symtable_exit_block(st, s)) if (!symtable_exit_block(st, s))
return 0; VISIT_QUIT(st, 0);
break; break;
case ClassDef_kind: { case ClassDef_kind: {
PyObject *tmp; PyObject *tmp;
if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL)) if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL))
return 0; VISIT_QUIT(st, 0);
VISIT_SEQ(st, expr, s->v.ClassDef.bases); VISIT_SEQ(st, expr, s->v.ClassDef.bases);
VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
if (s->v.ClassDef.starargs) if (s->v.ClassDef.starargs)
...@@ -1144,20 +1178,20 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) ...@@ -1144,20 +1178,20 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
(void *)s, s->lineno, s->col_offset)) (void *)s, s->lineno, s->col_offset))
return 0; VISIT_QUIT(st, 0);
if (!GET_IDENTIFIER(__class__) || if (!GET_IDENTIFIER(__class__) ||
!symtable_add_def(st, __class__, DEF_LOCAL) || !symtable_add_def(st, __class__, DEF_LOCAL) ||
!GET_IDENTIFIER(__locals__) || !GET_IDENTIFIER(__locals__) ||
!symtable_add_def(st, __locals__, DEF_PARAM)) { !symtable_add_def(st, __locals__, DEF_PARAM)) {
symtable_exit_block(st, s); symtable_exit_block(st, s);
return 0; VISIT_QUIT(st, 0);
} }
tmp = st->st_private; tmp = st->st_private;
st->st_private = s->v.ClassDef.name; st->st_private = s->v.ClassDef.name;
VISIT_SEQ(st, stmt, s->v.ClassDef.body); VISIT_SEQ(st, stmt, s->v.ClassDef.body);
st->st_private = tmp; st->st_private = tmp;
if (!symtable_exit_block(st, s)) if (!symtable_exit_block(st, s))
return 0; VISIT_QUIT(st, 0);
break; break;
} }
case Return_kind: case Return_kind:
...@@ -1241,7 +1275,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) ...@@ -1241,7 +1275,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
identifier name = (identifier)asdl_seq_GET(seq, i); identifier name = (identifier)asdl_seq_GET(seq, i);
long cur = symtable_lookup(st, name); long cur = symtable_lookup(st, name);
if (cur < 0) if (cur < 0)
return 0; VISIT_QUIT(st, 0);
if (cur & (DEF_LOCAL | USE)) { if (cur & (DEF_LOCAL | USE)) {
char buf[256]; char buf[256];
char *c_name = _PyUnicode_AsString(name); char *c_name = _PyUnicode_AsString(name);
...@@ -1256,12 +1290,12 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) ...@@ -1256,12 +1290,12 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
GLOBAL_AFTER_USE, GLOBAL_AFTER_USE,
c_name); c_name);
if (!symtable_warn(st, buf, s->lineno)) if (!symtable_warn(st, buf, s->lineno))
return 0; VISIT_QUIT(st, 0);
} }
if (!symtable_add_def(st, name, DEF_GLOBAL)) if (!symtable_add_def(st, name, DEF_GLOBAL))
return 0; VISIT_QUIT(st, 0);
if (!symtable_record_directive(st, name, s)) if (!symtable_record_directive(st, name, s))
return 0; VISIT_QUIT(st, 0);
} }
break; break;
} }
...@@ -1272,7 +1306,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) ...@@ -1272,7 +1306,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
identifier name = (identifier)asdl_seq_GET(seq, i); identifier name = (identifier)asdl_seq_GET(seq, i);
long cur = symtable_lookup(st, name); long cur = symtable_lookup(st, name);
if (cur < 0) if (cur < 0)
return 0; VISIT_QUIT(st, 0);
if (cur & (DEF_LOCAL | USE)) { if (cur & (DEF_LOCAL | USE)) {
char buf[256]; char buf[256];
char *c_name = _PyUnicode_AsString(name); char *c_name = _PyUnicode_AsString(name);
...@@ -1287,12 +1321,12 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) ...@@ -1287,12 +1321,12 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
NONLOCAL_AFTER_USE, NONLOCAL_AFTER_USE,
c_name); c_name);
if (!symtable_warn(st, buf, s->lineno)) if (!symtable_warn(st, buf, s->lineno))
return 0; VISIT_QUIT(st, 0);
} }
if (!symtable_add_def(st, name, DEF_NONLOCAL)) if (!symtable_add_def(st, name, DEF_NONLOCAL))
return 0; VISIT_QUIT(st, 0);
if (!symtable_record_directive(st, name, s)) if (!symtable_record_directive(st, name, s))
return 0; VISIT_QUIT(st, 0);
} }
break; break;
} }
...@@ -1309,12 +1343,17 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) ...@@ -1309,12 +1343,17 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_SEQ(st, stmt, s->v.With.body); VISIT_SEQ(st, stmt, s->v.With.body);
break; break;
} }
return 1; VISIT_QUIT(st, 1);
} }
static int static int
symtable_visit_expr(struct symtable *st, expr_ty e) symtable_visit_expr(struct symtable *st, expr_ty e)
{ {
if (++st->recursion_depth > st->recursion_limit) {
PyErr_SetString(PyExc_RuntimeError,
"maximum recursion depth exceeded during compilation");
VISIT_QUIT(st, 0);
}
switch (e->kind) { switch (e->kind) {
case BoolOp_kind: case BoolOp_kind:
VISIT_SEQ(st, expr, e->v.BoolOp.values); VISIT_SEQ(st, expr, e->v.BoolOp.values);
...@@ -1328,7 +1367,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) ...@@ -1328,7 +1367,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
break; break;
case Lambda_kind: { case Lambda_kind: {
if (!GET_IDENTIFIER(lambda)) if (!GET_IDENTIFIER(lambda))
return 0; VISIT_QUIT(st, 0);
if (e->v.Lambda.args->defaults) if (e->v.Lambda.args->defaults)
VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
if (e->v.Lambda.args->kw_defaults) if (e->v.Lambda.args->kw_defaults)
...@@ -1337,11 +1376,11 @@ symtable_visit_expr(struct symtable *st, expr_ty e) ...@@ -1337,11 +1376,11 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
if (!symtable_enter_block(st, lambda, if (!symtable_enter_block(st, lambda,
FunctionBlock, (void *)e, e->lineno, FunctionBlock, (void *)e, e->lineno,
e->col_offset)) e->col_offset))
return 0; VISIT_QUIT(st, 0);
VISIT(st, arguments, e->v.Lambda.args); VISIT(st, arguments, e->v.Lambda.args);
VISIT(st, expr, e->v.Lambda.body); VISIT(st, expr, e->v.Lambda.body);
if (!symtable_exit_block(st, (void *)e)) if (!symtable_exit_block(st, (void *)e))
return 0; VISIT_QUIT(st, 0);
break; break;
} }
case IfExp_kind: case IfExp_kind:
...@@ -1358,19 +1397,19 @@ symtable_visit_expr(struct symtable *st, expr_ty e) ...@@ -1358,19 +1397,19 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
break; break;
case GeneratorExp_kind: case GeneratorExp_kind:
if (!symtable_visit_genexp(st, e)) if (!symtable_visit_genexp(st, e))
return 0; VISIT_QUIT(st, 0);
break; break;
case ListComp_kind: case ListComp_kind:
if (!symtable_visit_listcomp(st, e)) if (!symtable_visit_listcomp(st, e))
return 0; VISIT_QUIT(st, 0);
break; break;
case SetComp_kind: case SetComp_kind:
if (!symtable_visit_setcomp(st, e)) if (!symtable_visit_setcomp(st, e))
return 0; VISIT_QUIT(st, 0);
break; break;
case DictComp_kind: case DictComp_kind:
if (!symtable_visit_dictcomp(st, e)) if (!symtable_visit_dictcomp(st, e))
return 0; VISIT_QUIT(st, 0);
break; break;
case Yield_kind: case Yield_kind:
case YieldFrom_kind: { case YieldFrom_kind: {
...@@ -1414,14 +1453,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) ...@@ -1414,14 +1453,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
case Name_kind: case Name_kind:
if (!symtable_add_def(st, e->v.Name.id, if (!symtable_add_def(st, e->v.Name.id,
e->v.Name.ctx == Load ? USE : DEF_LOCAL)) e->v.Name.ctx == Load ? USE : DEF_LOCAL))
return 0; VISIT_QUIT(st, 0);
/* Special-case super: it counts as a use of __class__ */ /* Special-case super: it counts as a use of __class__ */
if (e->v.Name.ctx == Load && if (e->v.Name.ctx == Load &&
st->st_cur->ste_type == FunctionBlock && st->st_cur->ste_type == FunctionBlock &&
!PyUnicode_CompareWithASCIIString(e->v.Name.id, "super")) { !PyUnicode_CompareWithASCIIString(e->v.Name.id, "super")) {
if (!GET_IDENTIFIER(__class__) || if (!GET_IDENTIFIER(__class__) ||
!symtable_add_def(st, __class__, USE)) !symtable_add_def(st, __class__, USE))
return 0; VISIT_QUIT(st, 0);
} }
break; break;
/* child nodes of List and Tuple will have expr_context set */ /* child nodes of List and Tuple will have expr_context set */
...@@ -1432,7 +1471,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) ...@@ -1432,7 +1471,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
VISIT_SEQ(st, expr, e->v.Tuple.elts); VISIT_SEQ(st, expr, e->v.Tuple.elts);
break; break;
} }
return 1; VISIT_QUIT(st, 1);
} }
static int static int
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment