Kaydet (Commit) cb41b276 authored tarafından INADA Naoki's avatar INADA Naoki Kaydeden (comit) Victor Stinner

bpo-29463: Add docstring field to some AST nodes. (#46)

* bpo-29463: Add docstring field to some AST nodes.

ClassDef, ModuleDef, FunctionDef, and AsyncFunctionDef has docstring
field for now.  It was first statement of there body.

* fix document.  thanks travis!

* doc fixes
üst 1bc15643
......@@ -133,9 +133,17 @@ and classes for traversing abstract syntax trees:
.. function:: get_docstring(node, clean=True)
Return the docstring of the given *node* (which must be a
:class:`FunctionDef`, :class:`ClassDef` or :class:`Module` node), or ``None``
if it has no docstring. If *clean* is true, clean up the docstring's
indentation with :func:`inspect.cleandoc`.
:class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`,
or :class:`Module` node), or ``None`` if it has no docstring.
If *clean* is true, clean up the docstring's indentation with
:func:`inspect.cleandoc`.
.. versionchanged:: 3.5
:class:`AsyncFunctionDef` is now supported.
.. versionchanged:: 3.7
The docstring is now exported from the node docstring field, instead of
the first body statement.
.. function:: fix_missing_locations(node)
......
......@@ -196,6 +196,12 @@ Changes in the Python API
Use the :meth:`~http.cookies.Morsel.set` method for setting them.
(Contributed by Serhiy Storchaka in :issue:`29192`.)
* ``Module``, ``FunctionDef``, ``AsyncFunctionDef``, and
``ClassDef`` AST nodes now have a new ``docstring`` field.
The first statement in their body is not considered as a docstring
anymore. ``co_firstlineno`` and ``co_lnotab`` of code object for class
and module are affected by this change.
(Contributed by INADA Naoki and Eugene Toder in :issue:`29463`.)
CPython bytecode changes
------------------------
......
......@@ -46,6 +46,7 @@ struct _mod {
union {
struct {
asdl_seq *body;
string docstring;
} Module;
struct {
......@@ -80,6 +81,7 @@ struct _stmt {
asdl_seq *body;
asdl_seq *decorator_list;
expr_ty returns;
string docstring;
} FunctionDef;
struct {
......@@ -88,6 +90,7 @@ struct _stmt {
asdl_seq *body;
asdl_seq *decorator_list;
expr_ty returns;
string docstring;
} AsyncFunctionDef;
struct {
......@@ -96,6 +99,7 @@ struct _stmt {
asdl_seq *keywords;
asdl_seq *body;
asdl_seq *decorator_list;
string docstring;
} ClassDef;
struct {
......@@ -439,26 +443,27 @@ struct _withitem {
};
#define Module(a0, a1) _Py_Module(a0, a1)
mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
#define Module(a0, a1, a2) _Py_Module(a0, a1, a2)
mod_ty _Py_Module(asdl_seq * body, string docstring, PyArena *arena);
#define Interactive(a0, a1) _Py_Interactive(a0, a1)
mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
#define Expression(a0, a1) _Py_Expression(a0, a1)
mod_ty _Py_Expression(expr_ty body, PyArena *arena);
#define Suite(a0, a1) _Py_Suite(a0, a1)
mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorator_list, expr_ty returns, int lineno,
int col_offset, PyArena *arena);
#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
asdl_seq * decorator_list, expr_ty returns, string
docstring, int lineno, int col_offset, PyArena *arena);
#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
body, asdl_seq * decorator_list, expr_ty returns,
int lineno, int col_offset, PyArena *arena);
#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
string docstring, int lineno, int col_offset,
PyArena *arena);
#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
asdl_seq * body, asdl_seq * decorator_list, int lineno,
int col_offset, PyArena *arena);
asdl_seq * body, asdl_seq * decorator_list, string
docstring, int lineno, int col_offset, PyArena *arena);
#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
......
......@@ -197,15 +197,7 @@ def get_docstring(node, clean=True):
"""
if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
raise TypeError("%r can't have docstrings" % node.__class__.__name__)
if not(node.body and isinstance(node.body[0], Expr)):
return
node = node.body[0].value
if isinstance(node, Str):
text = node.s
elif isinstance(node, Constant) and isinstance(node.value, str):
text = node.value
else:
return
text = node.docstring
if clean:
import inspect
text = inspect.cleandoc(text)
......
This diff is collapsed.
......@@ -27,7 +27,7 @@ class OpcodeTest(unittest.TestCase):
with open(ann_module.__file__) as f:
txt = f.read()
co = compile(txt, ann_module.__file__, 'exec')
self.assertEqual(co.co_firstlineno, 6)
self.assertEqual(co.co_firstlineno, 8)
except OSError:
pass
......
......@@ -19,6 +19,10 @@ Core and Builtins
- bpo-29347: Fixed possibly dereferencing undefined pointers
when creating weakref objects.
- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef,
and AsyncFunctionDef ast nodes. docstring is not first stmt in their body
anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object
for module and class.
- bpo-29438: Fixed use-after-free problem in key sharing dict.
......
......@@ -6,7 +6,7 @@
module Python
{
mod = Module(stmt* body)
mod = Module(stmt* body, string? docstring)
| Interactive(stmt* body)
| Expression(expr body)
......@@ -14,15 +14,18 @@ module Python
| Suite(stmt* body)
stmt = FunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns)
stmt* body, expr* decorator_list, expr? returns,
string? docstring)
| AsyncFunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns)
stmt* body, expr* decorator_list, expr? returns,
string? docstring)
| ClassDef(identifier name,
expr* bases,
keyword* keywords,
stmt* body,
expr* decorator_list)
expr* decorator_list,
string? docstring)
| Return(expr? value)
| Delete(expr* targets)
......
This diff is collapsed.
This diff is collapsed.
......@@ -1324,18 +1324,6 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
} \
}
static int
compiler_isdocstring(stmt_ty s)
{
if (s->kind != Expr_kind)
return 0;
if (s->v.Expr.value->kind == Str_kind)
return 1;
if (s->v.Expr.value->kind == Constant_kind)
return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
return 0;
}
static int
is_const(expr_ty e)
{
......@@ -1435,36 +1423,27 @@ find_ann(asdl_seq *stmts)
and for annotations. */
static int
compiler_body(struct compiler *c, asdl_seq *stmts)
compiler_body(struct compiler *c, asdl_seq *stmts, string docstring)
{
int i = 0;
stmt_ty st;
/* Set current line number to the line number of first statement.
This way line number for SETUP_ANNOTATIONS will always
coincide with the line number of first "real" statement in module.
If body is empy, then lineno will be set later in assemble. */
if (c->u->u_scope_type == COMPILER_SCOPE_MODULE &&
!c->u->u_lineno && asdl_seq_LEN(stmts)) {
st = (stmt_ty)asdl_seq_GET(stmts, 0);
stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
c->u->u_lineno = st->lineno;
}
/* Every annotated class and module should have __annotations__. */
if (find_ann(stmts)) {
ADDOP(c, SETUP_ANNOTATIONS);
}
if (!asdl_seq_LEN(stmts))
return 1;
st = (stmt_ty)asdl_seq_GET(stmts, 0);
if (compiler_isdocstring(st) && c->c_optimize < 2) {
/* don't generate docstrings if -OO */
i = 1;
VISIT(c, expr, st->v.Expr.value);
if (!compiler_nameop(c, __doc__, Store))
return 0;
/* if not -OO mode, set docstring */
if (c->c_optimize < 2 && docstring) {
ADDOP_O(c, LOAD_CONST, docstring, consts);
ADDOP_NAME(c, STORE_NAME, __doc__, names);
}
for (; i < asdl_seq_LEN(stmts); i++)
VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
VISIT_SEQ(c, stmt, stmts);
return 1;
}
......@@ -1484,7 +1463,7 @@ compiler_mod(struct compiler *c, mod_ty mod)
return NULL;
switch (mod->kind) {
case Module_kind:
if (!compiler_body(c, mod->v.Module.body)) {
if (!compiler_body(c, mod->v.Module.body, mod->v.Module.docstring)) {
compiler_exit_scope(c);
return 0;
}
......@@ -1812,15 +1791,13 @@ static int
compiler_function(struct compiler *c, stmt_ty s, int is_async)
{
PyCodeObject *co;
PyObject *qualname, *first_const = Py_None;
PyObject *qualname, *docstring = Py_None;
arguments_ty args;
expr_ty returns;
identifier name;
asdl_seq* decos;
asdl_seq *body;
stmt_ty st;
Py_ssize_t i, n, funcflags;
int docstring;
Py_ssize_t i, funcflags;
int annotations;
int scope_type;
......@@ -1866,27 +1843,18 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
return 0;
}
st = (stmt_ty)asdl_seq_GET(body, 0);
docstring = compiler_isdocstring(st);
if (docstring && c->c_optimize < 2) {
if (st->v.Expr.value->kind == Constant_kind)
first_const = st->v.Expr.value->v.Constant.value;
else
first_const = st->v.Expr.value->v.Str.s;
}
if (compiler_add_o(c, c->u->u_consts, first_const) < 0) {
/* if not -OO mode, add docstring */
if (c->c_optimize < 2 && s->v.FunctionDef.docstring)
docstring = s->v.FunctionDef.docstring;
if (compiler_add_o(c, c->u->u_consts, docstring) < 0) {
compiler_exit_scope(c);
return 0;
}
c->u->u_argcount = asdl_seq_LEN(args->args);
c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
n = asdl_seq_LEN(body);
/* if there was a docstring, we need to skip the first statement */
for (i = docstring; i < n; i++) {
st = (stmt_ty)asdl_seq_GET(body, i);
VISIT_IN_SCOPE(c, stmt, st);
}
VISIT_SEQ_IN_SCOPE(c, stmt, body);
co = assemble(c, 1);
qualname = c->u->u_qualname;
Py_INCREF(qualname);
......@@ -1967,7 +1935,7 @@ compiler_class(struct compiler *c, stmt_ty s)
}
Py_DECREF(str);
/* compile the body proper */
if (!compiler_body(c, s->v.ClassDef.body)) {
if (!compiler_body(c, s->v.ClassDef.body, s->v.ClassDef.docstring)) {
compiler_exit_scope(c);
return 0;
}
......
......@@ -61,7 +61,6 @@ static int
future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
{
int i, done = 0, prev_line = 0;
stmt_ty first;
if (!(mod->kind == Module_kind || mod->kind == Interactive_kind))
return 1;
......@@ -77,16 +76,7 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
but is preceded by a regular import.
*/
i = 0;
first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
if (first->kind == Expr_kind
&& (first->v.Expr.value->kind == Str_kind
|| (first->v.Expr.value->kind == Constant_kind
&& PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value))))
i++;
for (; i < asdl_seq_LEN(mod->v.Module.body); i++) {
for (i = 0; i < asdl_seq_LEN(mod->v.Module.body); i++) {
stmt_ty s = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
if (done && s->lineno > prev_line)
......
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -71,6 +71,8 @@ class Unparser:
########################################################
def _Module(self, tree):
if tree.docstring is not None:
self.fill(repr(tree.docstring))
for stmt in tree.body:
self.dispatch(stmt)
......@@ -235,6 +237,8 @@ class Unparser:
self.write(")")
self.enter()
if t.docstring is not None:
self.fill(repr(t.docstring))
self.dispatch(t.body)
self.leave()
......@@ -257,6 +261,8 @@ class Unparser:
self.write(" -> ")
self.dispatch(t.returns)
self.enter()
if t.docstring is not None:
self.fill(repr(t.docstring))
self.dispatch(t.body)
self.leave()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment