Skip to content

C
cpython
Kaydet (Commit) 5477f529 authored tarafından Jeremy Hylton's avatar Jeremy Hylton
Dosyalara gözat
Seçenekler

Revise implementations of getChildren() and getChildNodes(). 

Add support for floor division (// and //=)

The implementation of getChildren() and getChildNodes() is intended to
be faster, because it avoids calling flatten() on every return value.
But it's not clear that it is a lot faster, because constructing a
tuple with just the right values ends up being slow.  (Too many
attribute lookups probably.)

The ast.txt file is much more complicated, with funny characters at
the ends of names (*, &, !) to indicate the types of each child node.

The astgen script is also much more complex, making me wonder if it's
still useful.
üst 96d68d57
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 324 additions and 118 deletions
Lib/compiler/ast.py
Dosyayı görüntüle @ 5477f529
This diff is collapsed.
Tools/compiler/ast.txt
Dosyayı görüntüle @ 5477f529
Module: doc, node
Stmt: nodes
Function: name, argnames, defaults, flags, doc, code
Lambda: argnames, defaults, flags, code
Class: name, bases, doc, code
# This file describes the nodes of the AST in ast.py. The module is
# generated by astgen.py.
# The descriptions use the following special notation to describe
# properties of the children:
# * this child is not a node
# ! this child is a sequence that contains nodes in it
# & this child may be set to None
# = ... a default value for the node constructor (optional args)
Module: doc*, node
Stmt: nodes!
Function: name*, argnames*, defaults!, flags*, doc*, code
Lambda: argnames*, defaults!, flags*, code
Class: name*, bases!, doc*, code
Pass:
Break:
Continue:
For: assign, list, body, else_
While: test, body, else_
If: tests, else_
Exec: expr, locals, globals
From: modname, names
Import: names
Raise: expr1, expr2, expr3
For: assign, list, body, else_&
While: test, body, else_&
If: tests!, else_&
Exec: expr, locals&, globals&
From: modname*, names*
Import: names*
Raise: expr1&, expr2&, expr3&
TryFinally: body, final
TryExcept: body, handlers, else_
TryExcept: body, handlers!, else_&
Return: value
Yield: value
Const: value
Print: nodes, dest
Printnl: nodes, dest
Const: value*
Print: nodes!, dest&
Printnl: nodes!, dest&
Discard: expr
AugAssign: node, op, expr
Assign: nodes, expr
AssTuple: nodes
AssList: nodes
AssName: name, flags
AssAttr: expr, attrname, flags
ListComp: expr, quals
ListCompFor: assign, list, ifs
AugAssign: node, op*, expr
Assign: nodes!, expr
AssTuple: nodes!
AssList: nodes!
AssName: name*, flags*
AssAttr: expr, attrname*, flags*
ListComp: expr, quals!
ListCompFor: assign, list, ifs!
ListCompIf: test
List: nodes
Dict: items
List: nodes!
Dict: items!
Not: expr
Compare: expr, ops
Name: name
Compare: expr, ops!
Name: name*
Global: names
Backquote: expr
Getattr: expr, attrname
CallFunc: node, args, star_args = None, dstar_args = None
Keyword: name, expr
Subscript: expr, flags, subs
Getattr: expr, attrname*
CallFunc: node, args!, star_args& = None, dstar_args& = None
Keyword: name*, expr
Subscript: expr, flags*, subs!
Ellipsis:
Sliceobj: nodes
Slice: expr, flags, lower, upper
Assert: test, fail
Tuple: nodes
Or: nodes
And: nodes
Bitor: nodes
Bitxor: nodes
Bitand: nodes
Sliceobj: nodes!
Slice: expr, flags*, lower&, upper&
Assert: test, fail&
Tuple: nodes!
Or: nodes!
And: nodes!
Bitor: nodes!
Bitxor: nodes!
Bitand: nodes!
LeftShift: (left, right)
RightShift: (left, right)
Add: (left, right)
......@@ -59,6 +67,7 @@ Mul: (left, right)
Div: (left, right)
Mod: (left, right)
Power: (left, right)
FloorDiv: (left, right)
UnaryAdd: expr
UnarySub: expr
Invert: expr
......
Tools/compiler/astgen.py
Dosyayı görüntüle @ 5477f529
"""Generate ast module from specification"""
"""Generate ast module from specification
This script generates the ast module from a simple specification,
which makes it easy to accomodate changes in the grammar. This
approach would be quite reasonable if the grammar changed often.
Instead, it is rather complex to generate the appropriate code. And
the Node interface has changed more often than the grammar.
"""
import fileinput
import getopt
......@@ -24,7 +31,13 @@ def strip_default(arg):
i = arg.find('=')
if i == -1:
return arg
return arg[:i].strip()
t = arg[:i].strip()
return t
P_NODE = 1
P_OTHER = 2
P_NESTED = 3
P_NONE = 4
class NodeInfo:
"""Each instance describes a specific AST node"""
......@@ -32,9 +45,8 @@ class NodeInfo:
self.name = name
self.args = args.strip()
self.argnames = self.get_argnames()
self.argprops = self.get_argprops()
self.nargs = len(self.argnames)
self.children = COMMA.join(["self.%s" % c
for c in self.argnames])
self.init = []
def get_argnames(self):
......@@ -47,12 +59,48 @@ class NodeInfo:
return [strip_default(arg.strip())
for arg in args.split(',') if arg]
def get_argprops(self):
"""Each argument can have a property like '*' or '!'
XXX This method modifies the argnames in place!
"""
d = {}
hardest_arg = P_NODE
for i in range(len(self.argnames)):
arg = self.argnames[i]
if arg.endswith('*'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_OTHER
hardest_arg = P_OTHER
elif arg.endswith('!'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NESTED
hardest_arg = P_NESTED
elif arg.endswith('&'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NONE
hardest_arg = P_NONE
else:
d[arg] = P_NODE
self.hardest_arg = hardest_arg
if hardest_arg > P_NODE:
self.args = self.args.replace('*', '')
self.args = self.args.replace('!', '')
self.args = self.args.replace('&', '')
return d
def gen_source(self):
buf = StringIO()
print >> buf, "class %s(Node):" % self.name
print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name)
self._gen_init(buf)
print >> buf
self._gen_getChildren(buf)
print >> buf
self._gen_getChildNodes(buf)
print >> buf
self._gen_repr(buf)
buf.seek(0, 0)
return buf.read()
......@@ -68,14 +116,57 @@ class NodeInfo:
print >> buf, "".join([" " + line for line in self.init])
def _gen_getChildren(self, buf):
print >> buf, " def _getChildren(self):"
if self.argnames:
if self.nargs == 1:
print >> buf, " return %s," % self.children
else:
print >> buf, " return %s" % self.children
print >> buf, " def getChildren(self):"
if len(self.argnames) == 0:
print >> buf, " return ()"
else:
if self.hardest_arg < P_NESTED:
clist = COMMA.join(["self.%s" % c
for c in self.argnames])
if self.nargs == 1:
print >> buf, " return %s," % clist
else:
print >> buf, " return %s" % clist
else:
print >> buf, " children = []"
template = " children.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten(",
name, ")")
else:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(children)"
def _gen_getChildNodes(self, buf):
print >> buf, " def getChildNodes(self):"
if len(self.argnames) == 0:
print >> buf, " return ()"
else:
if self.hardest_arg < P_NESTED:
clist = ["self.%s" % c
for c in self.argnames
if self.argprops[c] == P_NODE]
if len(clist) == 0:
print >> buf, " return ()"
elif len(clist) == 1:
print >> buf, " return %s," % clist[0]
else:
print >> buf, " return %s" % COMMA.join(clist)
else:
print >> buf, " nodes = []"
template = " nodes.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NONE:
tmp = (" if self.%s is not None:"
" nodes.append(self.%s)")
print >> buf, tmp % (name, name)
elif self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten_nodes(",
name, ")")
elif self.argprops[name] == P_NODE:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(nodes)"
def _gen_repr(self, buf):
print >> buf, " def __repr__(self):"
......@@ -98,6 +189,8 @@ def parse_spec(file):
classes = {}
cur = None
for line in fileinput.input(file):
if line.strip().startswith('#'):
continue
mo = rx_init.search(line)
if mo is None:
if cur is None:
......@@ -149,6 +242,9 @@ def flatten(list):
l.append(elt)
return l
def flatten_nodes(list):
return [n for n in flatten(list) if isinstance(n, Node)]
def asList(nodes):
l = []
for item in nodes:
......@@ -164,21 +260,19 @@ def asList(nodes):
nodes = {}
class Node:
lineno = None
class Node: # an abstract base class
lineno = None # provide a lineno for nodes that don't have one
def getType(self):
pass
pass # implemented by subclass
def getChildren(self):
# XXX It would be better to generate flat values to begin with
return flatten(self._getChildren())
pass # implemented by subclasses
def asList(self):
return tuple(asList(self.getChildren()))
def getChildNodes(self):
return [n for n in self.getChildren() if isinstance(n, Node)]
pass # implemented by subclasses
class EmptyNode(Node):
def __init__(self):
self.lineno = None
pass
### EPILOGUE
klasses = globals()
......
Tools/compiler/compiler/ast.py
Dosyayı görüntüle @ 5477f529
This diff is collapsed.
Tools/compiler/compiler/ast.txt
Dosyayı görüntüle @ 5477f529
Module: doc, node
Stmt: nodes
Function: name, argnames, defaults, flags, doc, code
Lambda: argnames, defaults, flags, code
Class: name, bases, doc, code
# This file describes the nodes of the AST in ast.py. The module is
# generated by astgen.py.
# The descriptions use the following special notation to describe
# properties of the children:
# * this child is not a node
# ! this child is a sequence that contains nodes in it
# & this child may be set to None
# = ... a default value for the node constructor (optional args)
Module: doc*, node
Stmt: nodes!
Function: name*, argnames*, defaults!, flags*, doc*, code
Lambda: argnames*, defaults!, flags*, code
Class: name*, bases!, doc*, code
Pass:
Break:
Continue:
For: assign, list, body, else_
While: test, body, else_
If: tests, else_
Exec: expr, locals, globals
From: modname, names
Import: names
Raise: expr1, expr2, expr3
For: assign, list, body, else_&
While: test, body, else_&
If: tests!, else_&
Exec: expr, locals&, globals&
From: modname*, names*
Import: names*
Raise: expr1&, expr2&, expr3&
TryFinally: body, final
TryExcept: body, handlers, else_
TryExcept: body, handlers!, else_&
Return: value
Yield: value
Const: value
Print: nodes, dest
Printnl: nodes, dest
Const: value*
Print: nodes!, dest&
Printnl: nodes!, dest&
Discard: expr
AugAssign: node, op, expr
Assign: nodes, expr
AssTuple: nodes
AssList: nodes
AssName: name, flags
AssAttr: expr, attrname, flags
ListComp: expr, quals
ListCompFor: assign, list, ifs
AugAssign: node, op*, expr
Assign: nodes!, expr
AssTuple: nodes!
AssList: nodes!
AssName: name*, flags*
AssAttr: expr, attrname*, flags*
ListComp: expr, quals!
ListCompFor: assign, list, ifs!
ListCompIf: test
List: nodes
Dict: items
List: nodes!
Dict: items!
Not: expr
Compare: expr, ops
Name: name
Compare: expr, ops!
Name: name*
Global: names
Backquote: expr
Getattr: expr, attrname
CallFunc: node, args, star_args = None, dstar_args = None
Keyword: name, expr
Subscript: expr, flags, subs
Getattr: expr, attrname*
CallFunc: node, args!, star_args& = None, dstar_args& = None
Keyword: name*, expr
Subscript: expr, flags*, subs!
Ellipsis:
Sliceobj: nodes
Slice: expr, flags, lower, upper
Assert: test, fail
Tuple: nodes
Or: nodes
And: nodes
Bitor: nodes
Bitxor: nodes
Bitand: nodes
Sliceobj: nodes!
Slice: expr, flags*, lower&, upper&
Assert: test, fail&
Tuple: nodes!
Or: nodes!
And: nodes!
Bitor: nodes!
Bitxor: nodes!
Bitand: nodes!
LeftShift: (left, right)
RightShift: (left, right)
Add: (left, right)
......@@ -59,6 +67,7 @@ Mul: (left, right)
Div: (left, right)
Mod: (left, right)
Power: (left, right)
FloorDiv: (left, right)
UnaryAdd: expr
UnarySub: expr
Invert: expr
......
Tools/compiler/compiler/astgen.py
Dosyayı görüntüle @ 5477f529
"""Generate ast module from specification"""
"""Generate ast module from specification
This script generates the ast module from a simple specification,
which makes it easy to accomodate changes in the grammar. This
approach would be quite reasonable if the grammar changed often.
Instead, it is rather complex to generate the appropriate code. And
the Node interface has changed more often than the grammar.
"""
import fileinput
import getopt
......@@ -24,7 +31,13 @@ def strip_default(arg):
i = arg.find('=')
if i == -1:
return arg
return arg[:i].strip()
t = arg[:i].strip()
return t
P_NODE = 1
P_OTHER = 2
P_NESTED = 3
P_NONE = 4
class NodeInfo:
"""Each instance describes a specific AST node"""
......@@ -32,9 +45,8 @@ class NodeInfo:
self.name = name
self.args = args.strip()
self.argnames = self.get_argnames()
self.argprops = self.get_argprops()
self.nargs = len(self.argnames)
self.children = COMMA.join(["self.%s" % c
for c in self.argnames])
self.init = []
def get_argnames(self):
......@@ -47,12 +59,48 @@ class NodeInfo:
return [strip_default(arg.strip())
for arg in args.split(',') if arg]
def get_argprops(self):
"""Each argument can have a property like '*' or '!'
XXX This method modifies the argnames in place!
"""
d = {}
hardest_arg = P_NODE
for i in range(len(self.argnames)):
arg = self.argnames[i]
if arg.endswith('*'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_OTHER
hardest_arg = P_OTHER
elif arg.endswith('!'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NESTED
hardest_arg = P_NESTED
elif arg.endswith('&'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NONE
hardest_arg = P_NONE
else:
d[arg] = P_NODE
self.hardest_arg = hardest_arg
if hardest_arg > P_NODE:
self.args = self.args.replace('*', '')
self.args = self.args.replace('!', '')
self.args = self.args.replace('&', '')
return d
def gen_source(self):
buf = StringIO()
print >> buf, "class %s(Node):" % self.name
print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name)
self._gen_init(buf)
print >> buf
self._gen_getChildren(buf)
print >> buf
self._gen_getChildNodes(buf)
print >> buf
self._gen_repr(buf)
buf.seek(0, 0)
return buf.read()
......@@ -68,14 +116,57 @@ class NodeInfo:
print >> buf, "".join([" " + line for line in self.init])
def _gen_getChildren(self, buf):
print >> buf, " def _getChildren(self):"
if self.argnames:
if self.nargs == 1:
print >> buf, " return %s," % self.children
else:
print >> buf, " return %s" % self.children
print >> buf, " def getChildren(self):"
if len(self.argnames) == 0:
print >> buf, " return ()"
else:
if self.hardest_arg < P_NESTED:
clist = COMMA.join(["self.%s" % c
for c in self.argnames])
if self.nargs == 1:
print >> buf, " return %s," % clist
else:
print >> buf, " return %s" % clist
else:
print >> buf, " children = []"
template = " children.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten(",
name, ")")
else:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(children)"
def _gen_getChildNodes(self, buf):
print >> buf, " def getChildNodes(self):"
if len(self.argnames) == 0:
print >> buf, " return ()"
else:
if self.hardest_arg < P_NESTED:
clist = ["self.%s" % c
for c in self.argnames
if self.argprops[c] == P_NODE]
if len(clist) == 0:
print >> buf, " return ()"
elif len(clist) == 1:
print >> buf, " return %s," % clist[0]
else:
print >> buf, " return %s" % COMMA.join(clist)
else:
print >> buf, " nodes = []"
template = " nodes.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NONE:
tmp = (" if self.%s is not None:"
" nodes.append(self.%s)")
print >> buf, tmp % (name, name)
elif self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten_nodes(",
name, ")")
elif self.argprops[name] == P_NODE:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(nodes)"
def _gen_repr(self, buf):
print >> buf, " def __repr__(self):"
......@@ -98,6 +189,8 @@ def parse_spec(file):
classes = {}
cur = None
for line in fileinput.input(file):
if line.strip().startswith('#'):
continue
mo = rx_init.search(line)
if mo is None:
if cur is None:
......@@ -149,6 +242,9 @@ def flatten(list):
l.append(elt)
return l
def flatten_nodes(list):
return [n for n in flatten(list) if isinstance(n, Node)]
def asList(nodes):
l = []
for item in nodes:
......@@ -164,21 +260,19 @@ def asList(nodes):
nodes = {}
class Node:
lineno = None
class Node: # an abstract base class
lineno = None # provide a lineno for nodes that don't have one
def getType(self):
pass
pass # implemented by subclass
def getChildren(self):
# XXX It would be better to generate flat values to begin with
return flatten(self._getChildren())
pass # implemented by subclasses
def asList(self):
return tuple(asList(self.getChildren()))
def getChildNodes(self):
return [n for n in self.getChildren() if isinstance(n, Node)]
pass # implemented by subclasses
class EmptyNode(Node):
def __init__(self):
self.lineno = None
pass
### EPILOGUE
klasses = globals()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment