Skip to content

C
cpython
Kaydet (Commit) 5477f529 authored tarafından Jeremy Hylton's avatar Jeremy Hylton
Dosyalara gözat
Seçenekler

Revise implementations of getChildren() and getChildNodes(). 

Add support for floor division (// and //=)

The implementation of getChildren() and getChildNodes() is intended to
be faster, because it avoids calling flatten() on every return value.
But it's not clear that it is a lot faster, because constructing a
tuple with just the right values ends up being slow.  (Too many
attribute lookups probably.)

The ast.txt file is much more complicated, with funny characters at
the ends of names (*, &, !) to indicate the types of each child node.

The astgen script is also much more complex, making me wonder if it's
still useful.
üst 96d68d57
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 324 additions and 118 deletions
Lib/compiler/ast.py
Dosyayı görüntüle @ 5477f529
This diff is collapsed.
Tools/compiler/ast.txt
Dosyayı görüntüle @ 5477f529
Module: doc, node # This file describes the nodes of the AST in ast.py. The module is
Stmt: nodes # generated by astgen.py.
Function: name, argnames, defaults, flags, doc, code # The descriptions use the following special notation to describe
Lambda: argnames, defaults, flags, code # properties of the children:
Class: name, bases, doc, code # * this child is not a node
# ! this child is a sequence that contains nodes in it
# & this child may be set to None
# = ... a default value for the node constructor (optional args)
Module: doc*, node
Stmt: nodes!
Function: name*, argnames*, defaults!, flags*, doc*, code
Lambda: argnames*, defaults!, flags*, code
Class: name*, bases!, doc*, code
Pass: Pass:
Break: Break:
Continue: Continue:
For: assign, list, body, else_ For: assign, list, body, else_&
While: test, body, else_ While: test, body, else_&
If: tests, else_ If: tests!, else_&
Exec: expr, locals, globals Exec: expr, locals&, globals&
From: modname, names From: modname*, names*
Import: names Import: names*
Raise: expr1, expr2, expr3 Raise: expr1&, expr2&, expr3&
TryFinally: body, final TryFinally: body, final
TryExcept: body, handlers, else_ TryExcept: body, handlers!, else_&
Return: value Return: value
Yield: value Yield: value
Const: value Const: value*
Print: nodes, dest Print: nodes!, dest&
Printnl: nodes, dest Printnl: nodes!, dest&
Discard: expr Discard: expr
AugAssign: node, op, expr AugAssign: node, op*, expr
Assign: nodes, expr Assign: nodes!, expr
AssTuple: nodes AssTuple: nodes!
AssList: nodes AssList: nodes!
AssName: name, flags AssName: name*, flags*
AssAttr: expr, attrname, flags AssAttr: expr, attrname*, flags*
ListComp: expr, quals ListComp: expr, quals!
ListCompFor: assign, list, ifs ListCompFor: assign, list, ifs!
ListCompIf: test ListCompIf: test
List: nodes List: nodes!
Dict: items Dict: items!
Not: expr Not: expr
Compare: expr, ops Compare: expr, ops!
Name: name Name: name*
Global: names Global: names
Backquote: expr Backquote: expr
Getattr: expr, attrname Getattr: expr, attrname*
CallFunc: node, args, star_args = None, dstar_args = None CallFunc: node, args!, star_args& = None, dstar_args& = None
Keyword: name, expr Keyword: name*, expr
Subscript: expr, flags, subs Subscript: expr, flags*, subs!
Ellipsis: Ellipsis:
Sliceobj: nodes Sliceobj: nodes!
Slice: expr, flags, lower, upper Slice: expr, flags*, lower&, upper&
Assert: test, fail Assert: test, fail&
Tuple: nodes Tuple: nodes!
Or: nodes Or: nodes!
And: nodes And: nodes!
Bitor: nodes Bitor: nodes!
Bitxor: nodes Bitxor: nodes!
Bitand: nodes Bitand: nodes!
LeftShift: (left, right) LeftShift: (left, right)
RightShift: (left, right) RightShift: (left, right)
Add: (left, right) Add: (left, right)
...@@ -59,6 +67,7 @@ Mul: (left, right) ...@@ -59,6 +67,7 @@ Mul: (left, right)
Div: (left, right) Div: (left, right)
Mod: (left, right) Mod: (left, right)
Power: (left, right) Power: (left, right)
FloorDiv: (left, right)
UnaryAdd: expr UnaryAdd: expr
UnarySub: expr UnarySub: expr
Invert: expr Invert: expr
......
Tools/compiler/astgen.py
Dosyayı görüntüle @ 5477f529
"""Generate ast module from specification""" """Generate ast module from specification
This script generates the ast module from a simple specification,
which makes it easy to accomodate changes in the grammar. This
approach would be quite reasonable if the grammar changed often.
Instead, it is rather complex to generate the appropriate code. And
the Node interface has changed more often than the grammar.
"""
import fileinput import fileinput
import getopt import getopt
...@@ -24,7 +31,13 @@ def strip_default(arg): ...@@ -24,7 +31,13 @@ def strip_default(arg):
i = arg.find('=') i = arg.find('=')
if i == -1: if i == -1:
return arg return arg
return arg[:i].strip() t = arg[:i].strip()
return t
P_NODE = 1
P_OTHER = 2
P_NESTED = 3
P_NONE = 4
class NodeInfo: class NodeInfo:
"""Each instance describes a specific AST node""" """Each instance describes a specific AST node"""
...@@ -32,9 +45,8 @@ class NodeInfo: ...@@ -32,9 +45,8 @@ class NodeInfo:
self.name = name self.name = name
self.args = args.strip() self.args = args.strip()
self.argnames = self.get_argnames() self.argnames = self.get_argnames()
self.argprops = self.get_argprops()
self.nargs = len(self.argnames) self.nargs = len(self.argnames)
self.children = COMMA.join(["self.%s" % c
for c in self.argnames])
self.init = [] self.init = []
def get_argnames(self): def get_argnames(self):
...@@ -47,12 +59,48 @@ class NodeInfo: ...@@ -47,12 +59,48 @@ class NodeInfo:
return [strip_default(arg.strip()) return [strip_default(arg.strip())
for arg in args.split(',') if arg] for arg in args.split(',') if arg]
def get_argprops(self):
"""Each argument can have a property like '*' or '!'
XXX This method modifies the argnames in place!
"""
d = {}
hardest_arg = P_NODE
for i in range(len(self.argnames)):
arg = self.argnames[i]
if arg.endswith('*'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_OTHER
hardest_arg = P_OTHER
elif arg.endswith('!'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NESTED
hardest_arg = P_NESTED
elif arg.endswith('&'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NONE
hardest_arg = P_NONE
else:
d[arg] = P_NODE
self.hardest_arg = hardest_arg
if hardest_arg > P_NODE:
self.args = self.args.replace('*', '')
self.args = self.args.replace('!', '')
self.args = self.args.replace('&', '')
return d
def gen_source(self): def gen_source(self):
buf = StringIO() buf = StringIO()
print >> buf, "class %s(Node):" % self.name print >> buf, "class %s(Node):" % self.name
print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name) print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name)
self._gen_init(buf) self._gen_init(buf)
print >> buf
self._gen_getChildren(buf) self._gen_getChildren(buf)
print >> buf
self._gen_getChildNodes(buf)
print >> buf
self._gen_repr(buf) self._gen_repr(buf)
buf.seek(0, 0) buf.seek(0, 0)
return buf.read() return buf.read()
...@@ -68,14 +116,57 @@ class NodeInfo: ...@@ -68,14 +116,57 @@ class NodeInfo:
print >> buf, "".join([" " + line for line in self.init]) print >> buf, "".join([" " + line for line in self.init])
def _gen_getChildren(self, buf): def _gen_getChildren(self, buf):
print >> buf, " def _getChildren(self):" print >> buf, " def getChildren(self):"
if self.argnames: if len(self.argnames) == 0:
if self.nargs == 1: print >> buf, " return ()"
print >> buf, " return %s," % self.children
else:
print >> buf, " return %s" % self.children
else: else:
if self.hardest_arg < P_NESTED:
clist = COMMA.join(["self.%s" % c
for c in self.argnames])
if self.nargs == 1:
print >> buf, " return %s," % clist
else:
print >> buf, " return %s" % clist
else:
print >> buf, " children = []"
template = " children.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten(",
name, ")")
else:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(children)"
def _gen_getChildNodes(self, buf):
print >> buf, " def getChildNodes(self):"
if len(self.argnames) == 0:
print >> buf, " return ()" print >> buf, " return ()"
else:
if self.hardest_arg < P_NESTED:
clist = ["self.%s" % c
for c in self.argnames
if self.argprops[c] == P_NODE]
if len(clist) == 0:
print >> buf, " return ()"
elif len(clist) == 1:
print >> buf, " return %s," % clist[0]
else:
print >> buf, " return %s" % COMMA.join(clist)
else:
print >> buf, " nodes = []"
template = " nodes.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NONE:
tmp = (" if self.%s is not None:"
" nodes.append(self.%s)")
print >> buf, tmp % (name, name)
elif self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten_nodes(",
name, ")")
elif self.argprops[name] == P_NODE:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(nodes)"
def _gen_repr(self, buf): def _gen_repr(self, buf):
print >> buf, " def __repr__(self):" print >> buf, " def __repr__(self):"
...@@ -98,6 +189,8 @@ def parse_spec(file): ...@@ -98,6 +189,8 @@ def parse_spec(file):
classes = {} classes = {}
cur = None cur = None
for line in fileinput.input(file): for line in fileinput.input(file):
if line.strip().startswith('#'):
continue
mo = rx_init.search(line) mo = rx_init.search(line)
if mo is None: if mo is None:
if cur is None: if cur is None:
...@@ -149,6 +242,9 @@ def flatten(list): ...@@ -149,6 +242,9 @@ def flatten(list):
l.append(elt) l.append(elt)
return l return l
def flatten_nodes(list):
return [n for n in flatten(list) if isinstance(n, Node)]
def asList(nodes): def asList(nodes):
l = [] l = []
for item in nodes: for item in nodes:
...@@ -164,21 +260,19 @@ def asList(nodes): ...@@ -164,21 +260,19 @@ def asList(nodes):
nodes = {} nodes = {}
class Node: class Node: # an abstract base class
lineno = None lineno = None # provide a lineno for nodes that don't have one
def getType(self): def getType(self):
pass pass # implemented by subclass
def getChildren(self): def getChildren(self):
# XXX It would be better to generate flat values to begin with pass # implemented by subclasses
return flatten(self._getChildren())
def asList(self): def asList(self):
return tuple(asList(self.getChildren())) return tuple(asList(self.getChildren()))
def getChildNodes(self): def getChildNodes(self):
return [n for n in self.getChildren() if isinstance(n, Node)] pass # implemented by subclasses
class EmptyNode(Node): class EmptyNode(Node):
def __init__(self): pass
self.lineno = None
### EPILOGUE ### EPILOGUE
klasses = globals() klasses = globals()
......
Tools/compiler/compiler/ast.py
Dosyayı görüntüle @ 5477f529
This diff is collapsed.
Tools/compiler/compiler/ast.txt
Dosyayı görüntüle @ 5477f529
Module: doc, node # This file describes the nodes of the AST in ast.py. The module is
Stmt: nodes # generated by astgen.py.
Function: name, argnames, defaults, flags, doc, code # The descriptions use the following special notation to describe
Lambda: argnames, defaults, flags, code # properties of the children:
Class: name, bases, doc, code # * this child is not a node
# ! this child is a sequence that contains nodes in it
# & this child may be set to None
# = ... a default value for the node constructor (optional args)
Module: doc*, node
Stmt: nodes!
Function: name*, argnames*, defaults!, flags*, doc*, code
Lambda: argnames*, defaults!, flags*, code
Class: name*, bases!, doc*, code
Pass: Pass:
Break: Break:
Continue: Continue:
For: assign, list, body, else_ For: assign, list, body, else_&
While: test, body, else_ While: test, body, else_&
If: tests, else_ If: tests!, else_&
Exec: expr, locals, globals Exec: expr, locals&, globals&
From: modname, names From: modname*, names*
Import: names Import: names*
Raise: expr1, expr2, expr3 Raise: expr1&, expr2&, expr3&
TryFinally: body, final TryFinally: body, final
TryExcept: body, handlers, else_ TryExcept: body, handlers!, else_&
Return: value Return: value
Yield: value Yield: value
Const: value Const: value*
Print: nodes, dest Print: nodes!, dest&
Printnl: nodes, dest Printnl: nodes!, dest&
Discard: expr Discard: expr
AugAssign: node, op, expr AugAssign: node, op*, expr
Assign: nodes, expr Assign: nodes!, expr
AssTuple: nodes AssTuple: nodes!
AssList: nodes AssList: nodes!
AssName: name, flags AssName: name*, flags*
AssAttr: expr, attrname, flags AssAttr: expr, attrname*, flags*
ListComp: expr, quals ListComp: expr, quals!
ListCompFor: assign, list, ifs ListCompFor: assign, list, ifs!
ListCompIf: test ListCompIf: test
List: nodes List: nodes!
Dict: items Dict: items!
Not: expr Not: expr
Compare: expr, ops Compare: expr, ops!
Name: name Name: name*
Global: names Global: names
Backquote: expr Backquote: expr
Getattr: expr, attrname Getattr: expr, attrname*
CallFunc: node, args, star_args = None, dstar_args = None CallFunc: node, args!, star_args& = None, dstar_args& = None
Keyword: name, expr Keyword: name*, expr
Subscript: expr, flags, subs Subscript: expr, flags*, subs!
Ellipsis: Ellipsis:
Sliceobj: nodes Sliceobj: nodes!
Slice: expr, flags, lower, upper Slice: expr, flags*, lower&, upper&
Assert: test, fail Assert: test, fail&
Tuple: nodes Tuple: nodes!
Or: nodes Or: nodes!
And: nodes And: nodes!
Bitor: nodes Bitor: nodes!
Bitxor: nodes Bitxor: nodes!
Bitand: nodes Bitand: nodes!
LeftShift: (left, right) LeftShift: (left, right)
RightShift: (left, right) RightShift: (left, right)
Add: (left, right) Add: (left, right)
...@@ -59,6 +67,7 @@ Mul: (left, right) ...@@ -59,6 +67,7 @@ Mul: (left, right)
Div: (left, right) Div: (left, right)
Mod: (left, right) Mod: (left, right)
Power: (left, right) Power: (left, right)
FloorDiv: (left, right)
UnaryAdd: expr UnaryAdd: expr
UnarySub: expr UnarySub: expr
Invert: expr Invert: expr
......
Tools/compiler/compiler/astgen.py
Dosyayı görüntüle @ 5477f529
"""Generate ast module from specification""" """Generate ast module from specification
This script generates the ast module from a simple specification,
which makes it easy to accomodate changes in the grammar. This
approach would be quite reasonable if the grammar changed often.
Instead, it is rather complex to generate the appropriate code. And
the Node interface has changed more often than the grammar.
"""
import fileinput import fileinput
import getopt import getopt
...@@ -24,7 +31,13 @@ def strip_default(arg): ...@@ -24,7 +31,13 @@ def strip_default(arg):
i = arg.find('=') i = arg.find('=')
if i == -1: if i == -1:
return arg return arg
return arg[:i].strip() t = arg[:i].strip()
return t
P_NODE = 1
P_OTHER = 2
P_NESTED = 3
P_NONE = 4
class NodeInfo: class NodeInfo:
"""Each instance describes a specific AST node""" """Each instance describes a specific AST node"""
...@@ -32,9 +45,8 @@ class NodeInfo: ...@@ -32,9 +45,8 @@ class NodeInfo:
self.name = name self.name = name
self.args = args.strip() self.args = args.strip()
self.argnames = self.get_argnames() self.argnames = self.get_argnames()
self.argprops = self.get_argprops()
self.nargs = len(self.argnames) self.nargs = len(self.argnames)
self.children = COMMA.join(["self.%s" % c
for c in self.argnames])
self.init = [] self.init = []
def get_argnames(self): def get_argnames(self):
...@@ -47,12 +59,48 @@ class NodeInfo: ...@@ -47,12 +59,48 @@ class NodeInfo:
return [strip_default(arg.strip()) return [strip_default(arg.strip())
for arg in args.split(',') if arg] for arg in args.split(',') if arg]
def get_argprops(self):
"""Each argument can have a property like '*' or '!'
XXX This method modifies the argnames in place!
"""
d = {}
hardest_arg = P_NODE
for i in range(len(self.argnames)):
arg = self.argnames[i]
if arg.endswith('*'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_OTHER
hardest_arg = P_OTHER
elif arg.endswith('!'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NESTED
hardest_arg = P_NESTED
elif arg.endswith('&'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NONE
hardest_arg = P_NONE
else:
d[arg] = P_NODE
self.hardest_arg = hardest_arg
if hardest_arg > P_NODE:
self.args = self.args.replace('*', '')
self.args = self.args.replace('!', '')
self.args = self.args.replace('&', '')
return d
def gen_source(self): def gen_source(self):
buf = StringIO() buf = StringIO()
print >> buf, "class %s(Node):" % self.name print >> buf, "class %s(Node):" % self.name
print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name) print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name)
self._gen_init(buf) self._gen_init(buf)
print >> buf
self._gen_getChildren(buf) self._gen_getChildren(buf)
print >> buf
self._gen_getChildNodes(buf)
print >> buf
self._gen_repr(buf) self._gen_repr(buf)
buf.seek(0, 0) buf.seek(0, 0)
return buf.read() return buf.read()
...@@ -68,14 +116,57 @@ class NodeInfo: ...@@ -68,14 +116,57 @@ class NodeInfo:
print >> buf, "".join([" " + line for line in self.init]) print >> buf, "".join([" " + line for line in self.init])
def _gen_getChildren(self, buf): def _gen_getChildren(self, buf):
print >> buf, " def _getChildren(self):" print >> buf, " def getChildren(self):"
if self.argnames: if len(self.argnames) == 0:
if self.nargs == 1: print >> buf, " return ()"
print >> buf, " return %s," % self.children
else:
print >> buf, " return %s" % self.children
else: else:
if self.hardest_arg < P_NESTED:
clist = COMMA.join(["self.%s" % c
for c in self.argnames])
if self.nargs == 1:
print >> buf, " return %s," % clist
else:
print >> buf, " return %s" % clist
else:
print >> buf, " children = []"
template = " children.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten(",
name, ")")
else:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(children)"
def _gen_getChildNodes(self, buf):
print >> buf, " def getChildNodes(self):"
if len(self.argnames) == 0:
print >> buf, " return ()" print >> buf, " return ()"
else:
if self.hardest_arg < P_NESTED:
clist = ["self.%s" % c
for c in self.argnames
if self.argprops[c] == P_NODE]
if len(clist) == 0:
print >> buf, " return ()"
elif len(clist) == 1:
print >> buf, " return %s," % clist[0]
else:
print >> buf, " return %s" % COMMA.join(clist)
else:
print >> buf, " nodes = []"
template = " nodes.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NONE:
tmp = (" if self.%s is not None:"
" nodes.append(self.%s)")
print >> buf, tmp % (name, name)
elif self.argprops[name] == P_NESTED:
print >> buf, template % ("extend", "flatten_nodes(",
name, ")")
elif self.argprops[name] == P_NODE:
print >> buf, template % ("append", "", name, "")
print >> buf, " return tuple(nodes)"
def _gen_repr(self, buf): def _gen_repr(self, buf):
print >> buf, " def __repr__(self):" print >> buf, " def __repr__(self):"
...@@ -98,6 +189,8 @@ def parse_spec(file): ...@@ -98,6 +189,8 @@ def parse_spec(file):
classes = {} classes = {}
cur = None cur = None
for line in fileinput.input(file): for line in fileinput.input(file):
if line.strip().startswith('#'):
continue
mo = rx_init.search(line) mo = rx_init.search(line)
if mo is None: if mo is None:
if cur is None: if cur is None:
...@@ -149,6 +242,9 @@ def flatten(list): ...@@ -149,6 +242,9 @@ def flatten(list):
l.append(elt) l.append(elt)
return l return l
def flatten_nodes(list):
return [n for n in flatten(list) if isinstance(n, Node)]
def asList(nodes): def asList(nodes):
l = [] l = []
for item in nodes: for item in nodes:
...@@ -164,21 +260,19 @@ def asList(nodes): ...@@ -164,21 +260,19 @@ def asList(nodes):
nodes = {} nodes = {}
class Node: class Node: # an abstract base class
lineno = None lineno = None # provide a lineno for nodes that don't have one
def getType(self): def getType(self):
pass pass # implemented by subclass
def getChildren(self): def getChildren(self):
# XXX It would be better to generate flat values to begin with pass # implemented by subclasses
return flatten(self._getChildren())
def asList(self): def asList(self):
return tuple(asList(self.getChildren())) return tuple(asList(self.getChildren()))
def getChildNodes(self): def getChildNodes(self):
return [n for n in self.getChildren() if isinstance(n, Node)] pass # implemented by subclasses
class EmptyNode(Node): class EmptyNode(Node):
def __init__(self): pass
self.lineno = None
### EPILOGUE ### EPILOGUE
klasses = globals() klasses = globals()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment