New batch from Fred

Lib/AST.py

 """Object-oriented interface to the parser module.
 
-This module exports three classes which together provide an interface
+This module exports four classes which together provide an interface
 to the parser module.  Together, the three classes represent two ways
 to create parsed representations of Python source and the two starting
 data types (source text and tuple representations).  Each class
 provides interfaces which are identical other than the constructors.
 The constructors are described in detail in the documentation for each
 class and the remaining, shared portion of the interface is documented
-below.  Briefly, the three classes provided are:
+below.  Briefly, the classes provided are:
 
 AST
     Defines the primary interface to the AST objects and supports creation
@@ -23,6 +23,9 @@ FileSuiteAST
     Convenience subclass of the `SuiteAST' class; loads source text of the
     suite from an external file.
 
+Common Methods
+--------------
+
 Aside from the constructors, several methods are provided to allow
 access to the various interpretations of the parse tree and to check
 conditions of the construct represented by the parse tree.
@@ -68,8 +71,8 @@ class AST:
     This base class provides all of the query methods for subclass
     objects defined in this module.
     """
-    _p = __import__('parser')		# import internally to avoid
-					# namespace pollution at the
+    import parser			# import internally to avoid
+    _p = parser				# namespace pollution at the
 					# top level
     _text = None
     _code = None
@@ -84,7 +87,8 @@ class AST:
 	    The tuple tree to convert.
 
 	The tuple-tree may represent either an expression or a suite; the
-	type will be determined automatically.
+	type will be determined automatically.  Line number information may
+	optionally be present for any subset of the terminal tokens.
 	"""
 	if type(tuple) is not type(()):
 	    raise TypeError, 'Base AST class requires tuple parameter.'
@@ -93,11 +97,24 @@ class AST:
 	self._ast  = self._p.tuple2ast(tuple)
 	self._type = (self._p.isexpr(self._ast) and 'expression') or 'suite'
 
-    def tuple(self):
+    def list(self, line_info = 0):
+	"""Returns a fresh list representing the parse tree.
+
+	line_info
+	    If true, includes line number information for terminal tokens in
+	    the output data structure,
+	"""
+	return self._p.ast2list(self._ast, line_info)
+
+    def tuple(self, line_info = 0):
 	"""Returns the tuple representing the parse tree.
+
+	line_info
+	    If true, includes line number information for terminal tokens in
+	    the output data structure,
 	"""
 	if self._tupl is None:
-	    self._tupl = self._p.ast2tuple(self._ast)
+	    self._tupl = self._p.ast2tuple(self._ast, line_info)
 	return self._tupl
 
     def code(self):

Lib/symbol.py

-# Non-terminal symbols of Python grammar (from "graminit.h")
+#! /usr/bin/env python
+#
+#  Non-terminal symbols of Python grammar (from "graminit.h")
+#
+#  This file is automatically generated; please don't muck it up!
+#
+#  To update the symbols in this file, 'cd' to the top directory of
+#  the python source tree after building the interpreter and run:
+#
+#    PYTHONPATH=Lib:Modules ./python Lib/symbol.py
+#
+#  (this path allows the import of string.py, token.py, and regexmodule.so
+#  for a site with no installation in place)
 
+#--start constants--
 single_input = 256
 file_input = 257
 eval_input = 258
@@ -23,39 +36,40 @@ raise_stmt = 275
 import_stmt = 276
 dotted_name = 277
 global_stmt = 278
-access_stmt = 279
-accesstype = 280
-exec_stmt = 281
-compound_stmt = 282
-if_stmt = 283
-while_stmt = 284
-for_stmt = 285
-try_stmt = 286
-except_clause = 287
-suite = 288
-test = 289
-and_test = 290
-not_test = 291
-comparison = 292
-comp_op = 293
-expr = 294
-xor_expr = 295
-and_expr = 296
-shift_expr = 297
-arith_expr = 298
-term = 299
-factor = 300
-power = 301
-atom = 302
-lambdef = 303
-trailer = 304
-subscript = 305
+exec_stmt = 279
+compound_stmt = 280
+if_stmt = 281
+while_stmt = 282
+for_stmt = 283
+try_stmt = 284
+except_clause = 285
+suite = 286
+test = 287
+and_test = 288
+not_test = 289
+comparison = 290
+comp_op = 291
+expr = 292
+xor_expr = 293
+and_expr = 294
+shift_expr = 295
+arith_expr = 296
+term = 297
+factor = 298
+power = 299
+atom = 300
+lambdef = 301
+trailer = 302
+subscriptlist = 303
+subscript = 304
+sliceop = 305
 exprlist = 306
 testlist = 307
 dictmaker = 308
 classdef = 309
 arglist = 310
 argument = 311
+#--end constants--
 
 names = dir()
 sym_name = {}
@@ -63,3 +77,17 @@ for name in names:
     number = eval(name)
     if type(number) is type(0):
 	sym_name[number] = name
+
+
+def main():
+    import sys
+    import token
+    if len(sys.argv) == 1:
+	sys.argv = sys.argv + ["Include/graminit.h", "Lib/symbol.py"]
+    token.main()
+
+if __name__ == "__main__":
+    main()
+
+#
+#  end of file

Lib/token.py

-# Tokens (from "token.h")
+#! /usr/bin/env python
+#
+#  Tokens (from "token.h")
+#
+#  This file is automatically generated; please don't muck it up!
+#
+#  To update the symbols in this file, 'cd' to the top directory of
+#  the python source tree after building the interpreter and run:
+#
+#    PYTHONPATH=./Lib ./python Lib/token.py
+#
+#  (this path allows the import of string.py and regexmodule.so
+#  for a site with no installation in place)
 
+#--start constants--
 ENDMARKER = 0
 NAME = 1
 NUMBER = 2
@@ -39,6 +52,9 @@ RIGHTSHIFT = 35
 DOUBLESTAR = 36
 OP = 37
 ERRORTOKEN = 38
+N_TOKENS = 39
+NT_OFFSET = 256
+#--end constants--
 
 names = dir()
 tok_name = {}
@@ -47,9 +63,6 @@ for name in names:
     if type(number) is type(0):
 	tok_name[number] = name
 
-N_TOKENS = 39				# Number of tokens including ERRORTOKEN
-NT_OFFSET = 256				# Start of non-terminal symbols
-
 def ISTERMINAL(x):
     return x < NT_OFFSET
 
@@ -58,3 +71,61 @@ def ISNONTERMINAL(x):
 
 def ISEOF(x):
     return x == ENDMARKER
+
+
+def main():
+    import regex
+    import string
+    import sys
+    args = sys.argv[1:]
+    inFileName = args and args[0] or "Include/token.h"
+    outFileName = "Lib/token.py"
+    if len(args) > 1:
+	outFileName = args[1]
+    try:
+	fp = open(inFileName)
+    except IOError, err:
+	sys.stdout.write("I/O error: %s\n" % str(err))
+	sys.exit(1)
+    lines = string.splitfields(fp.read(), "\n")
+    fp.close()
+    re = regex.compile(
+	"#define[ \t][ \t]*\([A-Z][A-Z_]*\)[ \t][ \t]*\([0-9][0-9]*\)",
+	regex.casefold)
+    tokens = {}
+    for line in lines:
+	if re.match(line) > -1:
+	    name, val = re.group(1, 2)
+	    val = string.atoi(val)
+	    tokens[val] = name		# reverse so we can sort them...
+    keys = tokens.keys()
+    keys.sort()
+    # load the output skeleton from the target:
+    try:
+	fp = open(outFileName)
+    except IOError, err:
+	sys.stderr.write("I/O error: %s\n" % str(err))
+	sys.exit(2)
+    format = string.splitfields(fp.read(), "\n")
+    fp.close()
+    try:
+	start = format.index("#--start constants--") + 1
+	end = format.index("#--end constants--")
+    except ValueError:
+	sys.stderr.write("target does not contain format markers")
+	sys.exit(3)
+    lines = []
+    for val in keys:
+	lines.append("%s = %d" % (tokens[val], val))
+    format[start:end] = lines
+    try:
+	fp = open(outFileName, 'w')
+    except IOError, err:
+	sys.stderr.write("I/O error: %s\n" % str(err))
+	sys.exit(4)
+    fp.write(string.joinfields(format, "\n"))
+    fp.close()
+
+
+if __name__ == "__main__":
+    main()