token.py 2.88 KB
Newer Older
Guido van Rossum's avatar
Guido van Rossum committed
1
#! /usr/bin/env python
2 3 4

"""Token constants (from "token.h")."""

Guido van Rossum's avatar
Guido van Rossum committed
5 6 7 8 9
#  This file is automatically generated; please don't muck it up!
#
#  To update the symbols in this file, 'cd' to the top directory of
#  the python source tree after building the interpreter and run:
#
10
#    python Lib/token.py
11

Guido van Rossum's avatar
Guido van Rossum committed
12
#--start constants--
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
ENDMARKER = 0
NAME = 1
NUMBER = 2
STRING = 3
NEWLINE = 4
INDENT = 5
DEDENT = 6
LPAR = 7
RPAR = 8
LSQB = 9
RSQB = 10
COLON = 11
COMMA = 12
SEMI = 13
PLUS = 14
MINUS = 15
STAR = 16
SLASH = 17
VBAR = 18
AMPER = 19
LESS = 20
GREATER = 21
EQUAL = 22
DOT = 23
PERCENT = 24
BACKQUOTE = 25
LBRACE = 26
RBRACE = 27
EQEQUAL = 28
NOTEQUAL = 29
LESSEQUAL = 30
GREATEREQUAL = 31
TILDE = 32
CIRCUMFLEX = 33
LEFTSHIFT = 34
RIGHTSHIFT = 35
49
DOUBLESTAR = 36
50 51 52 53 54 55 56 57 58 59 60
PLUSEQUAL = 37
MINEQUAL = 38
STAREQUAL = 39
SLASHEQUAL = 40
PERCENTEQUAL = 41
AMPEREQUAL = 42
VBAREQUAL = 43
CIRCUMFLEXEQUAL = 44
LEFTSHIFTEQUAL = 45
RIGHTSHIFTEQUAL = 46
DOUBLESTAREQUAL = 47
61 62
DOUBLESLASH = 48
DOUBLESLASHEQUAL = 49
63 64 65 66
AT = 50
OP = 51
ERRORTOKEN = 52
N_TOKENS = 53
Guido van Rossum's avatar
Guido van Rossum committed
67 68
NT_OFFSET = 256
#--end constants--
69 70

tok_name = {}
71 72
for _name, _value in globals().items():
    if type(_value) is type(0):
73
        tok_name[_value] = _name
74
del _name, _value
75

76

77 78 79 80 81 82 83 84
def ISTERMINAL(x):
    return x < NT_OFFSET

def ISNONTERMINAL(x):
    return x >= NT_OFFSET

def ISEOF(x):
    return x == ENDMARKER
Guido van Rossum's avatar
Guido van Rossum committed
85 86 87


def main():
88
    import re
Guido van Rossum's avatar
Guido van Rossum committed
89 90 91 92 93
    import sys
    args = sys.argv[1:]
    inFileName = args and args[0] or "Include/token.h"
    outFileName = "Lib/token.py"
    if len(args) > 1:
94
        outFileName = args[1]
Guido van Rossum's avatar
Guido van Rossum committed
95
    try:
96
        fp = open(inFileName)
Guido van Rossum's avatar
Guido van Rossum committed
97
    except IOError, err:
98 99
        sys.stdout.write("I/O error: %s\n" % str(err))
        sys.exit(1)
100
    lines = fp.read().split("\n")
Guido van Rossum's avatar
Guido van Rossum committed
101
    fp.close()
102
    prog = re.compile(
103
        "#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
104
        re.IGNORECASE)
Guido van Rossum's avatar
Guido van Rossum committed
105 106
    tokens = {}
    for line in lines:
107 108 109
        match = prog.match(line)
        if match:
            name, val = match.group(1, 2)
Eric S. Raymond's avatar
Eric S. Raymond committed
110
            val = int(val)
111
            tokens[val] = name          # reverse so we can sort them...
Guido van Rossum's avatar
Guido van Rossum committed
112 113 114 115
    keys = tokens.keys()
    keys.sort()
    # load the output skeleton from the target:
    try:
116
        fp = open(outFileName)
Guido van Rossum's avatar
Guido van Rossum committed
117
    except IOError, err:
118 119
        sys.stderr.write("I/O error: %s\n" % str(err))
        sys.exit(2)
120
    format = fp.read().split("\n")
Guido van Rossum's avatar
Guido van Rossum committed
121 122
    fp.close()
    try:
123 124
        start = format.index("#--start constants--") + 1
        end = format.index("#--end constants--")
Guido van Rossum's avatar
Guido van Rossum committed
125
    except ValueError:
126 127
        sys.stderr.write("target does not contain format markers")
        sys.exit(3)
Guido van Rossum's avatar
Guido van Rossum committed
128 129
    lines = []
    for val in keys:
130
        lines.append("%s = %d" % (tokens[val], val))
Guido van Rossum's avatar
Guido van Rossum committed
131 132
    format[start:end] = lines
    try:
133
        fp = open(outFileName, 'w')
Guido van Rossum's avatar
Guido van Rossum committed
134
    except IOError, err:
135 136
        sys.stderr.write("I/O error: %s\n" % str(err))
        sys.exit(4)
137
    fp.write("\n".join(format))
Guido van Rossum's avatar
Guido van Rossum committed
138 139 140 141 142
    fp.close()


if __name__ == "__main__":
    main()