Issue #13629: Renumber the tokens in token.h to match the _PyParser_TokenNames indexes.

Include/token.h

@@ -34,38 +34,37 @@ extern "C" {
 #define EQUAL		22
 #define DOT		23
 #define PERCENT		24
-/* #define BACKQUOTE	25 */
-#define LBRACE		26
-#define RBRACE		27
-#define EQEQUAL		28
-#define NOTEQUAL	29
-#define LESSEQUAL	30
-#define GREATEREQUAL	31
-#define TILDE		32
-#define CIRCUMFLEX	33
-#define LEFTSHIFT	34
-#define RIGHTSHIFT	35
-#define DOUBLESTAR	36
-#define PLUSEQUAL	37
-#define MINEQUAL	38
-#define STAREQUAL	39
-#define SLASHEQUAL	40
-#define PERCENTEQUAL	41
-#define AMPEREQUAL	42
-#define VBAREQUAL	43
-#define CIRCUMFLEXEQUAL	44
-#define LEFTSHIFTEQUAL	45
-#define RIGHTSHIFTEQUAL	46
-#define DOUBLESTAREQUAL	47
-#define DOUBLESLASH	48
-#define DOUBLESLASHEQUAL 49
-#define AT              50	
-#define RARROW          51
-#define ELLIPSIS        52
+#define LBRACE		25
+#define RBRACE		26
+#define EQEQUAL		27
+#define NOTEQUAL	28
+#define LESSEQUAL	29
+#define GREATEREQUAL	30
+#define TILDE		31
+#define CIRCUMFLEX	32
+#define LEFTSHIFT	33
+#define RIGHTSHIFT	34
+#define DOUBLESTAR	35
+#define PLUSEQUAL	36
+#define MINEQUAL	37
+#define STAREQUAL	38
+#define SLASHEQUAL	39
+#define PERCENTEQUAL	40
+#define AMPEREQUAL	41
+#define VBAREQUAL	42
+#define CIRCUMFLEXEQUAL	43
+#define LEFTSHIFTEQUAL	44
+#define RIGHTSHIFTEQUAL	45
+#define DOUBLESTAREQUAL	46
+#define DOUBLESLASH	47
+#define DOUBLESLASHEQUAL 48
+#define AT              49	
+#define RARROW          50
+#define ELLIPSIS        51
 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
-#define OP		53
-#define ERRORTOKEN	54
-#define N_TOKENS	55
+#define OP		52
+#define ERRORTOKEN	53
+#define N_TOKENS	54
 
 /* Special definitions for cooperation with parser */
 

Lib/token.py

@@ -35,36 +35,36 @@ GREATER = 21
 EQUAL = 22
 DOT = 23
 PERCENT = 24
-LBRACE = 26
-RBRACE = 27
-EQEQUAL = 28
-NOTEQUAL = 29
-LESSEQUAL = 30
-GREATEREQUAL = 31
-TILDE = 32
-CIRCUMFLEX = 33
-LEFTSHIFT = 34
-RIGHTSHIFT = 35
-DOUBLESTAR = 36
-PLUSEQUAL = 37
-MINEQUAL = 38
-STAREQUAL = 39
-SLASHEQUAL = 40
-PERCENTEQUAL = 41
-AMPEREQUAL = 42
-VBAREQUAL = 43
-CIRCUMFLEXEQUAL = 44
-LEFTSHIFTEQUAL = 45
-RIGHTSHIFTEQUAL = 46
-DOUBLESTAREQUAL = 47
-DOUBLESLASH = 48
-DOUBLESLASHEQUAL = 49
-AT = 50
-RARROW = 51
-ELLIPSIS = 52
-OP = 53
-ERRORTOKEN = 54
-N_TOKENS = 55
+LBRACE = 25
+RBRACE = 26
+EQEQUAL = 27
+NOTEQUAL = 28
+LESSEQUAL = 29
+GREATEREQUAL = 30
+TILDE = 31
+CIRCUMFLEX = 32
+LEFTSHIFT = 33
+RIGHTSHIFT = 34
+DOUBLESTAR = 35
+PLUSEQUAL = 36
+MINEQUAL = 37
+STAREQUAL = 38
+SLASHEQUAL = 39
+PERCENTEQUAL = 40
+AMPEREQUAL = 41
+VBAREQUAL = 42
+CIRCUMFLEXEQUAL = 43
+LEFTSHIFTEQUAL = 44
+RIGHTSHIFTEQUAL = 45
+DOUBLESTAREQUAL = 46
+DOUBLESLASH = 47
+DOUBLESLASHEQUAL = 48
+AT = 49
+RARROW = 50
+ELLIPSIS = 51
+OP = 52
+ERRORTOKEN = 53
+N_TOKENS = 54
 NT_OFFSET = 256
 #--end constants--
 

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.2.3?
 Core and Builtins
 -----------------
 
+- Issue #13629: Renumber the tokens in token.h so that they match the indexes
+  into _PyParser_TokenNames.
+
 - Fix the fix for issue #12149: it was incorrect, although it had the side
   effect of appearing to resolve the issue.  Thanks to Mark Shannon for
   noticing.

Python/graminit.c

@@ -1978,7 +1978,7 @@ static label labels[168] = {
     {258, 0},
     {327, 0},
     {259, 0},
-    {50, 0},
+    {49, 0},
     {289, 0},
     {7, 0},
     {330, 0},
@@ -1990,7 +1990,7 @@ static label labels[168] = {
     {1, "def"},
     {1, 0},
     {263, 0},
-    {51, 0},
+    {50, 0},
     {302, 0},
     {11, 0},
     {301, 0},
@@ -1999,7 +1999,7 @@ static label labels[168] = {
     {22, 0},
     {12, 0},
     {16, 0},
-    {36, 0},
+    {35, 0},
     {266, 0},
     {267, 0},
     {270, 0},
@@ -2016,6 +2016,7 @@ static label labels[168] = {
     {273, 0},
     {336, 0},
     {311, 0},
+    {36, 0},
     {37, 0},
     {38, 0},
     {39, 0},
@@ -2026,8 +2027,7 @@ static label labels[168] = {
     {44, 0},
     {45, 0},
     {46, 0},
-    {47, 0},
-    {49, 0},
+    {48, 0},
     {1, "del"},
     {326, 0},
     {1, "pass"},
@@ -2046,7 +2046,7 @@ static label labels[168] = {
     {1, "import"},
     {288, 0},
     {23, 0},
-    {52, 0},
+    {51, 0},
     {287, 0},
     {285, 0},
     {1, "as"},
@@ -2088,38 +2088,38 @@ static label labels[168] = {
     {310, 0},
     {20, 0},
     {21, 0},
-    {28, 0},
-    {31, 0},
+    {27, 0},
     {30, 0},
     {29, 0},
-    {29, 0},
+    {28, 0},
+    {28, 0},
     {1, "is"},
     {313, 0},
     {18, 0},
     {314, 0},
-    {33, 0},
+    {32, 0},
     {315, 0},
     {19, 0},
     {316, 0},
+    {33, 0},
     {34, 0},
-    {35, 0},
     {317, 0},
     {14, 0},
     {15, 0},
     {318, 0},
     {17, 0},
     {24, 0},
-    {48, 0},
-    {32, 0},
+    {47, 0},
+    {31, 0},
     {319, 0},
     {320, 0},
     {322, 0},
     {321, 0},
     {9, 0},
     {10, 0},
-    {26, 0},
+    {25, 0},
     {328, 0},
-    {27, 0},
+    {26, 0},
     {2, 0},
     {3, 0},
     {1, "None"},