fix an ambiguity in the grammar from the implementation of extended unpacking

(one which was strangely "resolved" by pgen)

This also kills the unused testlist1 rule and fixes parse tree validation of
extended unpacking.

Grammar/Grammar

@@ -37,8 +37,9 @@ stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
-expr_stmt: testlist (augassign (yield_expr|testlist) |
-                     ('=' (yield_expr|testlist))*)
+expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')
 # For normal assignments, additional restrictions enforced by the interpreter
@@ -86,9 +87,9 @@ lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
 or_test: and_test ('or' and_test)*
 and_test: not_test ('and' not_test)*
 not_test: 'not' not_test | comparison
-comparison: star_expr (comp_op star_expr)*
+comparison: expr (comp_op expr)*
 comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
-star_expr: ['*'] expr
+star_expr: '*' expr
 expr: xor_expr ('|' xor_expr)*
 xor_expr: and_expr ('^' and_expr)*
 and_expr: shift_expr ('&' shift_expr)*
@@ -101,12 +102,12 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
-testlist_comp: test ( comp_for | (',' test)* [','] )
+testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
-exprlist: star_expr (',' star_expr)* [',']
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
 dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
                   (test (comp_for | (',' test)* [','])) )
@@ -123,8 +124,6 @@ comp_iter: comp_for | comp_if
 comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
-testlist1: test (',' test)*
-
 # not used in grammar, but may appear in "node" passed from Parser to Compiler
 encoding_decl: NAME
 

Include/graminit.h

@@ -16,68 +16,68 @@
 #define simple_stmt 269
 #define small_stmt 270
 #define expr_stmt 271
-#define augassign 272
-#define del_stmt 273
-#define pass_stmt 274
-#define flow_stmt 275
-#define break_stmt 276
-#define continue_stmt 277
-#define return_stmt 278
-#define yield_stmt 279
-#define raise_stmt 280
-#define import_stmt 281
-#define import_name 282
-#define import_from 283
-#define import_as_name 284
-#define dotted_as_name 285
-#define import_as_names 286
-#define dotted_as_names 287
-#define dotted_name 288
-#define global_stmt 289
-#define nonlocal_stmt 290
-#define assert_stmt 291
-#define compound_stmt 292
-#define if_stmt 293
-#define while_stmt 294
-#define for_stmt 295
-#define try_stmt 296
-#define with_stmt 297
-#define with_item 298
-#define except_clause 299
-#define suite 300
-#define test 301
-#define test_nocond 302
-#define lambdef 303
-#define lambdef_nocond 304
-#define or_test 305
-#define and_test 306
-#define not_test 307
-#define comparison 308
-#define comp_op 309
-#define star_expr 310
-#define expr 311
-#define xor_expr 312
-#define and_expr 313
-#define shift_expr 314
-#define arith_expr 315
-#define term 316
-#define factor 317
-#define power 318
-#define atom 319
-#define testlist_comp 320
-#define trailer 321
-#define subscriptlist 322
-#define subscript 323
-#define sliceop 324
-#define exprlist 325
-#define testlist 326
-#define dictorsetmaker 327
-#define classdef 328
-#define arglist 329
-#define argument 330
-#define comp_iter 331
-#define comp_for 332
-#define comp_if 333
-#define testlist1 334
+#define testlist_star_expr 272
+#define augassign 273
+#define del_stmt 274
+#define pass_stmt 275
+#define flow_stmt 276
+#define break_stmt 277
+#define continue_stmt 278
+#define return_stmt 279
+#define yield_stmt 280
+#define raise_stmt 281
+#define import_stmt 282
+#define import_name 283
+#define import_from 284
+#define import_as_name 285
+#define dotted_as_name 286
+#define import_as_names 287
+#define dotted_as_names 288
+#define dotted_name 289
+#define global_stmt 290
+#define nonlocal_stmt 291
+#define assert_stmt 292
+#define compound_stmt 293
+#define if_stmt 294
+#define while_stmt 295
+#define for_stmt 296
+#define try_stmt 297
+#define with_stmt 298
+#define with_item 299
+#define except_clause 300
+#define suite 301
+#define test 302
+#define test_nocond 303
+#define lambdef 304
+#define lambdef_nocond 305
+#define or_test 306
+#define and_test 307
+#define not_test 308
+#define comparison 309
+#define comp_op 310
+#define star_expr 311
+#define expr 312
+#define xor_expr 313
+#define and_expr 314
+#define shift_expr 315
+#define arith_expr 316
+#define term 317
+#define factor 318
+#define power 319
+#define atom 320
+#define testlist_comp 321
+#define trailer 322
+#define subscriptlist 323
+#define subscript 324
+#define sliceop 325
+#define exprlist 326
+#define testlist 327
+#define dictorsetmaker 328
+#define classdef 329
+#define arglist 330
+#define argument 331
+#define comp_iter 332
+#define comp_for 333
+#define comp_if 334
 #define encoding_decl 335
 #define yield_expr 336

Lib/symbol.py

@@ -26,69 +26,69 @@ stmt = 268
 simple_stmt = 269
 small_stmt = 270
 expr_stmt = 271
-augassign = 272
-del_stmt = 273
-pass_stmt = 274
-flow_stmt = 275
-break_stmt = 276
-continue_stmt = 277
-return_stmt = 278
-yield_stmt = 279
-raise_stmt = 280
-import_stmt = 281
-import_name = 282
-import_from = 283
-import_as_name = 284
-dotted_as_name = 285
-import_as_names = 286
-dotted_as_names = 287
-dotted_name = 288
-global_stmt = 289
-nonlocal_stmt = 290
-assert_stmt = 291
-compound_stmt = 292
-if_stmt = 293
-while_stmt = 294
-for_stmt = 295
-try_stmt = 296
-with_stmt = 297
-with_item = 298
-except_clause = 299
-suite = 300
-test = 301
-test_nocond = 302
-lambdef = 303
-lambdef_nocond = 304
-or_test = 305
-and_test = 306
-not_test = 307
-comparison = 308
-comp_op = 309
-star_expr = 310
-expr = 311
-xor_expr = 312
-and_expr = 313
-shift_expr = 314
-arith_expr = 315
-term = 316
-factor = 317
-power = 318
-atom = 319
-testlist_comp = 320
-trailer = 321
-subscriptlist = 322
-subscript = 323
-sliceop = 324
-exprlist = 325
-testlist = 326
-dictorsetmaker = 327
-classdef = 328
-arglist = 329
-argument = 330
-comp_iter = 331
-comp_for = 332
-comp_if = 333
-testlist1 = 334
+testlist_star_expr = 272
+augassign = 273
+del_stmt = 274
+pass_stmt = 275
+flow_stmt = 276
+break_stmt = 277
+continue_stmt = 278
+return_stmt = 279
+yield_stmt = 280
+raise_stmt = 281
+import_stmt = 282
+import_name = 283
+import_from = 284
+import_as_name = 285
+dotted_as_name = 286
+import_as_names = 287
+dotted_as_names = 288
+dotted_name = 289
+global_stmt = 290
+nonlocal_stmt = 291
+assert_stmt = 292
+compound_stmt = 293
+if_stmt = 294
+while_stmt = 295
+for_stmt = 296
+try_stmt = 297
+with_stmt = 298
+with_item = 299
+except_clause = 300
+suite = 301
+test = 302
+test_nocond = 303
+lambdef = 304
+lambdef_nocond = 305
+or_test = 306
+and_test = 307
+not_test = 308
+comparison = 309
+comp_op = 310
+star_expr = 311
+expr = 312
+xor_expr = 313
+and_expr = 314
+shift_expr = 315
+arith_expr = 316
+term = 317
+factor = 318
+power = 319
+atom = 320
+testlist_comp = 321
+trailer = 322
+subscriptlist = 323
+subscript = 324
+sliceop = 325
+exprlist = 326
+testlist = 327
+dictorsetmaker = 328
+classdef = 329
+arglist = 330
+argument = 331
+comp_iter = 332
+comp_for = 333
+comp_if = 334
 encoding_decl = 335
 yield_expr = 336
 #--end constants--

Lib/test/test_parser.py

@@ -242,6 +242,12 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
             (0, '', 2, -1)],
                          terminals)
 
+    def test_extended_unpacking(self):
+        self.check_suite("*a = y")
+        self.check_suite("x, *b, = m")
+        self.check_suite("[*a, *b] = y")
+        self.check_suite("for [*x, b] in x: pass")
+
 
 #
 #  Second, we take *invalid* trees and make sure we get ParserError

Misc/NEWS

@@ -12,6 +12,8 @@ What's New in Python 3.2 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #5460: Fix an ambiguity in the grammar.
+
 - Issue #1766304: Improve performance of membership tests on range objects.
 
 - Issue #6713: Improve performance of integer -> string conversions.

Modules/parsermodule.c

@@ -943,7 +943,7 @@ VALIDATER(return_stmt);         VALIDATER(raise_stmt);
 VALIDATER(import_stmt);         VALIDATER(import_stmt);
 VALIDATER(import_name);         VALIDATER(yield_stmt);
 VALIDATER(global_stmt);         VALIDATER(assert_stmt);
-VALIDATER(compound_stmt);
+VALIDATER(compound_stmt);       VALIDATER(test_or_star_expr);
 VALIDATER(while);               VALIDATER(for);
 VALIDATER(try);                 VALIDATER(except_clause);
 VALIDATER(test);                VALIDATER(and_test);
@@ -958,10 +958,10 @@ VALIDATER(trailer);             VALIDATER(subscript);
 VALIDATER(subscriptlist);       VALIDATER(sliceop);
 VALIDATER(exprlist);            VALIDATER(dictorsetmaker);
 VALIDATER(arglist);             VALIDATER(argument);
-VALIDATER(testlist1);           VALIDATER(comp_for);
+VALIDATER(comp_for);
 VALIDATER(comp_iter);           VALIDATER(comp_if);
 VALIDATER(testlist_comp);       VALIDATER(yield_expr);
-VALIDATER(yield_or_testlist);   VALIDATER(or_test);
+VALIDATER(or_test);
 VALIDATER(test_nocond);         VALIDATER(lambdef_nocond);
 
 #undef VALIDATER
@@ -1183,12 +1183,11 @@ validate_testlist(node *tree)
                                     validate_test, "testlist"));
 }
 
-
 static int
-validate_testlist1(node *tree)
+validate_testlist_star_expr(node *tl)
 {
-    return (validate_repeating_list(tree, testlist1,
-                                    validate_test, "testlist1"));
+    return (validate_repeating_list(tl, testlist_star_expr, validate_test_or_star_expr,
+                                    "testlist"));
 }
 
 
@@ -1516,12 +1515,17 @@ validate_compound_stmt(node *tree)
 }
 
 static int
-validate_yield_or_testlist(node *tree)
+validate_yield_or_testlist(node *tree, int tse)
 {
-	if (TYPE(tree) == yield_expr)
-		return validate_yield_expr(tree);
-	else
-		return validate_testlist(tree);
+    if (TYPE(tree) == yield_expr) {
+        return validate_yield_expr(tree);
+    }
+    else {
+        if (tse)
+            return validate_testlist_star_expr(tree);
+        else
+            return validate_testlist(tree);
+    }
 }
 
 static int
@@ -1531,12 +1535,12 @@ validate_expr_stmt(node *tree)
     int nch = NCH(tree);
     int res = (validate_ntype(tree, expr_stmt)
                && is_odd(nch)
-               && validate_testlist(CHILD(tree, 0)));
+               && validate_testlist_star_expr(CHILD(tree, 0)));
 
     if (res && nch == 3
         && TYPE(CHILD(tree, 1)) == augassign) {
         res = validate_numnodes(CHILD(tree, 1), 1, "augassign")
-		&& validate_yield_or_testlist(CHILD(tree, 2));
+            && validate_yield_or_testlist(CHILD(tree, 2), 0);
 
         if (res) {
             char *s = STR(CHILD(CHILD(tree, 1), 0));
@@ -1560,7 +1564,7 @@ validate_expr_stmt(node *tree)
     else {
         for (j = 1; res && (j < nch); j += 2)
             res = validate_equal(CHILD(tree, j))
-                   && validate_yield_or_testlist(CHILD(tree, j + 1));
+                && validate_yield_or_testlist(CHILD(tree, j + 1), 1);
     }
     return (res);
 }
@@ -2077,11 +2081,11 @@ validate_comparison(node *tree)
     int nch = NCH(tree);
     int res = (validate_ntype(tree, comparison)
                && is_odd(nch)
-               && validate_star_expr(CHILD(tree, 0)));
+               && validate_expr(CHILD(tree, 0)));
 
     for (pos = 1; res && (pos < nch); pos += 2)
         res = (validate_comp_op(CHILD(tree, pos))
-               && validate_star_expr(CHILD(tree, pos + 1)));
+               && validate_expr(CHILD(tree, pos + 1)));
 
     return (res);
 }
@@ -2143,12 +2147,10 @@ validate_star_expr(node *tree)
 {
     int res = validate_ntype(tree, star_expr);
     if (!res) return res;
-    if (NCH(tree) == 2) {
-        return validate_ntype(CHILD(tree, 0), STAR) && \
-               validate_expr(CHILD(tree, 1));
-    } else {
-        return validate_expr(CHILD(tree, 0));
-    }
+    if (!validate_numnodes(tree, 2, "star_expr"))
+        return 0;
+    return validate_ntype(CHILD(tree, 0), STAR) &&      \
+        validate_expr(CHILD(tree, 1));
 }
 
 
@@ -2379,7 +2381,7 @@ validate_testlist_comp(node *tree)
     if (nch == 0)
         err_string("missing child nodes of testlist_comp");
     else {
-        ok = validate_test(CHILD(tree, 0));
+        ok = validate_test_or_star_expr(CHILD(tree, 0));
     }
 
     /*
@@ -2392,7 +2394,7 @@ validate_testlist_comp(node *tree)
         int i = 1;
         while (ok && nch - i >= 2) {
             ok = (validate_comma(CHILD(tree, i))
-                  && validate_test(CHILD(tree, i+1)));
+                  && validate_test_or_star_expr(CHILD(tree, i+1)));
             i += 2;
         }
         if (ok && i == nch-1)
@@ -2785,11 +2787,28 @@ validate_sliceop(node *tree)
 }
 
 
+static int
+validate_test_or_star_expr(node *n)
+{
+    if (TYPE(n) == test)
+        return validate_test(n);
+    return validate_star_expr(n);
+}
+
+static int
+validate_expr_or_star_expr(node *n)
+{
+    if (TYPE(n) == expr)
+        return validate_expr(n);
+    return validate_star_expr(n);
+}
+
+
 static int
 validate_exprlist(node *tree)
 {
     return (validate_repeating_list(tree, exprlist,
-                                    validate_star_expr, "exprlist"));
+                                    validate_expr_or_star_expr, "exprlist"));
 }
 
 
@@ -2970,9 +2989,6 @@ validate_node(node *tree)
           case yield_expr:
             res = validate_yield_expr(tree);
             break;
-          case testlist1:
-            res = validate_testlist1(tree);
-            break;
           case test:
             res = validate_test(tree);
             break;

Python/ast.c

@@ -603,20 +603,23 @@ ast_for_comp_op(struct compiling *c, const node *n)
 static asdl_seq *
 seq_for_testlist(struct compiling *c, const node *n)
 {
-    /* testlist: test (',' test)* [','] */
+    /* testlist: test (',' test)* [','] 
+       testlist_star_expr: test|star_expr (',' test|star_expr)* [',']
+    */
     asdl_seq *seq;
     expr_ty expression;
     int i;
-    assert(TYPE(n) == testlist || TYPE(n) == testlist_comp);
+    assert(TYPE(n) == testlist || TYPE(n) == testlist_star_expr || TYPE(n) == testlist_comp);
 
     seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
     if (!seq)
         return NULL;
 
     for (i = 0; i < NCH(n); i += 2) {
-        assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == test_nocond);
+	const node *ch = CHILD(n, i);
+        assert(TYPE(ch) == test || TYPE(ch) == test_nocond || TYPE(ch) == star_expr);
 
-        expression = ast_for_expr(c, CHILD(n, i));
+        expression = ast_for_expr(c, ch);
         if (!expression)
             return NULL;
 
@@ -1886,10 +1889,9 @@ ast_for_expr(struct compiling *c, const node *n)
             break;
 
         case star_expr:
-            if (TYPE(CHILD(n, 0)) == STAR) {
-                return ast_for_starred(c, n);
-            }
-            /* Fallthrough */
+            if (TYPE(CHILD(n, 0)) == STAR)
+	         return ast_for_starred(c, n);
+            /* Fall through to generic case. */
         /* The next five cases all handle BinOps.  The main body of code
            is the same in each case, but the switch turned inside out to
            reuse the code for each type of operator.
@@ -2067,7 +2069,6 @@ ast_for_testlist(struct compiling *c, const node* n)
 {
     /* testlist_comp: test (comp_for | (',' test)* [',']) */
     /* testlist: test (',' test)* [','] */
-    /* testlist1: test (',' test)* */
     assert(NCH(n) > 0);
     if (TYPE(n) == testlist_comp) {
         if (NCH(n) > 1)
@@ -2075,7 +2076,7 @@ ast_for_testlist(struct compiling *c, const node* n)
     }
     else {
         assert(TYPE(n) == testlist ||
-               TYPE(n) == testlist1);
+               TYPE(n) == testlist_star_expr);
     }
     if (NCH(n) == 1)
         return ast_for_expr(c, CHILD(n, 0));
@@ -2091,9 +2092,9 @@ static stmt_ty
 ast_for_expr_stmt(struct compiling *c, const node *n)
 {
     REQ(n, expr_stmt);
-    /* expr_stmt: testlist (augassign (yield_expr|testlist) 
+    /* expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) 
                 | ('=' (yield_expr|testlist))*)
-       testlist: test (',' test)* [',']
+       testlist_star_expr: (test|star_expr) (',' test|star_expr)* [',']
        augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^='
                 | '<<=' | '>>=' | '**=' | '//='
        test: ... here starts the operator precendence dance
@@ -2161,7 +2162,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
             asdl_seq_SET(targets, i / 2, e);
         }
         value = CHILD(n, NCH(n) - 1);
-        if (TYPE(value) == testlist)
+        if (TYPE(value) == testlist_star_expr)
             expression = ast_for_testlist(c, value);
         else
             expression = ast_for_expr(c, value);