[3.6] bpo-31852: Fix segfault caused by using the async soft keyword (GH-4122)

Lib/test/test_tokenize.py

@@ -630,6 +630,11 @@ def"', """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
+
+        self.check_tokenize("async\\", """\
+    ERRORTOKEN '\\\\'          (1, 5) (1, 6)
+    NAME       'async'       (1, 0) (1, 5)
     """)
 
         self.check_tokenize("a = (async = 1)", """\

Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst

+Fix a segmentation fault caused by a combination of the async soft keyword
+and continuation lines.

Parser/tokenizer.c

@@ -1563,6 +1563,9 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                 /* The current token is 'async'.
                    Look ahead one token.*/
 
+                int async_def_prev = tok->async_def;
+                tok->async_def = 2;
+
                 struct tok_state ahead_tok;
                 char *ahead_tok_start = NULL, *ahead_tok_end = NULL;
                 int ahead_tok_kind;
@@ -1581,6 +1584,9 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                     tok->async_def = 1;
                     return ASYNC;
                 }
+                else{
+                    tok->async_def = async_def_prev;
+                }
             }
         }
 
@@ -1844,6 +1850,10 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     /* Line continuation */
     if (c == '\\') {
         c = tok_nextc(tok);
+        if (tok->async_def == 2) {
+            tok->done = E_SYNTAX;
+            return ERRORTOKEN;
+        }
         if (c != '\n') {
             tok->done = E_LINECONT;
             tok->cur = tok->inp;