Fix #define ordering.

Parser/tokenizer.c

@@ -1533,7 +1533,6 @@ PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
 	return result;
 }
 
-#ifdef Py_USING_UNICODE
 /* This function is only called from parsetok. However, it cannot live
    there, as it must be empty for PGEN, and we can check for PGEN only
    in this file. */
@@ -1545,6 +1544,7 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)
 	return NULL;
 }
 #else
+#ifdef Py_USING_UNICODE
 static PyObject *
 dec_utf8(const char *enc, const char *text, size_t len) {
 	PyObject *ret = NULL;	
@@ -1592,10 +1592,9 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
 	return text;
 
 }
+#endif /* defined(Py_USING_UNICODE) */
 #endif
 
-#endif /* defined(Py_USING_UNICODE) */
-			   
 
 #ifdef Py_DEBUG
 

Parser/tokenizer.h

@@ -58,7 +58,7 @@ extern struct tok_state *PyTokenizer_FromString(const char *);
 extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
 extern void PyTokenizer_Free(struct tok_state *);
 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
-#ifdef Py_USING_UNICODE
+#if defined(PGEN) || defined(Py_USING_UNICODE)
 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, 
 					  int len, int *offset);
 #endif