bpo-36623: Clean parser headers and include files (GH-12253)

After the removal of pgen, multiple header and function prototypes that lack implementation or are unused are still lying around.

Doc/whatsnew/3.8.rst

@@ -814,6 +814,10 @@ Changes in the Python API
   by the installer).
   (See :issue:`36085`.)
 
+* The header files and functions related to pgen have been removed after its
+  replacement by a pure Python implementation. (Contributed by Pablo Galindo
+  in :issue:`36623`.)
+
 
 Changes in the C API
 --------------------

Include/bitset.h

@@ -8,23 +8,14 @@ extern "C" {
 /* Bitset interface */
 
 #define BYTE            char
-
 typedef BYTE *bitset;
 
-bitset newbitset(int nbits);
-void delbitset(bitset bs);
 #define testbit(ss, ibit) (((ss)[BIT2BYTE(ibit)] & BIT2MASK(ibit)) != 0)
-int addbit(bitset bs, int ibit); /* Returns 0 if already set */
-int samebitset(bitset bs1, bitset bs2, int nbits);
-void mergebitset(bitset bs1, bitset bs2, int nbits);
 
 #define BITSPERBYTE     (8*sizeof(BYTE))
-#define NBYTES(nbits)   (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE)
-
 #define BIT2BYTE(ibit)  ((ibit) / BITSPERBYTE)
 #define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE)
 #define BIT2MASK(ibit)  (1 << BIT2SHIFT(ibit))
-#define BYTE2BIT(ibyte) ((ibyte) * BITSPERBYTE)
 
 #ifdef __cplusplus
 }

Include/grammar.h

@@ -66,27 +66,11 @@ typedef struct {
 } grammar;
 
 /* FUNCTIONS */
-
-grammar *newgrammar(int start);
-void freegrammar(grammar *g);
-dfa *adddfa(grammar *g, int type, const char *name);
-int addstate(dfa *d);
-void addarc(dfa *d, int from, int to, int lbl);
 dfa *PyGrammar_FindDFA(grammar *g, int type);
-
-int addlabel(labellist *ll, int type, const char *str);
-int findlabel(labellist *ll, int type, const char *str);
 const char *PyGrammar_LabelRepr(label *lb);
-void translatelabels(grammar *g);
-
-void addfirstsets(grammar *g);
-
 void PyGrammar_AddAccelerators(grammar *g);
 void PyGrammar_RemoveAccelerators(grammar *);
 
-void printgrammar(grammar *g, FILE *fp);
-void printnonterminals(grammar *g, FILE *fp);
-
 #ifdef __cplusplus
 }
 #endif

Include/pgenheaders.h

-#ifndef Py_PGENHEADERS_H
-#define Py_PGENHEADERS_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-/* Include files and extern declarations used by most of the parser. */
-
-#include "Python.h"
-
-PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...)
-                        Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
-PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...)
-                        Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
-
-#define addarc _Py_addarc
-#define addbit _Py_addbit
-#define adddfa _Py_adddfa
-#define addfirstsets _Py_addfirstsets
-#define addlabel _Py_addlabel
-#define addstate _Py_addstate
-#define delbitset _Py_delbitset
-#define dumptree _Py_dumptree
-#define findlabel _Py_findlabel
-#define freegrammar _Py_freegrammar
-#define mergebitset _Py_mergebitset
-#define meta_grammar _Py_meta_grammar
-#define newbitset _Py_newbitset
-#define newgrammar _Py_newgrammar
-#define pgen _Py_pgen
-#define printgrammar _Py_printgrammar
-#define printnonterminals _Py_printnonterminals
-#define printtree _Py_printtree
-#define samebitset _Py_samebitset
-#define showtree _Py_showtree
-#define tok_dump _Py_tok_dump
-#define translatelabels _Py_translatelabels
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* !Py_PGENHEADERS_H */

Makefile.pre.in

@@ -1008,7 +1008,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/osdefs.h \
 		$(srcdir)/Include/osmodule.h \
 		$(srcdir)/Include/patchlevel.h \
-		$(srcdir)/Include/pgenheaders.h \
 		$(srcdir)/Include/pyarena.h \
 		$(srcdir)/Include/pycapsule.h \
 		$(srcdir)/Include/pyctype.h \

Misc/NEWS.d/next/Core and Builtins/2019-04-13-02-08-44.bpo-36623.HR_xhB.rst

+Remove parser headers and related function declarations that lack
+implementations after the removal of pgen.

PCbuild/pythoncore.vcxproj

@@ -175,7 +175,6 @@
     <ClInclude Include="..\Include\osmodule.h" />
     <ClInclude Include="..\Include\parsetok.h" />
     <ClInclude Include="..\Include\patchlevel.h" />
-    <ClInclude Include="..\Include\pgenheaders.h" />
     <ClInclude Include="..\Include\pyhash.h" />
     <ClInclude Include="..\Include\py_curses.h" />
     <ClInclude Include="..\Include\pyarena.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -267,9 +267,6 @@
     <ClInclude Include="..\Include\patchlevel.h">
       <Filter>Include</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\pgenheaders.h">
-      <Filter>Include</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\py_curses.h">
       <Filter>Include</Filter>
     </ClInclude>

Parser/acceler.c

@@ -10,7 +10,7 @@
    are not part of the static data structure written on graminit.[ch]
    by the parser generator. */
 
-#include "pgenheaders.h"
+#include "Python.h"
 #include "grammar.h"
 #include "node.h"
 #include "token.h"

Parser/grammar1.c

@@ -2,7 +2,6 @@
 /* Grammar subroutines needed by parser */
 
 #include "Python.h"
-#include "pgenheaders.h"
 #include "grammar.h"
 #include "token.h"
 

Parser/listnode.c

 
 /* List a node on a file */
 
-#include "pgenheaders.h"
+#include "Python.h"
 #include "token.h"
 #include "node.h"
 

Parser/parser.c

@@ -6,7 +6,6 @@
 /* XXX To do: error recovery */
 
 #include "Python.h"
-#include "pgenheaders.h"
 #include "token.h"
 #include "grammar.h"
 #include "node.h"

Parser/parser.h

@@ -38,6 +38,11 @@ int PyParser_AddToken(parser_state *ps, int type, char *str,
                       int *expected_ret);
 void PyGrammar_AddAccelerators(grammar *g);
 
+
+#define showtree _Py_showtree
+#define printtree _Py_printtree
+#define dumptree _Py_dumptree
+
 #ifdef __cplusplus
 }
 #endif

Parser/parsetok.c

 
 /* Parser-tokenizer link implementation */
 
-#include "pgenheaders.h"
+#include "Python.h"
 #include "tokenizer.h"
 #include "node.h"
 #include "grammar.h"

Parser/pgen/grammar.py

@@ -61,7 +61,6 @@ class Grammar:
     def produce_graminit_c(self, writer):
         writer("/* Generated by Parser/pgen */\n\n")
 
-        writer('#include "pgenheaders.h"\n')
         writer('#include "grammar.h"\n')
         writer("grammar _PyParser_Grammar;\n")
 

Parser/tokenizer.c

@@ -2,7 +2,6 @@
 /* Tokenizer implementation */
 
 #include "Python.h"
-#include "pgenheaders.h"
 
 #include <ctype.h>
 #include <assert.h>

Parser/tokenizer.h

@@ -80,6 +80,8 @@ extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*,
 extern void PyTokenizer_Free(struct tok_state *);
 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
 
+#define tok_dump _Py_tok_dump
+
 #ifdef __cplusplus
 }
 #endif

Python/graminit.c

 /* Generated by Parser/pgen */
 
-#include "pgenheaders.h"
 #include "grammar.h"
 grammar _PyParser_Grammar;
 static arc arcs_0_0[3] = {

Python/strdup.c

 /* strdup() replacement (from stdwin, if you must know) */
 
-#include "pgenheaders.h"
-
 char *
 strdup(const char *str)
 {