Add tool to check documentation against declaration.

Doc/tools/cmpcsyms

+#! /usr/bin/env python
+from __future__ import with_statement
+import errno
+import os
+import re
+import sys
+import string
+
+if __name__ == "__main__":
+    _base = sys.argv[0]
+else:
+    _base = __file__
+
+_script_home = os.path.abspath(os.path.dirname(_base))
+
+srcdir = os.path.dirname(os.path.dirname(_script_home))
+
+EXCLUDES = ["bitset.h", "cStringIO.h", "graminit.h", "grammar.h",
+            "longintrepr.h", "metagrammar.h",
+            "node.h", "opcode.h", "osdefs.h", "pgenheaders.h",
+            "py_curses.h", "parsetok.h", "symtable.h", "token.h"]
+
+
+def list_headers():
+    """Return a list of headers."""
+    incdir = os.path.join(srcdir, "Include")
+    return [os.path.join(incdir, fn) for fn in os.listdir(incdir)
+            if fn.endswith(".h") and fn not in EXCLUDES]
+
+
+def matcher(pattern):
+    return re.compile(pattern).search
+
+MATCHERS = [
+    # XXX this should also deal with ctypedesc, cvardesc and cmemberdesc
+    matcher(r"\\begin\{cfuncdesc\}\{(?P<result>[^}]*)\}\{(?P<sym>[^}]*)\}{(?P<params>[^}]*)\}"),
+    matcher(r"\\cfuncline\{(?P<result>[^})]*)\}\{(?P<sym>[^}]*)\}{(?P<params>[^}]*)\}"),
+    ]
+
+def list_documented_items():
+    """Return a list of everything that's already documented."""
+    apidir = os.path.join(srcdir, "Doc", "api")
+    files = [fn for fn in os.listdir(apidir) if fn.endswith(".tex")]
+    L = []
+    for fn in files:
+        fullname = os.path.join(apidir, fn)
+	data = open(fullname).read()
+        for matcher in MATCHERS:
+            pos = 0
+            while 1:
+                m = matcher(data, pos)
+                if not m: break
+                pos = m.end()
+                sym = m.group("sym")
+                result = m.group("result")
+                params = m.group("params")
+		# replace all whitespace with a single one
+		params = " ".join(params.split()) 
+                L.append((sym, result, params, fn))
+    return L
+
+def normalize_type(t):
+    t = t.strip()
+    s = t.rfind("*")
+    if s != -1:
+        # strip everything after the pointer name
+        t = t[:s+1]
+    # Drop the variable name
+    s = t.split()
+    typenames = 1
+    if len(s)>1 and s[0]=='unsigned' and s[1]=='int':
+        typenames = 2
+    if len(s) > typenames and s[-1][0] in string.letters:
+        del s[-1]
+    if not s:
+       print "XXX", t
+       return ""
+    # Drop register
+    if s[0] == "register":
+        del s[0]
+    # discard all spaces
+    return ''.join(s)
+    
+def compare_type(t1, t2):
+    t1 = normalize_type(t1)
+    t2 = normalize_type(t2)
+    if t1 == r'\moreargs' and t2 == '...':
+        return False
+    if t1 != t2:
+        #print "different:", t1, t2
+        return False
+    return True
+
+
+def compare_types(ret, params, hret, hparams):
+    if not compare_type(ret, hret):
+        return False
+    params = params.split(",")
+    hparams = hparams.split(",")
+    if not params and hparams == ['void']:
+        return True
+    if not hparams and params == ['void']:
+        return True
+    if len(params) != len(hparams):
+        return False
+    for p1, p2 in zip(params, hparams):
+        if not compare_type(p1, p2):
+            return False
+    return True
+
+def main():
+    headers = list_headers()
+    documented = list_documented_items()
+
+    lines = []
+    for h in headers:
+        data = open(h).read()
+        data, n = re.subn(r"PyAPI_FUNC\(([^)]*)\)", r"\1", data)
+        name = os.path.basename(h)
+        with open(name, "w") as f:
+            f.write(data)
+        cmd = ("ctags -f - --file-scope=no --c-kinds=p --fields=S "
+               "-Istaticforward -Istatichere=static " + name)
+        with os.popen(cmd) as f:
+            lines.extend(f.readlines())
+        os.unlink(name)
+    L = {}
+    prevsym = None
+    for line in lines:
+        if not line:
+            break
+        sym, filename, signature = line.split(None, 2)
+        if sym == prevsym:
+            continue
+	expr = "\^(.*)%s" % sym
+	m = re.search(expr, signature)
+        if not m:
+    	    print "Could not split",signature, "using",expr
+	rettype = m.group(1).strip()
+	m = re.search("signature:\(([^)]*)\)", signature)
+	if not m:
+	    print "Could not get signature from", signature
+	params = m.group(1)
+	L[sym] = (rettype, params)
+
+    for sym, ret, params, fn in documented:
+        if sym not in L:
+           print "No declaration for '%s'" % sym
+           continue
+        hret, hparams = L[sym]
+        if not compare_types(ret, params, hret, hparams):
+           print "Declaration error for %s (%s):" % (sym, fn)
+           print ret+": "+params
+           print hret+": "+hparams
+
+if __name__ == "__main__":
+    main()