Rewrite PHP tag creation script using up to date upstream tag definitions

Fixes #584.

scripts/create_php_tags.php

-#!/usr/bin/php
-<?php
-// Author:	Matti Mårds
-// License:	GPL V2 or later
-//
-// Script to generate a new php.tags file from a downloaded PHP function summary list from
-//
-// http://svn.php.net/repository/phpdoc/doc-base/trunk/funcsummary.txt
-//
-// - The script can be run from any directory
-// - The script downloads the file funcsummary.txt using PHP's stdlib
-
-# (from tagmanager/tm_tag.c:32)
-define("TA_NAME", 200);
-define("TA_TYPE", 204);
-define("TA_ARGLIST", 205);
-define("TA_VARTYPE", 207);
-
-# TMTagType (tagmanager/tm_tag.h:47)
-define("TYPE_FUNCTION", 128);
-
-// Create an array of the lines in the file
-$url = 'http://svn.php.net/repository/phpdoc/doc-base/trunk/funcsummary.txt';
-$file = file($url, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
-
-// Create template for a tag (tagmanager format)
-$tagTpl = "%s%c%d%c%s%c%s";
-
-// String to store the output
-$tagsOutput = array();
-
-// String data base path for tags.php
-$filePhpTags = implode( DIRECTORY_SEPARATOR,
-                        array(
-                        dirname(dirname(__FILE__)),
-                        'data',
-                        'php.tags'
-                      ));
-// Iterate through each line of the file
-for($line = 0, $lineCount = count($file); $line < $lineCount; ++$line) {
-
-    // If the line isn't a function definition, skip it
-    if(!preg_match('/^(?P<retType>\w+) (?P<funcName>[\w:]+)(?P<params>\(.*?\))/', $file[$line], $funcDefMatch)) {
-        continue;
-    }
-    // Skip methods as they aren't used for now
-    if (strpos($funcDefMatch['funcName'], '::') !== false) {
-        continue;
-    }
-
-    // Get the function description
-    //$funcDesc = trim($file[$line + 1]);
-    // Geany doesn't use the function description (yet?), so use an empty string to save space
-    $funcDesc = '';
-
-    // Remove the void parameter, it will only confuse some people
-    if($funcDefMatch['params'] === '(void)') {
-        $funcDefMatch['params'] = '()';
-    }
-
-    // $funcDefMatch['funcName'] = str_replace('::', '->', $funcDefMatch['funcName']);
-    $tagsOutput[] = sprintf($tagTpl, $funcDefMatch['funcName'], TA_TYPE, TYPE_FUNCTION,
-        TA_ARGLIST, $funcDefMatch['params'], TA_VARTYPE, $funcDefMatch['retType']);
-}
-
-$tagsOutput[] = sprintf(
-    '# format=tagmanager - Automatically generated file - do not edit (created on %s)',
-    date('r'));
-// Sort the output
-sort($tagsOutput);
-
-file_put_contents($filePhpTags, join("\n", $tagsOutput));
-echo "Created:\n${filePhpTags}\n";
-echo str_repeat('-',75)."\n";

scripts/create_php_tags.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Author:  Enrico Tröger
+# License: GPL v2 or later
+#
+# This script downloads the PHP tag definitions in JSON format from
+# http://doc.php.net/downloads/json/php_manual_en.json.
+# From those defintions all function tags are extracted and written
+# to ../data/php.tags (relative to the script's location, not $CWD).
+
+from json import loads
+from os.path import dirname, join
+from urllib2 import urlopen
+import re
+
+
+UPSTREAM_TAG_DEFINITION = 'http://doc.php.net/downloads/json/php_manual_en.json'
+PROTOTYPE_RE = r'^(?P<return_type>.*) {tag_name}(?P<arg_list>\(.*\))$'
+
+# (from tagmanager/src/tm_tag.c:90)
+TA_NAME = 200
+TA_TYPE = 204
+TA_ARGLIST = 205
+TA_SCOPE = 206
+TA_VARTYPE = 207
+# TMTagType (tagmanager/src/tm_tag.h:49)
+TYPE_CLASS = 1
+TYPE_FUNCTION = 16
+TYPE_MEMBER = 64
+TYPE_METHOD = 128
+TYPE_VARIABLE = 16384
+
+
+#----------------------------------------------------------------------
+def normalize_name(name):
+    """ Replace namespace separator with class separators, as Geany only
+        understands the latter """
+    return name.replace('\\', '::')
+
+
+#----------------------------------------------------------------------
+def split_scope(name):
+    """ Splits the scope from the member, and returns (scope, member).
+        Returned scope is None if the name is not a member """
+    name = normalize_name(name)
+    sep_pos = name.rfind('::')
+    if sep_pos < 0:
+        return None, name
+    else:
+        return name[:sep_pos], name[sep_pos+2:]
+
+
+#----------------------------------------------------------------------
+def parse_and_create_php_tags_file():
+    # download upstream definition
+    response = urlopen(UPSTREAM_TAG_DEFINITION)
+    try:
+        html = response.read()
+    finally:
+        response.close()
+
+    # parse JSON
+    definitions = loads(html)
+
+    # generate tags
+    tag_list = list()
+    for tag_name, tag_definition in definitions.items():
+        prototype_re = PROTOTYPE_RE.format(tag_name=re.escape(tag_name))
+        match = re.match(prototype_re, tag_definition['prototype'])
+        if match:
+            return_type = normalize_name(match.group('return_type'))
+            arg_list = match.group('arg_list')
+
+            scope, tag_name = split_scope(tag_name)
+            if tag_name[0] == '$':
+                tag_type = TYPE_MEMBER if scope is not None else TYPE_VARIABLE
+            else:
+                tag_type = TYPE_METHOD if scope is not None else TYPE_FUNCTION
+            tag_list.append((tag_name, tag_type, return_type, arg_list, scope))
+            # Also create a class tag when encountering a __construct()
+            if tag_name == '__construct' and scope is not None:
+                scope, tag_name = split_scope(scope)
+                tag_list.append((tag_name, TYPE_CLASS, None, arg_list, scope))
+
+    # write tags
+    script_dir = dirname(__file__)
+    tags_file_path = join(script_dir, '..', 'data', 'php.tags')
+    with open(tags_file_path, 'w') as tags_file:
+        tags_file.write('# format=tagmanager\n')
+        for tag_name, tag_type, return_type, arg_list, scope in sorted(tag_list):
+            tag_line = '{}'.format(tag_name)
+            for attr, type in [(tag_type, TA_TYPE),
+                               (arg_list, TA_ARGLIST),
+                               (return_type, TA_VARTYPE),
+                               (scope, TA_SCOPE)]:
+                if attr is not None:
+                    tag_line += '{type:c}{attr}'.format(type=type, attr=attr)
+
+            tags_file.write(tag_line + '\n')
+    print(u'Created: {} with {} tags'.format(tags_file_path, len(tag_list)))
+
+
+if __name__ == '__main__':
+    parse_and_create_php_tags_file()