Fixed #16553 -- Refactored the `GeoIP` module, moving it…

Fixed #16553 -- Refactored the `GeoIP` module, moving it `django.contrib.gis.geoip`; fixed memory leaks, and encoding issues.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@16783 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/gis/geoip/__init__.py

+"""
+ This module houses the GeoIP object, a ctypes wrapper for the MaxMind GeoIP(R)
+ C API (http://www.maxmind.com/app/c).  This is an alternative to the GPL
+ licensed Python GeoIP interface provided by MaxMind.
+
+ GeoIP(R) is a registered trademark of MaxMind, LLC of Boston, Massachusetts.
+
+ For IP-based geolocation, this module requires the GeoLite Country and City
+ datasets, in binary format (CSV will not work!).  The datasets may be
+ downloaded from MaxMind at http://www.maxmind.com/download/geoip/database/.
+ Grab GeoIP.dat.gz and GeoLiteCity.dat.gz, and unzip them in the directory
+ corresponding to settings.GEOIP_PATH.
+"""
+try:
+    from django.contrib.gis.geoip.base import GeoIP, GeoIPException
+    HAS_GEOIP = True
+except:
+    HAS_GEOIP = False

django/contrib/gis/geoip/libgeoip.py

+import os
+from ctypes import CDLL
+from ctypes.util import find_library
+from django.conf import settings
+
+# Creating the settings dictionary with any settings, if needed.
+GEOIP_SETTINGS = dict((key, getattr(settings, key))
+                      for key in ('GEOIP_PATH', 'GEOIP_LIBRARY_PATH', 'GEOIP_COUNTRY', 'GEOIP_CITY')
+                      if hasattr(settings, key))
+lib_path = GEOIP_SETTINGS.get('GEOIP_LIBRARY_PATH', None)
+
+# The shared library for the GeoIP C API.  May be downloaded
+#  from http://www.maxmind.com/download/geoip/api/c/
+if lib_path:
+    lib_name = None
+else:
+    # TODO: Is this really the library name for Windows?
+    lib_name = 'GeoIP'
+
+# Getting the path to the GeoIP library.
+if lib_name: lib_path = find_library(lib_name)
+if lib_path is None: raise GeoIPException('Could not find the GeoIP library (tried "%s"). '
+                                          'Try setting GEOIP_LIBRARY_PATH in your settings.' % lib_name)
+lgeoip = CDLL(lib_path)
+
+# Getting the C `free` for the platform.
+if os.name == 'nt':
+    libc = CDLL('msvcrt')
+else:
+    libc = CDLL(None)
+free = libc.free

django/contrib/gis/geoip/prototypes.py

+from ctypes import c_char_p, c_float, c_int, string_at, Structure, POINTER
+from django.contrib.gis.geoip.libgeoip import lgeoip, free
+
+#### GeoIP C Structure definitions ####
+
+class GeoIPRecord(Structure):
+    _fields_ = [('country_code', c_char_p),
+                ('country_code3', c_char_p),
+                ('country_name', c_char_p),
+                ('region', c_char_p),
+                ('city', c_char_p),
+                ('postal_code', c_char_p),
+                ('latitude', c_float),
+                ('longitude', c_float),
+                # TODO: In 1.4.6 this changed from `int dma_code;` to
+                # `union {int metro_code; int dma_code;};`.  Change
+                # to a `ctypes.Union` in to accomodate in future when
+                # pre-1.4.6 versions are no longer distributed.
+                ('dma_code', c_int),
+                ('area_code', c_int),
+                ('charset', c_int),
+                ('continent_code', c_char_p),
+                ]
+geoip_char_fields = [name for name, ctype in GeoIPRecord._fields_ if ctype is c_char_p]
+geoip_encodings = { 0: 'iso-8859-1',
+                    1: 'utf8',
+                    }
+
+class GeoIPTag(Structure): pass
+
+RECTYPE = POINTER(GeoIPRecord)
+DBTYPE = POINTER(GeoIPTag)
+
+#### ctypes function prototypes ####
+
+# GeoIP_lib_version appeared in version 1.4.7.
+if hasattr(lgeoip, 'GeoIP_lib_version'):
+    GeoIP_lib_version = lgeoip.GeoIP_lib_version
+    GeoIP_lib_version.argtypes = None
+    GeoIP_lib_version.restype = c_char_p
+else:
+    GeoIP_lib_version = None
+
+# For freeing memory allocated within a record
+GeoIPRecord_delete = lgeoip.GeoIPRecord_delete
+GeoIPRecord_delete.argtypes = [RECTYPE]
+GeoIPRecord_delete.restype = None
+
+# For retrieving records by name or address.
+def check_record(result, func, cargs):
+    if bool(result):
+        # Checking the pointer to the C structure, if valid pull out elements
+        # into a dicionary.
+        rec = result.contents
+        record = dict((fld, getattr(rec, fld)) for fld, ctype in rec._fields_)
+
+        # Now converting the strings to unicode using the proper encoding.
+        encoding = geoip_encodings[record['charset']]
+        for char_field in geoip_char_fields:
+            if record[char_field]:
+                record[char_field] = record[char_field].decode(encoding)
+
+        # Free the memory allocated for the struct & return.
+        GeoIPRecord_delete(result)
+        return record
+    else:
+        return None
+
+def record_output(func):
+    func.argtypes = [DBTYPE, c_char_p]
+    func.restype = RECTYPE
+    func.errcheck = check_record
+    return func
+GeoIP_record_by_addr = record_output(lgeoip.GeoIP_record_by_addr)
+GeoIP_record_by_name = record_output(lgeoip.GeoIP_record_by_name)
+
+
+# For opening & closing GeoIP database files.
+GeoIP_open = lgeoip.GeoIP_open
+GeoIP_open.restype = DBTYPE
+GeoIP_delete = lgeoip.GeoIP_delete
+GeoIP_delete.argtypes = [DBTYPE]
+GeoIP_delete.restype = None
+
+# This is so the string pointer can be freed within Python.
+class geoip_char_p(c_char_p):
+    pass
+
+def check_string(result, func, cargs):
+    if result:
+        s = string_at(result)
+        free(result)
+    else:
+        s = ''
+    return s
+
+GeoIP_database_info = lgeoip.GeoIP_database_info
+GeoIP_database_info.restype = geoip_char_p
+GeoIP_database_info.errcheck = check_string
+
+# String output routines.
+def string_output(func):
+    func.restype = c_char_p
+    return func
+
+GeoIP_country_code_by_addr = string_output(lgeoip.GeoIP_country_code_by_addr)
+GeoIP_country_code_by_name = string_output(lgeoip.GeoIP_country_code_by_name)
+GeoIP_country_name_by_addr = string_output(lgeoip.GeoIP_country_name_by_addr)
+GeoIP_country_name_by_name = string_output(lgeoip.GeoIP_country_name_by_name)

django/contrib/gis/tests/test_geoip.py → django/contrib/gis/geoip/tests.py

 import os
-import unittest
-from django.db import settings
+from django.conf import settings
 from django.contrib.gis.geos import GEOSGeometry
-from django.contrib.gis.utils import GeoIP, GeoIPException
+from django.contrib.gis.geoip import GeoIP, GeoIPException
+from django.utils import unittest
 
 # Note: Requires use of both the GeoIP country and city datasets.
 # The GEOIP_DATA path should be the only setting set (the directory
@@ -69,8 +69,8 @@ class GeoIPTest(unittest.TestCase):
         "Testing GeoIP city querying methods."
         g = GeoIP(country='<foo>')
 
-        addr = '130.80.29.3'
-        fqdn = 'chron.com'
+        addr = '128.249.1.1'
+        fqdn = 'tmc.edu'
         for query in (fqdn, addr):
             # Country queries should still work.
             for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
@@ -88,17 +88,24 @@ class GeoIPTest(unittest.TestCase):
             self.assertEqual(713, d['area_code'])
             geom = g.geos(query)
             self.failIf(not isinstance(geom, GEOSGeometry))
-            lon, lat = (-95.3670, 29.7523)
+            lon, lat = (-95.4010, 29.7079)
             lat_lon = g.lat_lon(query)
             lat_lon = (lat_lon[1], lat_lon[0])
             for tup in (geom.tuple, g.coords(query), g.lon_lat(query), lat_lon):
                 self.assertAlmostEqual(lon, tup[0], 4)
                 self.assertAlmostEqual(lat, tup[1], 4)
 
+    def test05_unicode(self):
+        "Testing that GeoIP strings are properly encoded, see #16553."
+        g = GeoIP()
+        d = g.city('62.224.93.23')
+        self.assertEqual(u'Sch\xf6mberg', d['city'])
+
+
 def suite():
     s = unittest.TestSuite()
     s.addTest(unittest.makeSuite(GeoIPTest))
     return s
 
-def run(verbosity=2):
+def run(verbosity=1):
     unittest.TextTestRunner(verbosity=verbosity).run(suite())

django/contrib/gis/tests/__init__.py

@@ -78,10 +78,10 @@ def geodjango_suite(apps=True):
         sys.stderr.write('GDAL not available - no tests requiring GDAL will be run.\n')
 
     # Add GeoIP tests to the suite, if the library and data is available.
-    from django.contrib.gis.utils import HAS_GEOIP
+    from django.contrib.gis.geoip import HAS_GEOIP
     if HAS_GEOIP and hasattr(settings, 'GEOIP_PATH'):
-        from django.contrib.gis.tests import test_geoip
-        suite.addTest(test_geoip.suite())
+        from django.contrib.gis.geoip import tests as geoip_tests
+        suite.addTest(geoip_tests.suite())
 
     # Finally, adding the suites for each of the GeoDjango test apps.
     if apps:

django/contrib/gis/utils/__init__.py

@@ -8,18 +8,18 @@ if HAS_GDAL:
     from django.contrib.gis.utils.ogrinspect import mapping, ogrinspect
     from django.contrib.gis.utils.srs import add_postgis_srs, add_srs_entry
     try:
-        # LayerMapping requires DJANGO_SETTINGS_MODULE to be set, 
+        # LayerMapping requires DJANGO_SETTINGS_MODULE to be set,
         # so this needs to be in try/except.
         from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError
     except:
         pass
-    
-# Attempting to import the GeoIP class.
-try:
-    from django.contrib.gis.utils.geoip import GeoIP, GeoIPException
-    HAS_GEOIP = True
-except:
-    HAS_GEOIP = False
 
-from django.contrib.gis.utils.wkt import precision_wkt
+# GeoIP now lives in `django.contrib.gis.geoip`; this shortcut will be
+# removed in Django 1.6.
+from django.contrib.gis.utils import geoip
+HAS_GEOIP = geoip.HAS_GEOIP
+if HAS_GEOIP:
+    GeoIP = geoip.GeoIP
+    GeoIPException = geoip.GeoIPException
 
+from django.contrib.gis.utils.wkt import precision_wkt

docs/internals/deprecation.txt

@@ -239,6 +239,10 @@ their deprecation, as per the :ref:`deprecation policy
       were deprecated since Django 1.4 and will be removed in favor
       of the ``django.utils.text.Truncator`` class.
 
+    * The :class:`~django.contrib.gis.geoip.GeoIP` class was moved to
+      :mod:`django.contrib.gis.geoip` in 1.4 -- the shortcut in
+      :mod:`django.contrib.gis.utils` will be removed.
+
 2.0
 ---
 

docs/ref/contrib/gis/geoip.txt

@@ -4,10 +4,17 @@
 Geolocation with GeoIP
 ======================
 
-.. module:: django.contrib.gis.utils.geoip
+.. module:: django.contrib.gis.geoip
    :synopsis: High-level Python interface for MaxMind's GeoIP C library.
 
-.. currentmodule:: django.contrib.gis.utils
+.. versionchanged:: 1.4
+
+.. note::
+
+    In Django 1.4, the :class:`GeoIP` object was moved out of
+    :mod:`django.contrib.gis.utils` and into its own module, 
+    :mod:`django.contrib.gis.geoip`. A shortcut is still provided
+    in ``utils``, but will be removed in Django 1.6.
 
 The :class:`GeoIP` object is a ctypes wrapper for the
 `MaxMind GeoIP C API`__. [#]_  This interface is a BSD-licensed alternative
@@ -136,7 +143,7 @@ Querying
 
 All the following querying routines may take either a string IP address
 or a fully qualified domain name (FQDN).  For example, both 
-``'24.124.1.80'`` and ``'djangoproject.com'`` would be valid query 
+``'205.186.163.125'`` and ``'djangoproject.com'`` would be valid query 
 parameters. 
 
 .. method:: GeoIP.city(query)
@@ -144,7 +151,7 @@ parameters.
 Returns a dictionary of city information for the given query.  Some
 of the values in the dictionary may be undefined (``None``).
 
-.. method:: GeoIPcountry(query)
+.. method:: GeoIP.country(query)
 
 Returns a dictionary with the country code and country for the given 
 query.
@@ -190,7 +197,7 @@ This property returns information about the GeoIP city database.
 .. attribute:: GeoIP.info
 
 This property returns information about all GeoIP databases (both city
-and country).
+and country), and the version of the GeoIP C library (if supported).
 
 GeoIP-Python API compatibility methods
 ----------------------------------------

docs/ref/contrib/gis/index.txt

@@ -22,6 +22,7 @@ of spatially enabled data.
    measure
    geos
    gdal
+   geoip
    utils
    commands
    admin

docs/ref/contrib/gis/utils.txt

@@ -13,20 +13,5 @@ useful in creating geospatial Web applications.
 .. toctree::
    :maxdepth: 2
 
-   geoip
    layermapping
    ogrinspect
-
-GeoIP
-=====
-
-Interface to the MaxMind GeoIP library for performing IP-based geolocation
-from GeoDjango.  See :ref:`GeoIP reference <ref-geoip>` documentation for
-more information.
-
-LayerMapping
-============
-
-The :class:`~django.contrib.gis.utils.LayerMapping` simplifies the process
-of importing spatial data and attributes into your GeoDjango models.
-