Fixed #10109 -- Removed the use of raw SQL in many-to-many fields by introducing…

Fixed #10109 -- Removed the use of raw SQL in many-to-many fields by introducing an autogenerated through model.

This is the first part of Alex Gaynor's GSoC project to add Multi-db support to Django.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@11710 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/admin/options.py

@@ -153,8 +153,9 @@ class BaseModelAdmin(object):
         """
         Get a form Field for a ManyToManyField.
         """
-        # If it uses an intermediary model, don't show field in admin.
-        if db_field.rel.through is not None:
+        # If it uses an intermediary model that isn't auto created, don't show
+        # a field in admin.
+        if not db_field.rel.through._meta.auto_created:
             return None
 
         if db_field.name in self.raw_id_fields:

django/contrib/contenttypes/generic.py

@@ -105,8 +105,6 @@ class GenericRelation(RelatedField, Field):
                             limit_choices_to=kwargs.pop('limit_choices_to', None),
                             symmetrical=kwargs.pop('symmetrical', True))
 
-        # By its very nature, a GenericRelation doesn't create a table.
-        self.creates_table = False
 
         # Override content-type/object-id field names on the related class
         self.object_id_field_name = kwargs.pop("object_id_field", "object_id")

django/core/management/commands/syncdb.py

@@ -57,12 +57,15 @@ class Command(NoArgsCommand):
         # Create the tables for each model
         for app in models.get_apps():
             app_name = app.__name__.split('.')[-2]
-            model_list = models.get_models(app)
+            model_list = models.get_models(app, include_auto_created=True)
             for model in model_list:
                 # Create the model's database table, if it doesn't already exist.
                 if verbosity >= 2:
                     print "Processing %s.%s model" % (app_name, model._meta.object_name)
-                if connection.introspection.table_name_converter(model._meta.db_table) in tables:
+                opts = model._meta
+                if (connection.introspection.table_name_converter(opts.db_table) in tables or
+                    (opts.auto_created and
+                    connection.introspection.table_name_converter(opts.auto_created._meta.db_table in tables))):
                     continue
                 sql, references = connection.creation.sql_create_model(model, self.style, seen_models)
                 seen_models.add(model)
@@ -78,19 +81,6 @@ class Command(NoArgsCommand):
                     cursor.execute(statement)
                 tables.append(connection.introspection.table_name_converter(model._meta.db_table))
 
-        # Create the m2m tables. This must be done after all tables have been created
-        # to ensure that all referred tables will exist.
-        for app in models.get_apps():
-            app_name = app.__name__.split('.')[-2]
-            model_list = models.get_models(app)
-            for model in model_list:
-                if model in created_models:
-                    sql = connection.creation.sql_for_many_to_many(model, self.style)
-                    if sql:
-                        if verbosity >= 2:
-                            print "Creating many-to-many tables for %s.%s model" % (app_name, model._meta.object_name)
-                        for statement in sql:
-                            cursor.execute(statement)
 
         transaction.commit_unless_managed()
 

django/core/management/sql.py

@@ -23,7 +23,7 @@ def sql_create(app, style):
     # We trim models from the current app so that the sqlreset command does not
     # generate invalid SQL (leaving models out of known_models is harmless, so
     # we can be conservative).
-    app_models = models.get_models(app)
+    app_models = models.get_models(app, include_auto_created=True)
     final_output = []
     tables = connection.introspection.table_names()
     known_models = set([model for model in connection.introspection.installed_models(tables) if model not in app_models])
@@ -40,10 +40,6 @@ def sql_create(app, style):
         # Keep track of the fact that we've created the table for this model.
         known_models.add(model)
 
-    # Create the many-to-many join tables.
-    for model in app_models:
-        final_output.extend(connection.creation.sql_for_many_to_many(model, style))
-
     # Handle references to tables that are from other apps
     # but don't exist physically.
     not_installed_models = set(pending_references.keys())
@@ -82,7 +78,7 @@ def sql_delete(app, style):
     to_delete = set()
 
     references_to_delete = {}
-    app_models = models.get_models(app)
+    app_models = models.get_models(app, include_auto_created=True)
     for model in app_models:
         if cursor and connection.introspection.table_name_converter(model._meta.db_table) in table_names:
             # The table exists, so it needs to be dropped
@@ -97,13 +93,6 @@ def sql_delete(app, style):
         if connection.introspection.table_name_converter(model._meta.db_table) in table_names:
             output.extend(connection.creation.sql_destroy_model(model, references_to_delete, style))
 
-    # Output DROP TABLE statements for many-to-many tables.
-    for model in app_models:
-        opts = model._meta
-        for f in opts.local_many_to_many:
-            if cursor and connection.introspection.table_name_converter(f.m2m_db_table()) in table_names:
-                output.extend(connection.creation.sql_destroy_many_to_many(model, f, style))
-
     # Close database connection explicitly, in case this output is being piped
     # directly into a database client, to avoid locking issues.
     if cursor:

django/core/serializers/python.py

@@ -56,7 +56,7 @@ class Serializer(base.Serializer):
         self._current[field.name] = smart_unicode(related, strings_only=True)
 
     def handle_m2m_field(self, obj, field):
-        if field.creates_table:
+        if field.rel.through._meta.auto_created:
             self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True)
                                for related in getattr(obj, field.name).iterator()]
 

django/core/serializers/xml_serializer.py

@@ -98,7 +98,7 @@ class Serializer(base.Serializer):
         serialized as references to the object's PK (i.e. the related *data*
         is not dumped, just the relation).
         """
-        if field.creates_table:
+        if field.rel.through._meta.auto_created:
             self._start_relational_field(field)
             for relobj in getattr(obj, field.name).iterator():
                 self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
@@ -233,4 +233,3 @@ def getInnerText(node):
         else:
            pass
     return u"".join(inner_text)
-

django/db/models/base.py

@@ -434,7 +434,7 @@ class Model(object):
         else:
             meta = cls._meta
 
-        if origin:
+        if origin and not meta.auto_created:
             signals.pre_save.send(sender=origin, instance=self, raw=raw)
 
         # If we are in a raw save, save the object exactly as presented.
@@ -507,7 +507,7 @@ class Model(object):
                     setattr(self, meta.pk.attname, result)
             transaction.commit_unless_managed()
 
-        if origin:
+        if origin and not meta.auto_created:
             signals.post_save.send(sender=origin, instance=self,
                 created=(not record_exists), raw=raw)
 
@@ -544,7 +544,12 @@ class Model(object):
                         rel_descriptor = cls.__dict__[rel_opts_name]
                         break
                 else:
-                    raise AssertionError("Should never get here.")
+                    # in the case of a hidden fkey just skip it, it'll get
+                    # processed as an m2m
+                    if not related.field.rel.is_hidden():
+                        raise AssertionError("Should never get here.")
+                    else:
+                        continue
                 delete_qs = rel_descriptor.delete_manager(self).all()
                 for sub_obj in delete_qs:
                     sub_obj._collect_sub_objects(seen_objs, self.__class__, related.field.null)

django/db/models/loading.py

@@ -131,19 +131,25 @@ class AppCache(object):
         self._populate()
         return self.app_errors
 
-    def get_models(self, app_mod=None):
+    def get_models(self, app_mod=None, include_auto_created=False):
         """
         Given a module containing models, returns a list of the models.
         Otherwise returns a list of all installed models.
+
+        By default, auto-created models (i.e., m2m models without an
+        explicit intermediate table) are not included. However, if you
+        specify include_auto_created=True, they will be.
         """
         self._populate()
         if app_mod:
-            return self.app_models.get(app_mod.__name__.split('.')[-2], SortedDict()).values()
+            model_list = self.app_models.get(app_mod.__name__.split('.')[-2], SortedDict()).values()
         else:
             model_list = []
             for app_entry in self.app_models.itervalues():
                 model_list.extend(app_entry.values())
-            return model_list
+        if not include_auto_created:
+            return filter(lambda o: not o._meta.auto_created, model_list)
+        return model_list
 
     def get_model(self, app_label, model_name, seed_cache=True):
         """

django/db/models/options.py

@@ -21,7 +21,7 @@ get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|
 DEFAULT_NAMES = ('verbose_name', 'db_table', 'ordering',
                  'unique_together', 'permissions', 'get_latest_by',
                  'order_with_respect_to', 'app_label', 'db_tablespace',
-                 'abstract', 'managed', 'proxy')
+                 'abstract', 'managed', 'proxy', 'auto_created')
 
 class Options(object):
     def __init__(self, meta, app_label=None):
@@ -47,6 +47,7 @@ class Options(object):
         self.proxy_for_model = None
         self.parents = SortedDict()
         self.duplicate_targets = {}
+        self.auto_created = False
 
         # To handle various inheritance situations, we need to track where
         # managers came from (concrete or abstract base classes).
@@ -487,4 +488,3 @@ class Options(object):
         Returns the index of the primary key field in the self.fields list.
         """
         return self.fields.index(self.pk)
-

django/db/models/query.py

@@ -1028,7 +1028,8 @@ def delete_objects(seen_objs):
 
             # Pre-notify all instances to be deleted.
             for pk_val, instance in items:
-                signals.pre_delete.send(sender=cls, instance=instance)
+                if not cls._meta.auto_created:
+                    signals.pre_delete.send(sender=cls, instance=instance)
 
             pk_list = [pk for pk,instance in items]
             del_query = sql.DeleteQuery(cls, connection)
@@ -1062,7 +1063,8 @@ def delete_objects(seen_objs):
                     if field.rel and field.null and field.rel.to in seen_objs:
                         setattr(instance, field.attname, None)
 
-                signals.post_delete.send(sender=cls, instance=instance)
+                if not cls._meta.auto_created:
+                    signals.post_delete.send(sender=cls, instance=instance)
                 setattr(instance, cls._meta.pk.attname, None)
 
         if forced_managed:

docs/internals/deprecation.txt

@@ -25,6 +25,9 @@ their deprecation, as per the :ref:`Django deprecation policy
         * ``SMTPConnection``. The 1.2 release deprecated the ``SMTPConnection``
           class in favor of a generic E-mail backend API.
 
+        * The many to many SQL generation functions on the database backends
+          will be removed.  These have been deprecated since the 1.2 release.
+
     * 2.0
         * ``django.views.defaults.shortcut()``. This function has been moved
           to ``django.contrib.contenttypes.views.shortcut()`` as part of the

tests/modeltests/invalid_models/models.py

@@ -182,6 +182,7 @@ class UniqueM2M(models.Model):
     """ Model to test for unique ManyToManyFields, which are invalid. """
     unique_people = models.ManyToManyField( Person, unique=True )
 
+
 model_errors = """invalid_models.fielderrors: "charfield": CharFields require a "max_length" attribute.
 invalid_models.fielderrors: "decimalfield": DecimalFields require a "decimal_places" attribute.
 invalid_models.fielderrors: "decimalfield": DecimalFields require a "max_digits" attribute.

tests/modeltests/m2m_through/models.py

@@ -133,7 +133,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'add'
 >>> rock.members.create(name='Anne')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use m2m_through.Membership's Manager instead.
 
 # Remove has similar complications, and is not provided either.
 >>> rock.members.remove(jim)
@@ -160,7 +160,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'remove'
 >>> rock.members = backup
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through.Membership's Manager instead.
 
 # Let's re-save those instances that we've cleared.
 >>> m1.save()
@@ -184,7 +184,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'add'
 >>> bob.group_set.create(name='Funk')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use m2m_through.Membership's Manager instead.
 
 # Remove has similar complications, and is not provided either.
 >>> jim.group_set.remove(rock)
@@ -209,7 +209,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'remove'
 >>> jim.group_set = backup
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through.Membership's Manager instead.
 
 # Let's re-save those instances that we've cleared.
 >>> m1.save()
@@ -334,4 +334,4 @@ AttributeError: Cannot set values on a ManyToManyField which specifies an interm
 # QuerySet's distinct() method can correct this problem.
 >>> Person.objects.filter(membership__date_joined__gt=datetime(2004, 1, 1)).distinct()
 [<Person: Jane>, <Person: Jim>]
-"""}
\ No newline at end of file
+"""}

tests/regressiontests/m2m_through_regress/models.py

@@ -84,22 +84,22 @@ __test__ = {'API_TESTS':"""
 >>> bob.group_set = []
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 >>> roll.members = []
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 >>> rock.members.create(name='Anne')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 >>> bob.group_set.create(name='Funk')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 # Now test that the intermediate with a relationship outside
 # the current app (i.e., UserMembership) workds

tests/regressiontests/model_inheritance_regress/models.py

@@ -110,6 +110,36 @@ class DerivedM(BaseM):
         return "PK = %d, base_name = %s, derived_name = %s" \
                 % (self.customPK, self.base_name, self.derived_name)
 
+# Check that abstract classes don't get m2m tables autocreated.
+class Person(models.Model):
+    name = models.CharField(max_length=100)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return self.name
+
+class AbstractEvent(models.Model):
+    name = models.CharField(max_length=100)
+    attendees = models.ManyToManyField(Person, related_name="%(class)s_set")
+
+    class Meta:
+        abstract = True
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return self.name
+
+class BirthdayParty(AbstractEvent):
+    pass
+
+class BachelorParty(AbstractEvent):
+    pass
+
+class MessyBachelorParty(BachelorParty):
+    pass
+
 __test__ = {'API_TESTS':"""
 # Regression for #7350, #7202
 # Check that when you create a Parent object with a specific reference to an
@@ -318,5 +348,41 @@ True
 >>> ParkingLot3._meta.get_ancestor_link(Place).name  # the child->parent link
 "parent"
 
+# Check that many-to-many relations defined on an abstract base class
+# are correctly inherited (and created) on the child class.
+>>> p1 = Person.objects.create(name='Alice')
+>>> p2 = Person.objects.create(name='Bob')
+>>> p3 = Person.objects.create(name='Carol')
+>>> p4 = Person.objects.create(name='Dave')
+
+>>> birthday = BirthdayParty.objects.create(name='Birthday party for Alice')
+>>> birthday.attendees = [p1, p3]
+
+>>> bachelor = BachelorParty.objects.create(name='Bachelor party for Bob')
+>>> bachelor.attendees = [p2, p4]
+
+>>> print p1.birthdayparty_set.all()
+[<BirthdayParty: Birthday party for Alice>]
+
+>>> print p1.bachelorparty_set.all()
+[]
+
+>>> print p2.bachelorparty_set.all()
+[<BachelorParty: Bachelor party for Bob>]
+
+# Check that a subclass of a subclass of an abstract model
+# doesn't get it's own accessor.
+>>> p2.messybachelorparty_set.all()
+Traceback (most recent call last):
+...
+AttributeError: 'Person' object has no attribute 'messybachelorparty_set'
+
+# ... but it does inherit the m2m from it's parent
+>>> messy = MessyBachelorParty.objects.create(name='Bachelor party for Dave')
+>>> messy.attendees = [p4]
+
+>>> p4.bachelorparty_set.all()
+[<BachelorParty: Bachelor party for Bob>, <BachelorParty: Bachelor party for Dave>]
+
 """}
 

tests/regressiontests/serializers_regress/models.py

@@ -105,6 +105,9 @@ class Anchor(models.Model):
 
     data = models.CharField(max_length=30)
 
+    class Meta:
+        ordering = ('id',)
+
 class UniqueAnchor(models.Model):
     """This is a model that can be used as
     something for other models to point at"""
@@ -135,7 +138,7 @@ class FKDataToO2O(models.Model):
 
 class M2MIntermediateData(models.Model):
     data = models.ManyToManyField(Anchor, null=True, through='Intermediate')
-    
+
 class Intermediate(models.Model):
     left = models.ForeignKey(M2MIntermediateData)
     right = models.ForeignKey(Anchor)
@@ -242,7 +245,7 @@ class AbstractBaseModel(models.Model):
 
 class InheritAbstractModel(AbstractBaseModel):
     child_data = models.IntegerField()
-    
+
 class BaseModel(models.Model):
     parent_data = models.IntegerField()
 
@@ -252,4 +255,3 @@ class InheritBaseModel(BaseModel):
 class ExplicitInheritBaseModel(BaseModel):
     parent = models.OneToOneField(BaseModel)
     child_data = models.IntegerField()
-    
\ No newline at end of file

tests/regressiontests/signals_regress/models.py

+"""
+Testing signals before/after saving and deleting.
+"""
+
+from django.db import models
+
+class Author(models.Model):
+    name = models.CharField(max_length=20)
+
+    def __unicode__(self):
+        return self.name
+
+class Book(models.Model):
+    name = models.CharField(max_length=20)
+    authors = models.ManyToManyField(Author)
+
+    def __unicode__(self):
+        return self.name
+
+def pre_save_test(signal, sender, instance, **kwargs):
+    print 'pre_save signal,', instance
+    if kwargs.get('raw'):
+        print 'Is raw'
+
+def post_save_test(signal, sender, instance, **kwargs):
+    print 'post_save signal,', instance
+    if 'created' in kwargs:
+        if kwargs['created']:
+            print 'Is created'
+        else:
+            print 'Is updated'
+    if kwargs.get('raw'):
+        print 'Is raw'
+
+def pre_delete_test(signal, sender, instance, **kwargs):
+    print 'pre_delete signal,', instance
+    print 'instance.id is not None: %s' % (instance.id != None)
+
+def post_delete_test(signal, sender, instance, **kwargs):
+    print 'post_delete signal,', instance
+    print 'instance.id is not None: %s' % (instance.id != None)
+
+__test__ = {'API_TESTS':"""
+
+# Save up the number of connected signals so that we can check at the end
+# that all the signals we register get properly unregistered (#9989)
+>>> pre_signals = (len(models.signals.pre_save.receivers),
+...                len(models.signals.post_save.receivers),
+...                len(models.signals.pre_delete.receivers),
+...                len(models.signals.post_delete.receivers))
+
+>>> models.signals.pre_save.connect(pre_save_test)
+>>> models.signals.post_save.connect(post_save_test)
+>>> models.signals.pre_delete.connect(pre_delete_test)
+>>> models.signals.post_delete.connect(post_delete_test)
+
+>>> a1 = Author(name='Neal Stephenson')
+>>> a1.save()
+pre_save signal, Neal Stephenson
+post_save signal, Neal Stephenson
+Is created
+
+>>> b1 = Book(name='Snow Crash')
+>>> b1.save()
+pre_save signal, Snow Crash
+post_save signal, Snow Crash
+Is created
+
+# Assigning to m2m shouldn't generate an m2m signal
+>>> b1.authors = [a1]
+
+# Removing an author from an m2m shouldn't generate an m2m signal
+>>> b1.authors = []
+
+>>> models.signals.post_delete.disconnect(post_delete_test)
+>>> models.signals.pre_delete.disconnect(pre_delete_test)
+>>> models.signals.post_save.disconnect(post_save_test)
+>>> models.signals.pre_save.disconnect(pre_save_test)
+
+# Check that all our signals got disconnected properly.
+>>> post_signals = (len(models.signals.pre_save.receivers),
+...                 len(models.signals.post_save.receivers),
+...                 len(models.signals.pre_delete.receivers),
+...                 len(models.signals.post_delete.receivers))
+
+>>> pre_signals == post_signals
+True
+
+"""}