Removed global timezone-aware datetime adapters.

Refs #23820.

Fixed #19738.

Refs #17755. In order not to introduce a regression for raw queries,
parameters are passed through the connection.ops.value_to_db_* methods,
depending on their type.

django/db/backends/base/operations.py

@@ -433,6 +433,25 @@ class BaseDatabaseOperations(object):
         """
         return value
 
+    def value_to_db_unknown(self, value):
+        """
+        Transforms a value to something compatible with the backend driver.
+
+        This method only depends on the type of the value. It's designed for
+        cases where the target type isn't known, such as .raw() SQL queries.
+        As a consequence it may not work perfectly in all circumstances.
+        """
+        if isinstance(value, datetime.datetime):   # must be before date
+            return self.value_to_db_datetime(value)
+        elif isinstance(value, datetime.date):
+            return self.value_to_db_date(value)
+        elif isinstance(value, datetime.time):
+            return self.value_to_db_time(value)
+        elif isinstance(value, decimal.Decimal):
+            return self.value_to_db_decimal(value)
+        else:
+            return value
+
     def value_to_db_date(self, value):
         """
         Transforms a date value to an object compatible with what is expected
@@ -486,6 +505,8 @@ class BaseDatabaseOperations(object):
         """
         first = datetime.date(value, 1, 1)
         second = datetime.date(value, 12, 31)
+        first = self.value_to_db_date(first)
+        second = self.value_to_db_date(second)
         return [first, second]
 
     def year_lookup_bounds_for_datetime_field(self, value):
@@ -502,6 +523,8 @@ class BaseDatabaseOperations(object):
             tz = timezone.get_current_timezone()
             first = timezone.make_aware(first, tz)
             second = timezone.make_aware(second, tz)
+        first = self.value_to_db_datetime(first)
+        second = self.value_to_db_datetime(second)
         return [first, second]
 
     def get_db_converters(self, expression):

django/db/backends/mysql/base.py

@@ -16,6 +16,7 @@ from django.db import utils
 from django.db.backends import utils as backend_utils
 from django.db.backends.base.base import BaseDatabaseWrapper
 from django.utils import six, timezone
+from django.utils.deprecation import RemovedInDjango21Warning
 from django.utils.encoding import force_str
 from django.utils.functional import cached_property
 from django.utils.safestring import SafeBytes, SafeText
@@ -52,15 +53,14 @@ DatabaseError = Database.DatabaseError
 IntegrityError = Database.IntegrityError
 
 
-def adapt_datetime_with_timezone_support(value, conv):
-    # Equivalent to DateTimeField.get_db_prep_value. Used only by raw SQL.
-    if settings.USE_TZ:
-        if timezone.is_naive(value):
-            warnings.warn("MySQL received a naive datetime (%s)"
-                          " while time zone support is active." % value,
-                          RuntimeWarning)
-            default_timezone = timezone.get_default_timezone()
-            value = timezone.make_aware(value, default_timezone)
+def adapt_datetime_warn_on_aware_datetime(value, conv):
+    # Remove this function and rely on the default adapter in Django 2.1.
+    if settings.USE_TZ and timezone.is_aware(value):
+        warnings.warn(
+            "The MySQL database adapter received an aware datetime (%s), "
+            "probably from cursor.execute(). Update your code to pass a "
+            "naive datetime in the database connection's time zone (UTC by "
+            "default).", RemovedInDjango21Warning)
         value = value.astimezone(timezone.utc).replace(tzinfo=None)
     return Thing2Literal(value.strftime("%Y-%m-%d %H:%M:%S.%f"), conv)
 
@@ -74,7 +74,7 @@ django_conversions.update({
     FIELD_TYPE.TIME: backend_utils.typecast_time,
     FIELD_TYPE.DECIMAL: backend_utils.typecast_decimal,
     FIELD_TYPE.NEWDECIMAL: backend_utils.typecast_decimal,
-    datetime.datetime: adapt_datetime_with_timezone_support,
+    datetime.datetime: adapt_datetime_warn_on_aware_datetime,
 })
 
 # This should match the numerical portion of the version numbers (we can treat

django/db/backends/oracle/base.py

@@ -17,6 +17,7 @@ from django.db import utils
 from django.db.backends.base.base import BaseDatabaseWrapper
 from django.db.backends.base.validation import BaseDatabaseValidation
 from django.utils import six, timezone
+from django.utils.deprecation import RemovedInDjango21Warning
 from django.utils.duration import duration_string
 from django.utils.encoding import force_bytes, force_text
 from django.utils.functional import cached_property
@@ -336,13 +337,14 @@ class OracleParam(object):
         # without being converted by DateTimeField.get_db_prep_value.
         if settings.USE_TZ and (isinstance(param, datetime.datetime) and
                                 not isinstance(param, Oracle_datetime)):
-            if timezone.is_naive(param):
-                warnings.warn("Oracle received a naive datetime (%s)"
-                              " while time zone support is active." % param,
-                              RuntimeWarning)
-                default_timezone = timezone.get_default_timezone()
-                param = timezone.make_aware(param, default_timezone)
-            param = Oracle_datetime.from_datetime(param.astimezone(timezone.utc))
+            if timezone.is_aware(param):
+                warnings.warn(
+                    "The Oracle database adapter received an aware datetime (%s), "
+                    "probably from cursor.execute(). Update your code to pass a "
+                    "naive datetime in the database connection's time zone (UTC by "
+                    "default).", RemovedInDjango21Warning)
+                param = param.astimezone(timezone.utc).replace(tzinfo=None)
+            param = Oracle_datetime.from_datetime(param)
 
         if isinstance(param, datetime.timedelta):
             param = duration_string(param)

django/db/backends/oracle/operations.py

@@ -419,19 +419,6 @@ WHEN (new.%(col_name)s IS NULL)
         return Oracle_datetime(1900, 1, 1, value.hour, value.minute,
                                value.second, value.microsecond)
 
-    def year_lookup_bounds_for_date_field(self, value):
-        # Create bounds as real date values
-        first = datetime.date(value, 1, 1)
-        last = datetime.date(value, 12, 31)
-        return [first, last]
-
-    def year_lookup_bounds_for_datetime_field(self, value):
-        # cx_Oracle doesn't support tz-aware datetimes
-        bounds = super(DatabaseOperations, self).year_lookup_bounds_for_datetime_field(value)
-        if settings.USE_TZ:
-            bounds = [b.astimezone(timezone.utc) for b in bounds]
-        return [Oracle_datetime.from_datetime(b) for b in bounds]
-
     def combine_expression(self, connector, sub_expressions):
         "Oracle requires special cases for %% and & operators in query expressions"
         if connector == '%%':

django/db/backends/sqlite3/base.py

@@ -20,6 +20,7 @@ from django.utils import six, timezone
 from django.utils.dateparse import (
     parse_date, parse_datetime, parse_duration, parse_time,
 )
+from django.utils.deprecation import RemovedInDjango21Warning
 from django.utils.encoding import force_text
 from django.utils.safestring import SafeBytes
 
@@ -49,15 +50,14 @@ DatabaseError = Database.DatabaseError
 IntegrityError = Database.IntegrityError
 
 
-def adapt_datetime_with_timezone_support(value):
-    # Equivalent to DateTimeField.get_db_prep_value. Used only by raw SQL.
-    if settings.USE_TZ:
-        if timezone.is_naive(value):
-            warnings.warn("SQLite received a naive datetime (%s)"
-                          " while time zone support is active." % value,
-                          RuntimeWarning)
-            default_timezone = timezone.get_default_timezone()
-            value = timezone.make_aware(value, default_timezone)
+def adapt_datetime_warn_on_aware_datetime(value):
+    # Remove this function and rely on the default adapter in Django 2.1.
+    if settings.USE_TZ and timezone.is_aware(value):
+        warnings.warn(
+            "The SQLite database adapter received an aware datetime (%s), "
+            "probably from cursor.execute(). Update your code to pass a "
+            "naive datetime in the database connection's time zone (UTC by "
+            "default).", RemovedInDjango21Warning)
         value = value.astimezone(timezone.utc).replace(tzinfo=None)
     return value.isoformat(str(" "))
 
@@ -77,7 +77,7 @@ Database.register_converter(str("timestamp"), decoder(parse_datetime))
 Database.register_converter(str("TIMESTAMP"), decoder(parse_datetime))
 Database.register_converter(str("decimal"), decoder(backend_utils.typecast_decimal))
 
-Database.register_adapter(datetime.datetime, adapt_datetime_with_timezone_support)
+Database.register_adapter(datetime.datetime, adapt_datetime_warn_on_aware_datetime)
 Database.register_adapter(decimal.Decimal, backend_utils.rev_typecast_decimal)
 if six.PY2:
     Database.register_adapter(str, lambda s: s.decode('utf-8'))

django/db/models/sql/query.py

@@ -86,13 +86,29 @@ class RawQuery(object):
     def __repr__(self):
         return "<RawQuery: %s>" % self
 
+    @property
+    def params_type(self):
+        return dict if isinstance(self.params, Mapping) else tuple
+
     def __str__(self):
-        _type = dict if isinstance(self.params, Mapping) else tuple
-        return self.sql % _type(self.params)
+        return self.sql % self.params_type(self.params)
 
     def _execute_query(self):
-        self.cursor = connections[self.using].cursor()
-        self.cursor.execute(self.sql, self.params)
+        connection = connections[self.using]
+
+        # Adapt parameters to the database, as much as possible considering
+        # that the target type isn't known. See #17755.
+        params_type = self.params_type
+        adapter = connection.ops.value_to_db_unknown
+        if params_type is tuple:
+            params = tuple(adapter(val) for val in self.params)
+        elif params_type is dict:
+            params = dict((key, adapter(val)) for key, val in six.iteritems(self.params))
+        else:
+            raise RuntimeError("Unexpected params type: %s" % params_type)
+
+        self.cursor = connection.cursor()
+        self.cursor.execute(self.sql, params)
 
 
 class Query(object):

docs/internals/deprecation.txt

@@ -32,6 +32,11 @@ details on these changes.
 
 * ``django.db.models.fields.add_lazy_relation()`` will be removed.
 
+* When time zone support is enabled, database backends that don't support time
+  zones won't convert aware datetimes to naive values in UTC anymore when such
+  values are passed as parameters to SQL queries executed outside of the ORM,
+  e.g. with ``cursor.execute()``.
+
 * The ``django.contrib.auth.tests.utils.skipIfCustomUser()`` decorator will be
   removed.
 

docs/releases/1.9.txt

@@ -313,11 +313,15 @@ Database backend API
   doesn't implement this. You may want to review the implementation on the
   backends that Django includes for reference (:ticket:`24245`).
 
-* The recommended way to add time zone information to datetimes fetched from
-  databases that don't support time zones is to register a converter for
-  ``DateTimeField``. Do this in ``DatabaseOperations.get_db_converters()``.
-  Registering a global converter at the level of the DB-API module is
-  discouraged because it can conflict with other libraries.
+* Registering a global adapter or converter at the level of the DB-API module
+  to handle time zone information of :class:`~datetime.datetime` values passed
+  as query parameters or returned as query results on databases that don't
+  support time zones is discouraged. It can conflict with other libraries.
+
+  The recommended way to add a time zone to :class:`~datetime.datetime` values
+  fetched from the database is to register a converter for ``DateTimeField``
+  in ``DatabaseOperations.get_db_converters()``.
+
 
 Default settings that were tuples are now lists
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -418,20 +422,40 @@ console, for example.
 If you are overriding Django's default logging, you should check to see how
 your configuration merges with the new defaults.
 
-Removal of time zone aware global converters for datetimes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Removal of time zone aware global adapters and converters for datetimes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Django no longer registers global converters for returning time zone aware
-datetimes in database query results when :setting:`USE_TZ` is ``True``.
-Instead the ORM adds suitable time zone information.
+Django no longer registers global adapters and converters for managing time
+zone information on :class:`~datetime.datetime` values sent to the database as
+query parameters or read from the database in query results. This change
+affects projects that meet all the following conditions:
 
-As a consequence, SQL queries executed outside of the ORM, for instance with
-``cursor.execute(query, params)``, now return naive datetimes instead of aware
-datetimes on databases that do not support time zones: SQLite, MySQL, and
-Oracle. Since these datetimes are in UTC, you can make them aware as follows::
+* The :setting:`USE_TZ` setting is ``True``.
+* The database is SQLite, MySQL, Oracle, or a third-party database that
+  doesn't support time zones. In doubt, you can check the value of
+  ``connection.features.supports_timezones``.
+* The code queries the database outside of the ORM, typically with
+  ``cursor.execute(sql, params)``.
+
+If you're passing aware :class:`~datetime.datetime` parameters to such
+queries, you should turn them into naive datetimes in UTC::
 
     from django.utils import timezone
-    value = value.replace(tzinfo=timezone.utc)
+    param = timezone.make_naive(param, timezone.utc)
+
+If you fail to do so, Django 1.9 and 2.0 will perform the conversion like
+earlier versions but emit a deprecation warning. Django 2.1 won't perform any
+conversion, which may result in data corruption.
+
+If you're reading :class:`~datetime.datetime` values from the results, they
+will be naive instead of aware. You can compensate as follows::
+
+    from django.utils import timezone
+    value = timezone.make_aware(value, timezone.utc)
+
+You don't need any of this if you're querying the database through the ORM,
+even if you're using :meth:`raw() <django.db.models.query.QuerySet.raw>`
+queries. The ORM takes care of managing time zone information.
 
 Miscellaneous
 ~~~~~~~~~~~~~

tests/timezones/tests.py

@@ -266,6 +266,13 @@ class LegacyDatabaseTests(TestCase):
             [event],
             transform=lambda d: d)
 
+    def test_cursor_execute_accepts_naive_datetime(self):
+        dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
+        with connection.cursor() as cursor:
+            cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [dt])
+        event = Event.objects.get()
+        self.assertEqual(event.dt, dt)
+
     def test_cursor_execute_returns_naive_datetime(self):
         dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
         Event.objects.create(dt=dt)
@@ -564,6 +571,23 @@ class NewDatabaseTests(TestCase):
             [event],
             transform=lambda d: d)
 
+    @skipUnlessDBFeature('supports_timezones')
+    def test_cursor_execute_accepts_aware_datetime(self):
+        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
+        with connection.cursor() as cursor:
+            cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [dt])
+        event = Event.objects.get()
+        self.assertEqual(event.dt, dt)
+
+    @skipIfDBFeature('supports_timezones')
+    def test_cursor_execute_accepts_naive_datetime(self):
+        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
+        utc_naive_dt = timezone.make_naive(dt, timezone.utc)
+        with connection.cursor() as cursor:
+            cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [utc_naive_dt])
+        event = Event.objects.get()
+        self.assertEqual(event.dt, dt)
+
     @skipUnlessDBFeature('supports_timezones')
     def test_cursor_execute_returns_aware_datetime(self):
         dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
@@ -578,7 +602,7 @@ class NewDatabaseTests(TestCase):
         utc_naive_dt = timezone.make_naive(dt, timezone.utc)
         Event.objects.create(dt=dt)
         with connection.cursor() as cursor:
-            cursor.execute('SELECT dt FROM timezones_event WHERE dt = %s', [dt])
+            cursor.execute('SELECT dt FROM timezones_event WHERE dt = %s', [utc_naive_dt])
             self.assertEqual(cursor.fetchall()[0][0], utc_naive_dt)
 
     @requires_tz_support