Fixed #12422 -- Don't override global email charset behavior for utf-8.

Thanks simonb for the report, Claude Paroz and Susan Tan for their work
on a fix.

django/core/mail/message.py

@@ -22,7 +22,8 @@ from django.utils import six
 
 # Don't BASE64-encode UTF-8 messages so that we avoid unwanted attention from
 # some spam filters.
-Charset.add_charset('utf-8', Charset.SHORTEST, None, 'utf-8')
+utf8_charset = Charset.Charset('utf-8')
+utf8_charset.body_encoding = None  # Python defaults to BASE64
 
 # Default MIME type to use on attachments (if it is not explicitly given
 # and cannot be guessed).
@@ -145,7 +146,16 @@ class SafeMIMEText(MIMEText):
 
     def __init__(self, text, subtype, charset):
         self.encoding = charset
-        MIMEText.__init__(self, text, subtype, charset)
+        if charset == 'utf-8':
+            # Unfortunately, Python doesn't support setting a Charset instance
+            # as MIMEText init parameter (http://bugs.python.org/issue16324).
+            # We do it manually and trigger re-encoding of the payload.
+            MIMEText.__init__(self, text, subtype, None)
+            del self['Content-Transfer-Encoding']
+            self.set_payload(text, utf8_charset)
+            self.replace_header('Content-Type', 'text/%s; charset="%s"' % (subtype, charset))
+        else:
+            MIMEText.__init__(self, text, subtype, charset)
 
     def __setitem__(self, name, val):
         name, val = forbid_multi_line_headers(name, val, self.encoding)

tests/mail/tests.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import asyncore
 import email
+from email.mime.text import MIMEText
 import os
 import shutil
 import smtpd
@@ -20,11 +21,32 @@ from django.core.mail.message import BadHeaderError
 from django.test import TestCase
 from django.test.utils import override_settings
 from django.utils.encoding import force_str, force_text
-from django.utils.six import PY3, StringIO
+from django.utils.six import PY3, StringIO, string_types
 from django.utils.translation import ugettext_lazy
 
 
-class MailTests(TestCase):
+class HeadersCheckMixin(object):
+
+    def assertMessageHasHeaders(self, message, headers):
+        """
+        Check that :param message: has all :param headers: headers.
+
+        :param message: can be an instance of an email.Message subclass or a
+        string with the contens of an email message.
+        :param headers: should be a set of (header-name, header-value) tuples.
+        """
+        if isinstance(message, string_types):
+            just_headers = message.split('\n\n', 1)[0]
+            hlist = just_headers.split('\n')
+            pairs = [hl.split(':', 1) for hl in hlist]
+            msg_headers = {(n, v.lstrip()) for (n, v) in pairs}
+        else:
+            msg_headers = set(message.items())
+        self.assertTrue(headers.issubset(msg_headers), msg='Message is missing '
+                        'the following headers: %s' % (headers - msg_headers),)
+
+
+class MailTests(HeadersCheckMixin, TestCase):
     """
     Non-backend specific tests.
     """
@@ -93,7 +115,7 @@ class MailTests(TestCase):
         headers = {"date": "Fri, 09 Nov 2001 01:08:47 -0000", "Message-ID": "foo"}
         email = EmailMessage('subject', 'content', 'from@example.com', ['to@example.com'], headers=headers)
 
-        self.assertEqual(sorted(email.message().items()), [
+        self.assertMessageHasHeaders(email.message(), {
             ('Content-Transfer-Encoding', '7bit'),
             ('Content-Type', 'text/plain; charset="utf-8"'),
             ('From', 'from@example.com'),
@@ -102,7 +124,7 @@ class MailTests(TestCase):
             ('Subject', 'subject'),
             ('To', 'to@example.com'),
             ('date', 'Fri, 09 Nov 2001 01:08:47 -0000'),
-        ])
+        })
 
     def test_from_header(self):
         """
@@ -184,7 +206,13 @@ class MailTests(TestCase):
         email = EmailMessage('Subject', 'Firstname Sürname is a great guy.', 'from@example.com', ['other@example.com'])
         email.encoding = 'iso-8859-1'
         message = email.message()
-        self.assertTrue(message.as_string().startswith('Content-Type: text/plain; charset="iso-8859-1"\nMIME-Version: 1.0\nContent-Transfer-Encoding: quoted-printable\nSubject: Subject\nFrom: from@example.com\nTo: other@example.com'))
+        self.assertMessageHasHeaders(message, {
+            ('MIME-Version', '1.0'),
+            ('Content-Type', 'text/plain; charset="iso-8859-1"'),
+            ('Content-Transfer-Encoding', 'quoted-printable'),
+            ('Subject', 'Subject'),
+            ('From', 'from@example.com'),
+            ('To', 'other@example.com')})
         self.assertEqual(message.get_payload(), 'Firstname S=FCrname is a great guy.')
 
         # Make sure MIME attachments also works correctly with other encodings than utf-8
@@ -193,8 +221,18 @@ class MailTests(TestCase):
         msg = EmailMultiAlternatives('Subject', text_content, 'from@example.com', ['to@example.com'])
         msg.encoding = 'iso-8859-1'
         msg.attach_alternative(html_content, "text/html")
-        self.assertEqual(msg.message().get_payload(0).as_string(), 'Content-Type: text/plain; charset="iso-8859-1"\nMIME-Version: 1.0\nContent-Transfer-Encoding: quoted-printable\n\nFirstname S=FCrname is a great guy.')
-        self.assertEqual(msg.message().get_payload(1).as_string(), 'Content-Type: text/html; charset="iso-8859-1"\nMIME-Version: 1.0\nContent-Transfer-Encoding: quoted-printable\n\n<p>Firstname S=FCrname is a <strong>great</strong> guy.</p>')
+        payload0 = msg.message().get_payload(0)
+        self.assertMessageHasHeaders(payload0, {
+            ('MIME-Version', '1.0'),
+            ('Content-Type', 'text/plain; charset="iso-8859-1"'),
+            ('Content-Transfer-Encoding', 'quoted-printable')})
+        self.assertTrue(payload0.as_string().endswith('\n\nFirstname S=FCrname is a great guy.'))
+        payload1 = msg.message().get_payload(1)
+        self.assertMessageHasHeaders(payload1, {
+            ('MIME-Version', '1.0'),
+            ('Content-Type', 'text/html; charset="iso-8859-1"'),
+            ('Content-Transfer-Encoding', 'quoted-printable')})
+        self.assertTrue(payload1.as_string().endswith('\n\n<p>Firstname S=FCrname is a <strong>great</strong> guy.</p>'))
 
     def test_attachments(self):
         """Regression test for #9367"""
@@ -365,7 +403,31 @@ class MailTests(TestCase):
         self.assertTrue(str('Child Subject') in parent_s)
 
 
-class BaseEmailBackendTests(object):
+class PythonGlobalState(TestCase):
+    """
+    Tests for #12422 -- Django smarts (#2472/#11212) with charset of utf-8 text
+    parts shouldn't pollute global email Python package charset registry when
+    django.mail.message is imported.
+    """
+
+    def test_utf8(self):
+        txt = MIMEText('UTF-8 encoded body', 'plain', 'utf-8')
+        self.assertTrue('Content-Transfer-Encoding: base64' in txt.as_string())
+
+    def test_7bit(self):
+        txt = MIMEText('Body with only ASCII characters.', 'plain', 'utf-8')
+        self.assertTrue('Content-Transfer-Encoding: base64' in txt.as_string())
+
+    def test_8bit_latin(self):
+        txt = MIMEText('Body with latin characters: àáä.', 'plain', 'utf-8')
+        self.assertTrue(str('Content-Transfer-Encoding: base64') in txt.as_string())
+
+    def test_8bit_non_latin(self):
+        txt = MIMEText('Body with non latin characters: А Б В Г Д Е Ж Ѕ З И І К Л М Н О П.', 'plain', 'utf-8')
+        self.assertTrue(str('Content-Transfer-Encoding: base64') in txt.as_string())
+
+
+class BaseEmailBackendTests(HeadersCheckMixin, object):
     email_backend = None
 
     def setUp(self):
@@ -523,7 +585,15 @@ class BaseEmailBackendTests(object):
         email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com'], cc=['cc@example.com'])
         mail.get_connection().send_messages([email])
         message = self.get_the_message()
-        self.assertStartsWith(message.as_string(), 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nSubject: Subject\nFrom: from@example.com\nTo: to@example.com\nCc: cc@example.com\nDate: ')
+        self.assertMessageHasHeaders(message, {
+            ('MIME-Version', '1.0'),
+            ('Content-Type', 'text/plain; charset="utf-8"'),
+            ('Content-Transfer-Encoding', '7bit'),
+            ('Subject', 'Subject'),
+            ('From', 'from@example.com'),
+            ('To', 'to@example.com'),
+            ('Cc', 'cc@example.com')})
+        self.assertIn('\nDate: ', message.as_string())
 
     def test_idn_send(self):
         """
@@ -681,7 +751,14 @@ class ConsoleBackendTests(BaseEmailBackendTests, TestCase):
         s = StringIO()
         connection = mail.get_connection('django.core.mail.backends.console.EmailBackend', stream=s)
         send_mail('Subject', 'Content', 'from@example.com', ['to@example.com'], connection=connection)
-        self.assertTrue(s.getvalue().startswith('Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nSubject: Subject\nFrom: from@example.com\nTo: to@example.com\nDate: '))
+        self.assertMessageHasHeaders(s.getvalue(), {
+            ('MIME-Version', '1.0'),
+            ('Content-Type', 'text/plain; charset="utf-8"'),
+            ('Content-Transfer-Encoding', '7bit'),
+            ('Subject', 'Subject'),
+            ('From', 'from@example.com'),
+            ('To', 'to@example.com')})
+        self.assertIn('\nDate: ', s.getvalue())
 
 
 class FakeSMTPChannel(smtpd.SMTPChannel):