Fixed #20356 -- Prevented crash when HTTP_REFERER contains non-ascii

Thanks srusskih for the report and Aymeric Augustin for the review.

django/middleware/common.py

@@ -7,6 +7,7 @@ from django.conf import settings
 from django.core.mail import mail_managers
 from django.core import urlresolvers
 from django import http
+from django.utils.encoding import force_text
 from django.utils.http import urlquote
 from django.utils import six
 
@@ -140,7 +141,7 @@ class BrokenLinkEmailsMiddleware(object):
         if response.status_code == 404 and not settings.DEBUG:
             domain = request.get_host()
             path = request.get_full_path()
-            referer = request.META.get('HTTP_REFERER', '')
+            referer = force_text(request.META.get('HTTP_REFERER', ''), errors='replace')
             is_internal = self.is_internal_request(domain, referer)
             is_not_search_engine = '?' not in referer
             is_ignorable = self.is_ignorable_404(path)

tests/middleware/tests.py

@@ -22,7 +22,7 @@ from django.test.utils import override_settings
 from django.utils import six
 from django.utils.encoding import force_str
 from django.utils.six.moves import xrange
-from django.utils.unittest import expectedFailure
+from django.utils.unittest import expectedFailure, skipIf
 
 from transactions.tests import IgnorePendingDeprecationWarningsMixin
 
@@ -320,6 +320,14 @@ class BrokenLinkEmailsMiddlewareTest(TestCase):
         BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
         self.assertEqual(len(mail.outbox), 0)
 
+    @skipIf(six.PY3, "HTTP_REFERER is str type on Python 3")
+    def test_404_error_nonascii_referrer(self):
+        # Such referer strings should not happen, but anyway, if it happens,
+        # let's not crash
+        self.req.META['HTTP_REFERER'] = b'http://testserver/c/\xd0\xbb\xd0\xb8/'
+        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(len(mail.outbox), 1)
+
 
 class ConditionalGetMiddlewareTest(TestCase):
     urls = 'middleware.cond_get_urls'