File storage refactoring, adding far more flexibility to Django's file handling.…

File storage refactoring, adding far more flexibility to Django's file handling. The new files.txt document has details of the new features.

This is a backwards-incompatible change; consult BackwardsIncompatibleChanges for details.

Fixes #3567, #3621, #4345, #5361, #5655, #7415.

Many thanks to Marty Alchin who did the vast majority of this work.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@8244 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/conf/global_settings.py

@@ -226,6 +226,9 @@ SECRET_KEY = ''
 # Path to the "jing" executable -- needed to validate XMLFields
 JING_PATH = "/usr/bin/jing"
 
+# Default file storage mechanism that holds media.
+DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
+
 # Absolute path to the directory that holds media.
 # Example: "/home/media/media.lawrence.com/"
 MEDIA_ROOT = ''

django/contrib/admin/widgets.py

@@ -85,8 +85,8 @@ class AdminFileWidget(forms.FileInput):
     def render(self, name, value, attrs=None):
         output = []
         if value:
-            output.append('%s <a target="_blank" href="%s%s">%s</a> <br />%s ' % \
-                (_('Currently:'), settings.MEDIA_URL, value, value, _('Change:')))
+            output.append('%s <a target="_blank" href="%s">%s</a> <br />%s ' % \
+                (_('Currently:'), value.url, value, _('Change:')))
         output.append(super(AdminFileWidget, self).render(name, value, attrs))
         return mark_safe(u''.join(output))
 

django/core/files/__init__.py

+from django.core.files.base import File

django/core/files/base.py

+import os
+
+from django.utils.encoding import smart_str, smart_unicode
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+class File(object):
+    DEFAULT_CHUNK_SIZE = 64 * 2**10
+
+    def __init__(self, file):
+        self.file = file
+        self._name = file.name
+        self._mode = file.mode
+        self._closed = False
+
+    def __str__(self):
+        return smart_str(self.name or '')
+
+    def __unicode__(self):
+        return smart_unicode(self.name or u'')
+
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, self or "None")
+
+    def __nonzero__(self):
+        return not not self.name
+
+    def __len__(self):
+        return self.size
+
+    def _get_name(self):
+        return self._name
+    name = property(_get_name)
+
+    def _get_mode(self):
+        return self._mode
+    mode = property(_get_mode)
+
+    def _get_closed(self):
+        return self._closed
+    closed = property(_get_closed)
+
+    def _get_size(self):
+        if not hasattr(self, '_size'):
+            if hasattr(self.file, 'size'):
+                self._size = self.file.size
+            elif os.path.exists(self.file.name):
+                self._size = os.path.getsize(self.file.name)
+            else:
+                raise AttributeError("Unable to determine the file's size.")
+        return self._size
+
+    def _set_size(self, size):
+        self._size = size
+
+    size = property(_get_size, _set_size)
+
+    def chunks(self, chunk_size=None):
+        """
+        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
+        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
+        """
+        if not chunk_size:
+            chunk_size = self.__class__.DEFAULT_CHUNK_SIZE
+
+        if hasattr(self, 'seek'):
+            self.seek(0)
+        # Assume the pointer is at zero...
+        counter = self.size
+
+        while counter > 0:
+            yield self.read(chunk_size)
+            counter -= chunk_size
+
+    def multiple_chunks(self, chunk_size=None):
+        """
+        Returns ``True`` if you can expect multiple chunks.
+
+        NB: If a particular file representation is in memory, subclasses should
+        always return ``False`` -- there's no good reason to read from memory in
+        chunks.
+        """
+        if not chunk_size:
+            chunk_size = self.DEFAULT_CHUNK_SIZE
+        return self.size > chunk_size
+
+    def xreadlines(self):
+        return iter(self)
+
+    def readlines(self):
+        return list(self.xreadlines())
+
+    def __iter__(self):
+        # Iterate over this file-like object by newlines
+        buffer_ = None
+        for chunk in self.chunks():
+            chunk_buffer = StringIO(chunk)
+
+            for line in chunk_buffer:
+                if buffer_:
+                    line = buffer_ + line
+                    buffer_ = None
+
+                # If this is the end of a line, yield
+                # otherwise, wait for the next round
+                if line[-1] in ('\n', '\r'):
+                    yield line
+                else:
+                    buffer_ = line
+
+        if buffer_ is not None:
+            yield buffer_
+
+    def open(self, mode=None):
+        if not self.closed:
+            self.seek(0)
+        elif os.path.exists(self.file.name):
+            self.file = open(self.file.name, mode or self.file.mode)
+        else:
+            raise ValueError("The file cannot be reopened.")
+
+    def seek(self, position):
+        self.file.seek(position)
+
+    def tell(self):
+        return self.file.tell()
+
+    def read(self, num_bytes=None):
+        if num_bytes is None:
+            return self.file.read()
+        return self.file.read(num_bytes)
+
+    def write(self, content):
+        if not self.mode.startswith('w'):
+            raise IOError("File was not opened with write access.")
+        self.file.write(content)
+
+    def flush(self):
+        if not self.mode.startswith('w'):
+            raise IOError("File was not opened with write access.")
+        self.file.flush()
+
+    def close(self):
+        self.file.close()
+        self._closed = True
+
+class ContentFile(File):
+    """
+    A File-like object that takes just raw content, rather than an actual file.
+    """
+    def __init__(self, content):
+        self.file = StringIO(content or '')
+        self.size = len(content or '')
+        self.file.seek(0)
+        self._closed = False
+
+    def __str__(self):
+        return 'Raw content'
+
+    def __nonzero__(self):
+        return True
+
+    def open(self, mode=None):
+        if self._closed:
+            self._closed = False
+        self.seek(0)

django/core/files/images.py

+"""
+Utility functions for handling images.
+
+Requires PIL, as you might imagine.
+"""
+
+from PIL import ImageFile as PIL
+from django.core.files import File
+
+class ImageFile(File):
+    """
+    A mixin for use alongside django.core.files.base.File, which provides
+    additional features for dealing with images.
+    """
+    def _get_width(self):
+        return self._get_image_dimensions()[0]
+    width = property(_get_width)
+
+    def _get_height(self):
+        return self._get_image_dimensions()[1]
+    height = property(_get_height)
+
+    def _get_image_dimensions(self):
+        if not hasattr(self, '_dimensions_cache'):
+            self._dimensions_cache = get_image_dimensions(self)
+        return self._dimensions_cache
+
+def get_image_dimensions(file_or_path):
+    """Returns the (width, height) of an image, given an open file or a path."""
+    p = PIL.Parser()
+    if hasattr(file_or_path, 'read'):
+        file = file_or_path
+    else:
+        file = open(file_or_path, 'rb')
+    while 1:
+        data = file.read(1024)
+        if not data:
+            break
+        p.feed(data)
+        if p.image:
+            return p.image.size
+    return None

django/core/files/storage.py

+import os
+import urlparse
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
+from django.utils.encoding import force_unicode, smart_str
+from django.utils.text import force_unicode, get_valid_filename
+from django.utils._os import safe_join
+from django.core.files import locks, File
+
+__all__ = ('Storage', 'FileSystemStorage', 'DefaultStorage', 'default_storage')
+
+class Storage(object):
+    """
+    A base storage class, providing some default behaviors that all other
+    storage systems can inherit or override, as necessary.
+    """
+
+    # The following methods represent a public interface to private methods.
+    # These shouldn't be overridden by subclasses unless absolutely necessary.
+
+    def open(self, name, mode='rb', mixin=None):
+        """
+        Retrieves the specified file from storage, using the optional mixin
+        class to customize what features are available on the File returned.
+        """
+        file = self._open(name, mode)
+        if mixin:
+            # Add the mixin as a parent class of the File returned from storage.
+            file.__class__ = type(mixin.__name__, (mixin, file.__class__), {})
+        return file
+
+    def save(self, name, content):
+        """
+        Saves new content to the file specified by name. The content should be a
+        proper File object, ready to be read from the beginning.
+        """
+        # Check for old-style usage. Warn here first since there are multiple
+        # locations where we need to support both new and old usage.
+        if isinstance(content, basestring):
+            import warnings
+            warnings.warn(
+                message = "Representing files as strings is deprecated." \
+                          "Use django.core.files.base.ContentFile instead.",
+                category = DeprecationWarning,
+                stacklevel = 2
+            )
+            from django.core.files.base import ContentFile
+            content = ContentFile(content)
+
+        # Get the proper name for the file, as it will actually be saved.
+        if name is None:
+            name = content.name
+        name = self.get_available_name(name)
+
+        self._save(name, content)
+
+        # Store filenames with forward slashes, even on Windows
+        return force_unicode(name.replace('\\', '/'))
+
+    # These methods are part of the public API, with default implementations.
+
+    def get_valid_name(self, name):
+        """
+        Returns a filename, based on the provided filename, that's suitable for
+        use in the target storage system.
+        """
+        return get_valid_filename(name)
+
+    def get_available_name(self, name):
+        """
+        Returns a filename that's free on the target storage system, and
+        available for new content to be written to.
+        """
+        # If the filename already exists, keep adding an underscore to the name
+        # of the file until the filename doesn't exist.
+        while self.exists(name):
+            try:
+                dot_index = name.rindex('.')
+            except ValueError: # filename has no dot
+                name += '_'
+            else:
+                name = name[:dot_index] + '_' + name[dot_index:]
+        return name
+
+    def path(self, name):
+        """
+        Returns a local filesystem path where the file can be retrieved using
+        Python's built-in open() function. Storage systems that can't be
+        accessed using open() should *not* implement this method.
+        """
+        raise NotImplementedError("This backend doesn't support absolute paths.")
+
+    # The following methods form the public API for storage systems, but with
+    # no default implementations. Subclasses must implement *all* of these.
+
+    def delete(self, name):
+        """
+        Deletes the specified file from the storage system.
+        """
+        raise NotImplementedError()
+
+    def exists(self, name):
+        """
+        Returns True if a file referened by the given name already exists in the
+        storage system, or False if the name is available for a new file.
+        """
+        raise NotImplementedError()
+
+    def listdir(self, path):
+        """
+        Lists the contents of the specified path, returning a 2-tuple of lists;
+        the first item being directories, the second item being files.
+        """
+        raise NotImplementedError()
+
+    def size(self, name):
+        """
+        Returns the total size, in bytes, of the file specified by name.
+        """
+        raise NotImplementedError()
+
+    def url(self, name):
+        """
+        Returns an absolute URL where the file's contents can be accessed
+        directly by a web browser.
+        """
+        raise NotImplementedError()
+
+class FileSystemStorage(Storage):
+    """
+    Standard filesystem storage
+    """
+
+    def __init__(self, location=settings.MEDIA_ROOT, base_url=settings.MEDIA_URL):
+        self.location = os.path.abspath(location)
+        self.base_url = base_url
+
+    def _open(self, name, mode='rb'):
+        return File(open(self.path(name), mode))
+
+    def _save(self, name, content):
+        full_path = self.path(name)
+
+        directory = os.path.dirname(full_path)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+        elif not os.path.isdir(directory):
+            raise IOError("%s exists and is not a directory." % directory)
+            
+        if hasattr(content, 'temporary_file_path'):
+            # This file has a file path that we can move.
+            file_move_safe(content.temporary_file_path(), full_path)
+            content.close()
+        else:
+            # This is a normal uploadedfile that we can stream.
+            fp = open(full_path, 'wb')
+            locks.lock(fp, locks.LOCK_EX)
+            for chunk in content.chunks():
+                fp.write(chunk)
+            locks.unlock(fp)
+            fp.close()
+
+    def delete(self, name):
+        name = self.path(name)
+        # If the file exists, delete it from the filesystem.
+        if os.path.exists(name):
+            os.remove(name)
+
+    def exists(self, name):
+        return os.path.exists(self.path(name))
+
+    def listdir(self, path):
+        path = self.path(path)
+        directories, files = [], []
+        for entry in os.listdir(path):
+            if os.path.isdir(os.path.join(path, entry)):
+                directories.append(entry)
+            else:
+                files.append(entry)
+        return directories, files
+
+    def path(self, name):
+        try:
+            path = safe_join(self.location, name)
+        except ValueError:
+            raise SuspiciousOperation("Attempted access to '%s' denied." % name)
+        return os.path.normpath(path)
+
+    def size(self, name):
+        return os.path.getsize(self.path(name))
+
+    def url(self, name):
+        if self.base_url is None:
+            raise ValueError("This file is not accessible via a URL.")
+        return urlparse.urljoin(self.base_url, name).replace('\\', '/')
+
+def get_storage_class(import_path):
+    try:
+        dot = import_path.rindex('.')
+    except ValueError:
+        raise ImproperlyConfigured("%s isn't a storage module." % import_path)
+    module, classname = import_path[:dot], import_path[dot+1:]
+    try:
+        mod = __import__(module, {}, {}, [''])
+    except ImportError, e:
+        raise ImproperlyConfigured('Error importing storage module %s: "%s"' % (module, e))
+    try:
+        return getattr(mod, classname)
+    except AttributeError:
+        raise ImproperlyConfigured('Storage module "%s" does not define a "%s" class.' % (module, classname))
+
+DefaultStorage = get_storage_class(settings.DEFAULT_FILE_STORAGE)
+default_storage = DefaultStorage()

django/core/files/uploadedfile.py

@@ -10,6 +10,7 @@ except ImportError:
     from StringIO import StringIO
 
 from django.conf import settings
+from django.core.files.base import File
 
 from django.core.files import temp as tempfile
 
@@ -39,7 +40,7 @@ def deprecated_property(old, new, readonly=False):
     else:
         return property(getter, setter)
 
-class UploadedFile(object):
+class UploadedFile(File):
     """
     A abstract uploaded file (``TemporaryUploadedFile`` and
     ``InMemoryUploadedFile`` are the built-in concrete subclasses).
@@ -76,23 +77,6 @@ class UploadedFile(object):
 
     name = property(_get_name, _set_name)
 
-    def chunks(self, chunk_size=None):
-        """
-        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
-        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
-        """
-        if not chunk_size:
-            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
-
-        if hasattr(self, 'seek'):
-            self.seek(0)
-        # Assume the pointer is at zero...
-        counter = self.size
-
-        while counter > 0:
-            yield self.read(chunk_size)
-            counter -= chunk_size
-
     # Deprecated properties
     filename = deprecated_property(old="filename", new="name")
     file_name = deprecated_property(old="file_name", new="name")
@@ -108,18 +92,6 @@ class UploadedFile(object):
         return self.read()
     data = property(_get_data)
 
-    def multiple_chunks(self, chunk_size=None):
-        """
-        Returns ``True`` if you can expect multiple chunks.
-
-        NB: If a particular file representation is in memory, subclasses should
-        always return ``False`` -- there's no good reason to read from memory in
-        chunks.
-        """
-        if not chunk_size:
-            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
-        return self.size > chunk_size
-
     # Abstract methods; subclasses *must* define read() and probably should
     # define open/close.
     def read(self, num_bytes=None):
@@ -131,33 +103,6 @@ class UploadedFile(object):
     def close(self):
         pass
 
-    def xreadlines(self):
-        return self
-
-    def readlines(self):
-        return list(self.xreadlines())
-
-    def __iter__(self):
-        # Iterate over this file-like object by newlines
-        buffer_ = None
-        for chunk in self.chunks():
-            chunk_buffer = StringIO(chunk)
-
-            for line in chunk_buffer:
-                if buffer_:
-                    line = buffer_ + line
-                    buffer_ = None
-
-                # If this is the end of a line, yield
-                # otherwise, wait for the next round
-                if line[-1] in ('\n', '\r'):
-                    yield line
-                else:
-                    buffer_ = line
-
-        if buffer_ is not None:
-            yield buffer_
-
     # Backwards-compatible support for uploaded-files-as-dictionaries.
     def __getitem__(self, key):
         warnings.warn(

django/db/models/__init__.py

@@ -8,6 +8,7 @@ from django.db.models.manager import Manager
 from django.db.models.base import Model
 from django.db.models.fields import *
 from django.db.models.fields.subclassing import SubfieldBase
+from django.db.models.fields.files import FileField, ImageField
 from django.db.models.fields.related import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel, TABULAR, STACKED
 from django.db.models import signals
 

django/db/models/base.py

@@ -3,6 +3,7 @@ import types
 import sys
 import os
 from itertools import izip
+from warnings import warn
 try:
     set
 except NameError:
@@ -12,7 +13,7 @@ import django.db.models.manipulators    # Imported to register signal handler.
 import django.db.models.manager         # Ditto.
 from django.core import validators
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned, FieldError
-from django.db.models.fields import AutoField, ImageField
+from django.db.models.fields import AutoField
 from django.db.models.fields.related import OneToOneRel, ManyToOneRel, OneToOneField
 from django.db.models.query import delete_objects, Q, CollectedObjects
 from django.db.models.options import Options
@@ -463,110 +464,42 @@ class Model(object):
         return getattr(self, cachename)
 
     def _get_FIELD_filename(self, field):
-        if getattr(self, field.attname): # Value is not blank.
-            return os.path.normpath(os.path.join(settings.MEDIA_ROOT, getattr(self, field.attname)))
-        return ''
+        warn("instance.get_%s_filename() is deprecated. Use instance.%s.path instead." % \
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
+        try:
+            return getattr(self, field.attname).path
+        except ValueError:
+            return ''
 
     def _get_FIELD_url(self, field):
-        if getattr(self, field.attname): # Value is not blank.
-            import urlparse
-            return urlparse.urljoin(settings.MEDIA_URL, getattr(self, field.attname)).replace('\\', '/')
-        return ''
+        warn("instance.get_%s_url() is deprecated. Use instance.%s.url instead." % \
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
+        try:
+            return getattr(self, field.attname).url
+        except ValueError:
+            return ''
 
     def _get_FIELD_size(self, field):
-        return os.path.getsize(self._get_FIELD_filename(field))
-
-    def _save_FIELD_file(self, field, filename, raw_field, save=True):
-        # Create the upload directory if it doesn't already exist
-        directory = os.path.join(settings.MEDIA_ROOT, field.get_directory_name())
-        if not os.path.exists(directory):
-            os.makedirs(directory)
-        elif not os.path.isdir(directory):
-            raise IOError('%s exists and is not a directory' % directory)        
-
-        # Check for old-style usage (files-as-dictionaries). Warn here first
-        # since there are multiple locations where we need to support both new
-        # and old usage.
-        if isinstance(raw_field, dict):
-            import warnings
-            warnings.warn(
-                message = "Representing uploaded files as dictionaries is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
-                category = DeprecationWarning,
-                stacklevel = 2
-            )
-            from django.core.files.uploadedfile import SimpleUploadedFile
-            raw_field = SimpleUploadedFile.from_dict(raw_field)
-
-        elif isinstance(raw_field, basestring):
-            import warnings
-            warnings.warn(
-                message = "Representing uploaded files as strings is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
-                category = DeprecationWarning,
-                stacklevel = 2
-            )
-            from django.core.files.uploadedfile import SimpleUploadedFile
-            raw_field = SimpleUploadedFile(filename, raw_field)
-
-        if filename is None:
-            filename = raw_field.file_name
-
-        filename = field.get_filename(filename)
-
-        # If the filename already exists, keep adding an underscore to the name
-        # of the file until the filename doesn't exist.
-        while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
-            try:
-                dot_index = filename.rindex('.')
-            except ValueError: # filename has no dot.
-                filename += '_'
-            else:
-                filename = filename[:dot_index] + '_' + filename[dot_index:]
-
-        # Save the file name on the object and write the file to disk.
-        setattr(self, field.attname, filename)
-        full_filename = self._get_FIELD_filename(field)
-        if hasattr(raw_field, 'temporary_file_path'):
-            # This file has a file path that we can move.
-            file_move_safe(raw_field.temporary_file_path(), full_filename)
-            raw_field.close()
-        else:
-            # This is a normal uploadedfile that we can stream.
-            fp = open(full_filename, 'wb')
-            locks.lock(fp, locks.LOCK_EX)
-            for chunk in raw_field.chunks():
-                fp.write(chunk)
-            locks.unlock(fp)
-            fp.close()
-
-        # Save the width and/or height, if applicable.
-        if isinstance(field, ImageField) and \
-                (field.width_field or field.height_field):
-            from django.utils.images import get_image_dimensions
-            width, height = get_image_dimensions(full_filename)
-            if field.width_field:
-                setattr(self, field.width_field, width)
-            if field.height_field:
-                setattr(self, field.height_field, height)
-
-        # Save the object because it has changed, unless save is False.
-        if save:
-            self.save()
+        warn("instance.get_%s_size() is deprecated. Use instance.%s.size instead." % \
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
+        return getattr(self, field.attname).size
+
+    def _save_FIELD_file(self, field, filename, content, save=True):
+        warn("instance.save_%s_file() is deprecated. Use instance.%s.save() instead." % \
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
+        return getattr(self, field.attname).save(filename, content, save)
 
     _save_FIELD_file.alters_data = True
 
     def _get_FIELD_width(self, field):
-        return self._get_image_dimensions(field)[0]
+        warn("instance.get_%s_width() is deprecated. Use instance.%s.width instead." % \
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
+        return getattr(self, field.attname).width()
 
     def _get_FIELD_height(self, field):
-        return self._get_image_dimensions(field)[1]
-
-    def _get_image_dimensions(self, field):
-        cachename = "__%s_dimensions_cache" % field.name
-        if not hasattr(self, cachename):
-            from django.utils.images import get_image_dimensions
-            filename = self._get_FIELD_filename(field)
-            setattr(self, cachename, get_image_dimensions(filename))
-        return getattr(self, cachename)
+        warn("instance.get_%s_height() is deprecated. Use instance.%s.height instead." % \
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
+        return getattr(self, field.attname).height()
 
 
 ############################################

django/db/models/fields/__init__.py

@@ -10,6 +10,7 @@ except ImportError:
 from django.db import connection, get_creation_module
 from django.db.models import signals
 from django.db.models.query_utils import QueryWrapper
+from django.dispatch import dispatcher
 from django.conf import settings
 from django.core import validators
 from django import oldforms
@@ -757,131 +758,6 @@ class EmailField(CharField):
         defaults.update(kwargs)
         return super(EmailField, self).formfield(**defaults)
 
-class FileField(Field):
-    def __init__(self, verbose_name=None, name=None, upload_to='', **kwargs):
-        self.upload_to = upload_to
-        kwargs['max_length'] = kwargs.get('max_length', 100)
-        Field.__init__(self, verbose_name, name, **kwargs)
-
-    def get_internal_type(self):
-        return "FileField"
-
-    def get_db_prep_value(self, value):
-        "Returns field's value prepared for saving into a database."
-        # Need to convert UploadedFile objects provided via a form to unicode for database insertion
-        if hasattr(value, 'name'):
-            return value.name
-        elif value is None:
-            return None
-        else:
-            return unicode(value)
-
-    def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True):
-        field_list = Field.get_manipulator_fields(self, opts, manipulator, change, name_prefix, rel, follow)
-        if not self.blank:
-            if rel:
-                # This validator makes sure FileFields work in a related context.
-                class RequiredFileField(object):
-                    def __init__(self, other_field_names, other_file_field_name):
-                        self.other_field_names = other_field_names
-                        self.other_file_field_name = other_file_field_name
-                        self.always_test = True
-                    def __call__(self, field_data, all_data):
-                        if not all_data.get(self.other_file_field_name, False):
-                            c = validators.RequiredIfOtherFieldsGiven(self.other_field_names, ugettext_lazy("This field is required."))
-                            c(field_data, all_data)
-                # First, get the core fields, if any.
-                core_field_names = []
-                for f in opts.fields:
-                    if f.core and f != self:
-                        core_field_names.extend(f.get_manipulator_field_names(name_prefix))
-                # Now, if there are any, add the validator to this FormField.
-                if core_field_names:
-                    field_list[0].validator_list.append(RequiredFileField(core_field_names, field_list[1].field_name))
-            else:
-                v = validators.RequiredIfOtherFieldNotGiven(field_list[1].field_name, ugettext_lazy("This field is required."))
-                v.always_test = True
-                field_list[0].validator_list.append(v)
-                field_list[0].is_required = field_list[1].is_required = False
-
-        # If the raw path is passed in, validate it's under the MEDIA_ROOT.
-        def isWithinMediaRoot(field_data, all_data):
-            f = os.path.abspath(os.path.join(settings.MEDIA_ROOT, field_data))
-            if not f.startswith(os.path.abspath(os.path.normpath(settings.MEDIA_ROOT))):
-                raise validators.ValidationError, _("Enter a valid filename.")
-        field_list[1].validator_list.append(isWithinMediaRoot)
-        return field_list
-
-    def contribute_to_class(self, cls, name):
-        super(FileField, self).contribute_to_class(cls, name)
-        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
-        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
-        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
-        signals.post_delete.connect(self.delete_file, sender=cls)
-
-    def delete_file(self, instance, **kwargs):
-        if getattr(instance, self.attname):
-            file_name = getattr(instance, 'get_%s_filename' % self.name)()
-            # If the file exists and no other object of this type references it,
-            # delete it from the filesystem.
-            if os.path.exists(file_name) and \
-                not instance.__class__._default_manager.filter(**{'%s__exact' % self.name: getattr(instance, self.attname)}):
-                os.remove(file_name)
-
-    def get_manipulator_field_objs(self):
-        return [oldforms.FileUploadField, oldforms.HiddenField]
-
-    def get_manipulator_field_names(self, name_prefix):
-        return [name_prefix + self.name + '_file', name_prefix + self.name]
-
-    def save_file(self, new_data, new_object, original_object, change, rel, save=True):
-        upload_field_name = self.get_manipulator_field_names('')[0]
-        if new_data.get(upload_field_name, False):
-            if rel:
-                file = new_data[upload_field_name][0]
-            else:
-                file = new_data[upload_field_name]
-
-            if not file:
-                return
-
-            # Backwards-compatible support for files-as-dictionaries.
-            # We don't need to raise a warning because Model._save_FIELD_file will
-            # do so for us.
-            try:
-                file_name = file.name
-            except AttributeError:
-                file_name = file['filename']
-
-            func = getattr(new_object, 'save_%s_file' % self.name)
-            func(file_name, file, save)
-
-    def get_directory_name(self):
-        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
-
-    def get_filename(self, filename):
-        from django.utils.text import get_valid_filename
-        f = os.path.join(self.get_directory_name(), get_valid_filename(os.path.basename(filename)))
-        return os.path.normpath(f)
-
-    def save_form_data(self, instance, data):
-        from django.core.files.uploadedfile import UploadedFile
-        if data and isinstance(data, UploadedFile):
-            getattr(instance, "save_%s_file" % self.name)(data.name, data, save=False)
-
-    def formfield(self, **kwargs):
-        defaults = {'form_class': forms.FileField}
-        # If a file has been provided previously, then the form doesn't require
-        # that a new file is provided this time.
-        # The code to mark the form field as not required is used by
-        # form_for_instance, but can probably be removed once form_for_instance
-        # is gone. ModelForm uses a different method to check for an existing file.
-        if 'initial' in kwargs:
-            defaults['required'] = False
-        defaults.update(kwargs)
-        return super(FileField, self).formfield(**defaults)
-
 class FilePathField(Field):
     def __init__(self, verbose_name=None, name=None, path='', match=None, recursive=False, **kwargs):
         self.path, self.match, self.recursive = path, match, recursive
@@ -923,40 +799,6 @@ class FloatField(Field):
         defaults.update(kwargs)
         return super(FloatField, self).formfield(**defaults)
 
-class ImageField(FileField):
-    def __init__(self, verbose_name=None, name=None, width_field=None, height_field=None, **kwargs):
-        self.width_field, self.height_field = width_field, height_field
-        FileField.__init__(self, verbose_name, name, **kwargs)
-
-    def get_manipulator_field_objs(self):
-        return [oldforms.ImageUploadField, oldforms.HiddenField]
-
-    def contribute_to_class(self, cls, name):
-        super(ImageField, self).contribute_to_class(cls, name)
-        # Add get_BLAH_width and get_BLAH_height methods, but only if the
-        # image field doesn't have width and height cache fields.
-        if not self.width_field:
-            setattr(cls, 'get_%s_width' % self.name, curry(cls._get_FIELD_width, field=self))
-        if not self.height_field:
-            setattr(cls, 'get_%s_height' % self.name, curry(cls._get_FIELD_height, field=self))
-
-    def save_file(self, new_data, new_object, original_object, change, rel, save=True):
-        FileField.save_file(self, new_data, new_object, original_object, change, rel, save)
-        # If the image has height and/or width field(s) and they haven't
-        # changed, set the width and/or height field(s) back to their original
-        # values.
-        if change and (self.width_field or self.height_field) and save:
-            if self.width_field:
-                setattr(new_object, self.width_field, getattr(original_object, self.width_field))
-            if self.height_field:
-                setattr(new_object, self.height_field, getattr(original_object, self.height_field))
-            new_object.save()
-
-    def formfield(self, **kwargs):
-        defaults = {'form_class': forms.ImageField}
-        defaults.update(kwargs)
-        return super(ImageField, self).formfield(**defaults)
-
 class IntegerField(Field):
     empty_strings_allowed = False
     def get_db_prep_value(self, value):

django/db/models/manipulators.py

 from django.core.exceptions import ObjectDoesNotExist
 from django import oldforms
 from django.core import validators
-from django.db.models.fields import FileField, AutoField
+from django.db.models.fields import AutoField
+from django.db.models.fields.files import FileField
 from django.db.models import signals
 from django.utils.functional import curry
 from django.utils.datastructures import DotExpandedDict

django/utils/images.py

-"""
-Utility functions for handling images.
+import warnings
 
-Requires PIL, as you might imagine.
-"""
+from django.core.files.images import get_image_dimensions
 
-import ImageFile
-
-def get_image_dimensions(path):
-    """Returns the (width, height) of an image at a given path."""
-    p = ImageFile.Parser()
-    fp = open(path, 'rb')
-    while 1:
-        data = fp.read(1024)
-        if not data:
-            break
-        p.feed(data)
-        if p.image:
-            return p.image.size
-            break
-    fp.close()
-    return None
+warnings.warn("django.utils.images has been moved to django.core.files.images.", DeprecationWarning)

docs/custom_model_fields.txt

@@ -596,3 +596,42 @@ smoothly:
        instance, not a ``HandField``). So if your ``__unicode__()`` method
        automatically converts to the string form of your Python object, you can
        save yourself a lot of work.
+
+Writing a ``FileField`` subclass
+=================================
+
+In addition to the above methods, fields that deal with files have a few other
+special requirements which must be taken into account. The majority of the
+mechanics provided by ``FileField``, such as controlling database storage and
+retrieval, can remain unchanged, leaving subclasses to deal with the challenge
+of supporting a particular type of file.
+
+Django provides a ``File`` class, which is used as a proxy to the file's
+contents and operations. This can be subclassed to customzie hwo the file is
+accessed, and what methods are available. It lives at
+``django.db.models.fields.files``, and its default behavior is explained in the
+`file documentation`_.
+
+Once a subclass of ``File`` is created, the new ``FileField`` subclass must be
+told to use it. To do so, simply assign the new ``File`` subclass to the special
+``attr_class`` attribute of the ``FileField`` subclass.
+
+.. _file documentation: ../files/
+
+A few suggestions
+------------------
+
+In addition to the above details, there are a few guidelines which can greatly
+improve the efficiency and readability of the field's code.
+
+    1. The source for Django's own ``ImageField`` (in
+       ``django/db/models/fields/files.py``) is a great example of how to
+       subclass ``FileField`` to support a particular type of file, as it
+       incorporates all of the techniques described above.
+
+    2. Cache file attributes wherever possible. Since files may be stored in
+       remote storage systems, retrieving them may cost extra time, or even
+       money, that isn't always necessary. Once a file is retrieved to obtain
+       some data about its content, cache as much of that data as possible to
+       reduce the number of times the file must be retrieved on subsequent
+       calls for that information.

docs/db-api.txt

@@ -2298,53 +2298,34 @@ For a full example, see the `lookup API sample model`_.
 get_FOO_filename()
 ------------------
 
-For every ``FileField``, the object will have a ``get_FOO_filename()`` method,
-where ``FOO`` is the name of the field. This returns the full filesystem path
-to the file, according to your ``MEDIA_ROOT`` setting.
-
-.. note::
-    It is only valid to call this method **after** saving the model when the
-    field has been set. Prior to saving, the value returned will not contain
-    the upload directory (the `upload_to` parameter) in the path.
-
-Note that ``ImageField`` is technically a subclass of ``FileField``, so every
-model with an ``ImageField`` will also get this method.
+**Deprecated in Django development version**; use ``object.FOO.name`` instead.
+See `managing files`_ for details.
 
 get_FOO_url()
 -------------
 
-For every ``FileField``, the object will have a ``get_FOO_url()`` method,
-where ``FOO`` is the name of the field. This returns the full URL to the file,
-according to your ``MEDIA_URL`` setting. If the value is blank, this method
-returns an empty string.
-
-.. note::
-    As with ``get_FOO_filename()``, it is only valid to call this method
-    **after** saving the model, otherwise an incorrect result will be
-    returned.
+**Deprecated in Django development version**; use ``object.FOO.url`` instead.
+See `managing files`_ for details.
 
 get_FOO_size()
 --------------
 
-For every ``FileField``, the object will have a ``get_FOO_size()`` method,
-where ``FOO`` is the name of the field. This returns the size of the file, in
-bytes. (Behind the scenes, it uses ``os.path.getsize``.)
+**Deprecated in Django development version**; use ``object.FOO.size`` instead.
+See `managing files`_ for details.
 
 save_FOO_file(filename, raw_contents)
 -------------------------------------
 
-For every ``FileField``, the object will have a ``save_FOO_file()`` method,
-where ``FOO`` is the name of the field. This saves the given file to the
-filesystem, using the given filename. If a file with the given filename already
-exists, Django adds an underscore to the end of the filename (but before the
-extension) until the filename is available.
+**Deprecated in Django development version**; use ``object.FOO.save()`` instead.
+See `managing files`_ for details.
 
 get_FOO_height() and get_FOO_width()
 ------------------------------------
 
-For every ``ImageField``, the object will have ``get_FOO_height()`` and
-``get_FOO_width()`` methods, where ``FOO`` is the name of the field. This
-returns the height (or width) of the image, as an integer, in pixels.
+**Deprecated in Django development version**; use ``object.FOO.width`` and
+``object.FOO.height`` instead. See `managing files`_ for details.
+
+.. _`managing files`: ../files/
 
 Shortcuts
 =========

docs/model-api.txt

@@ -224,26 +224,64 @@ set to 75 by default, but you can specify it to override default behavior.
 ``FileField``
 ~~~~~~~~~~~~~
 
-A file-upload field. Has one **required** argument:
+A file-upload field. Has two special arguments, of which the first is
+**required**:
 
     ======================  ===================================================
     Argument                Description
     ======================  ===================================================
-    ``upload_to``           A local filesystem path that will be appended to
-                            your ``MEDIA_ROOT`` setting to determine the
-                            output of the ``get_<fieldname>_url()`` helper
-                            function.
+    ``upload_to``           Required. A filesystem-style path that will be
+                            prepended to the filename before being committed to
+                            the final storage destination.
+
+                            **New in Django development version**
+
+                            This may also be a callable, such as a function,
+                            which will be called to obtain the upload path,
+                            including the filename. See below for details.
+
+    ``storage``             **New in Django development version**
+
+                            Optional. A storage object, which handles the
+                            storage and retrieval of your files. See `managing
+                            files`_ for details on how to provide this object.
     ======================  ===================================================
 
-This path may contain `strftime formatting`_, which will be replaced by the
-date/time of the file upload (so that uploaded files don't fill up the given
-directory).
+.. _managing files: ../files/
+
+The ``upload_to`` path may contain `strftime formatting`_, which will be
+replaced by the date/time of the file upload (so that uploaded files don't fill
+up the given directory).
+
+**New in Django development version**
+
+If a callable is provided for the ``upload_to`` argument, that callable must be
+able to accept two arguments, and return a Unix-style path (with forward
+slashes) to be passed along to the storage system. The two arguments that will
+be passed are:
+
+    ======================  ===================================================
+    Argument                Description
+    ======================  ===================================================
+    ``instance``            An instance of the model where the ``FileField`` is
+                            defined. More specifically, this is the particular
+                            instance where the current file is being attached.
+
+                            **Note**: In most cases, this object will not have
+                            been saved to the database yet, so if it uses the
+                            default ``AutoField``, *it might not yet have a
+                            value for its primary key field*.
+
+    ``filename``            The filename that was originally given to the file.
+                            This may or may not be taken into account when
+                            determining the final destination path.
+    ======================  ===================================================
 
 The admin represents this field as an ``<input type="file">`` (a file-upload
 widget).
 
-Using a ``FileField`` or an ``ImageField`` (see below) in a model takes a few
-steps:
+Using a ``FileField`` or an ``ImageField`` (see below) in a model without a
+specified storage system takes a few steps:
 
     1. In your settings file, you'll need to define ``MEDIA_ROOT`` as the
        full path to a directory where you'd like Django to store uploaded

docs/settings.txt

@@ -426,6 +426,16 @@ Default content type to use for all ``HttpResponse`` objects, if a MIME type
 isn't manually specified. Used with ``DEFAULT_CHARSET`` to construct the
 ``Content-Type`` header.
 
+DEFAULT_FILE_STORAGE
+--------------------
+
+Default: ``'django.core.filestorage.filesystem.FileSystemStorage'``
+
+Default file storage class to be used for any file-related operations that don't
+specify a particular storage system. See the `file documentation`_ for details.
+
+.. _file documentation: ../files/
+
 DEFAULT_FROM_EMAIL
 ------------------
 

docs/upload_handling.txt

@@ -155,25 +155,8 @@ Three `settings`_ control Django's file upload behavior:
 ``UploadedFile`` objects
 ========================
 
-All ``UploadedFile`` objects define the following methods/attributes:
-
-    ``UploadedFile.read(self, num_bytes=None)``
-        Returns a byte string of length ``num_bytes``, or the complete file if
-        ``num_bytes`` is ``None``.
-
-    ``UploadedFile.chunks(self, chunk_size=None)``
-        A generator yielding small chunks from the file. If ``chunk_size`` isn't
-        given, chunks will be 64 KB.
-
-    ``UploadedFile.multiple_chunks(self, chunk_size=None)``
-        Returns ``True`` if you can expect more than one chunk when calling
-        ``UploadedFile.chunks(self, chunk_size)``.
-
-    ``UploadedFile.size``
-        The size, in bytes, of the uploaded file.
-
-    ``UploadedFile.name``
-        The name of the uploaded file as provided by the user.
+In addition to those inherited from `File`_, all ``UploadedFile`` objects define
+the following methods/attributes:
 
     ``UploadedFile.content_type``
         The content-type header uploaded with the file (e.g. ``text/plain`` or
@@ -186,13 +169,11 @@ All ``UploadedFile`` objects define the following methods/attributes:
         For ``text/*`` content-types, the character set (i.e. ``utf8``) supplied
         by the browser. Again, "trust but verify" is the best policy here.
 
-    ``UploadedFile.__iter__()``
-        Iterates over the lines in the file.
-
     ``UploadedFile.temporary_file_path()``
         Only files uploaded onto disk will have this method; it returns the full
         path to the temporary uploaded file.
 
+.. _File: ../files/
 
 Upload Handlers
 ===============

tests/modeltests/files/__init__.py

+

tests/modeltests/files/models.py

+"""
+42. Storing files according to a custom storage system
+
+FileField and its variations can take a "storage" argument to specify how and
+where files should be stored.
+"""
+
+import tempfile
+
+from django.db import models
+from django.core.files.base import ContentFile
+from django.core.files.storage import FileSystemStorage
+from django.core.cache import cache
+
+temp_storage = FileSystemStorage(location=tempfile.gettempdir())
+
+# Write out a file to be used as default content
+temp_storage.save('tests/default.txt', ContentFile('default content'))
+
+class Storage(models.Model):
+    def custom_upload_to(self, filename):
+        return 'foo'
+
+    def random_upload_to(self, filename):
+        # This returns a different result each time,
+        # to make sure it only gets called once.
+        import random
+        return '%s/%s' % (random.randint(100, 999), filename)
+
+    normal = models.FileField(storage=temp_storage, upload_to='tests')
+    custom = models.FileField(storage=temp_storage, upload_to=custom_upload_to)
+    random = models.FileField(storage=temp_storage, upload_to=random_upload_to)
+    default = models.FileField(storage=temp_storage, upload_to='tests', default='tests/default.txt')
+
+__test__ = {'API_TESTS':"""
+# An object without a file has limited functionality.
+
+>>> obj1 = Storage()
+>>> obj1.normal
+<FieldFile: None>
+>>> obj1.normal.size
+Traceback (most recent call last):
+...
+ValueError: The 'normal' attribute has no file associated with it.
+
+# Saving a file enables full functionality.
+
+>>> obj1.normal.save('django_test.txt', ContentFile('content'))
+>>> obj1.normal
+<FieldFile: tests/django_test.txt>
+>>> obj1.normal.size
+7
+>>> obj1.normal.read()
+'content'
+
+# Files can be read in a little at a time, if necessary.
+
+>>> obj1.normal.open()
+>>> obj1.normal.read(3)
+'con'
+>>> obj1.normal.read()
+'tent'
+>>> '-'.join(obj1.normal.chunks(chunk_size=2))
+'co-nt-en-t'
+
+# Save another file with the same name.
+
+>>> obj2 = Storage()
+>>> obj2.normal.save('django_test.txt', ContentFile('more content'))
+>>> obj2.normal
+<FieldFile: tests/django_test_.txt>
+>>> obj2.normal.size
+12
+
+# Push the objects into the cache to make sure they pickle properly
+
+>>> cache.set('obj1', obj1)
+>>> cache.set('obj2', obj2)
+>>> cache.get('obj2').normal
+<FieldFile: tests/django_test_.txt>
+
+# Deleting an object deletes the file it uses, if there are no other objects
+# still using that file.
+
+>>> obj2.delete()
+>>> obj2.normal.save('django_test.txt', ContentFile('more content'))
+>>> obj2.normal
+<FieldFile: tests/django_test_.txt>
+
+# Default values allow an object to access a single file.
+
+>>> obj3 = Storage.objects.create()
+>>> obj3.default
+<FieldFile: tests/default.txt>
+>>> obj3.default.read()
+'default content'
+
+# But it shouldn't be deleted, even if there are no more objects using it.
+
+>>> obj3.delete()
+>>> obj3 = Storage()
+>>> obj3.default.read()
+'default content'
+
+# Verify the fix for #5655, making sure the directory is only determined once.
+
+>>> obj4 = Storage()
+>>> obj4.random.save('random_file', ContentFile('random content'))
+>>> obj4.random
+<FieldFile: .../random_file>
+
+# Clean up the temporary files.
+
+>>> obj1.normal.delete()
+>>> obj2.normal.delete()
+>>> obj3.default.delete()
+>>> obj4.random.delete()
+"""}

tests/modeltests/model_forms/models.py

@@ -11,6 +11,9 @@ import os
 import tempfile
 
 from django.db import models
+from django.core.files.storage import FileSystemStorage
+
+temp_storage = FileSystemStorage(tempfile.gettempdir())
 
 ARTICLE_STATUS = (
     (1, 'Draft'),
@@ -60,7 +63,7 @@ class PhoneNumber(models.Model):
 
 class TextFile(models.Model):
     description = models.CharField(max_length=20)
-    file = models.FileField(upload_to=tempfile.gettempdir())
+    file = models.FileField(storage=temp_storage, upload_to='tests')
 
     def __unicode__(self):
         return self.description
@@ -73,9 +76,9 @@ class ImageFile(models.Model):
         # for PyPy, you need to check for the underlying modules
         # If PIL is not available, this test is equivalent to TextFile above.
         import Image, _imaging
-        image = models.ImageField(upload_to=tempfile.gettempdir())
+        image = models.ImageField(storage=temp_storage, upload_to='tests')
     except ImportError:
-        image = models.FileField(upload_to=tempfile.gettempdir())
+        image = models.FileField(storage=temp_storage, upload_to='tests')
 
     def __unicode__(self):
         return self.description
@@ -786,6 +789,8 @@ u'Assistance'
 
 # FileField ###################################################################
 
+# File forms.
+
 >>> class TextFileForm(ModelForm):
 ...     class Meta:
 ...         model = TextFile
@@ -808,9 +813,9 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.file
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
 >>> f.is_valid()
@@ -819,7 +824,7 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.file
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 
 # Edit an instance that already has the file defined in the model. This will not
 # save the file again, but leave it exactly as it is.
@@ -828,13 +833,13 @@ u'...test1.txt'
 >>> f.is_valid()
 True
 >>> f.cleaned_data['file']
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 >>> instance = f.save()
 >>> instance.file
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 # Override the file by uploading a new one.
 
@@ -843,20 +848,20 @@ u'...test1.txt'
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test2.txt'
+<FieldFile: tests/test2.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')})
 >>> f.is_valid()
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test2.txt'
+<FieldFile: tests/test2.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 >>> instance.delete()
 
@@ -868,17 +873,17 @@ u'...test2.txt'
 True
 >>> instance = f.save()
 >>> instance.file
-''
+<FieldFile: None>
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
 >>> f.is_valid()
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test3.txt'
+<FieldFile: tests/test3.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 >>> instance.delete()
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')})
@@ -886,10 +891,10 @@ u'...test3.txt'
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test3.txt'
+<FieldFile: tests/test3.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 >>> instance.delete()
 
 # ImageField ###################################################################
@@ -911,10 +916,10 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.image
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 
 >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
 >>> f.is_valid()
@@ -923,7 +928,7 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.image
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 
 # Edit an instance that already has the image defined in the model. This will not
 # save the image again, but leave it exactly as it is.
@@ -932,14 +937,14 @@ u'...test.png'
 >>> f.is_valid()
 True
 >>> f.cleaned_data['image']
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 >>> instance = f.save()
 >>> instance.image
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 
 # Delete the current image since this is not done by Django.
 
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 
 # Override the file by uploading a new one.
 
@@ -948,10 +953,10 @@ u'...test.png'
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test2.png'
+<ImageFieldFile: tests/test2.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 >>> instance.delete()
 
 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)})
@@ -959,10 +964,10 @@ u'...test2.png'
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test2.png'
+<ImageFieldFile: tests/test2.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 >>> instance.delete()
 
 # Test the non-required ImageField
@@ -973,17 +978,17 @@ u'...test2.png'
 True
 >>> instance = f.save()
 >>> instance.image
-''
+<ImageFieldFile: None>
 
 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
 >>> f.is_valid()
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test3.png'
+<ImageFieldFile: tests/test3.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 >>> instance.delete()
 
 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)})
@@ -991,7 +996,7 @@ u'...test3.png'
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test3.png'
+<ImageFieldFile: tests/test3.png>
 >>> instance.delete()
 
 # Media on a ModelForm ########################################################

tests/regressiontests/admin_widgets/models.py

 
 from django.conf import settings
 from django.db import models
+from django.core.files.storage import default_storage
 
 class Member(models.Model):
     name = models.CharField(max_length=100)
@@ -18,6 +19,7 @@ class Band(models.Model):
 class Album(models.Model):
     band = models.ForeignKey(Band)
     name = models.CharField(max_length=100)
+    cover_art = models.ImageField(upload_to='albums')
     
     def __unicode__(self):
         return self.name
@@ -46,12 +48,12 @@ HTML escaped.
 >>> print conditional_escape(w.render('test', datetime(2007, 12, 1, 9, 30)))
 <p class="datetime">Date: <input value="2007-12-01" type="text" class="vDateField" name="test_0" size="10" /><br />Time: <input value="09:30:00" type="text" class="vTimeField" name="test_1" size="8" /></p>
 
->>> w = AdminFileWidget()
->>> print conditional_escape(w.render('test', 'test'))
-Currently: <a target="_blank" href="%(MEDIA_URL)stest">test</a> <br />Change: <input type="file" name="test" />
-
 >>> band = Band.objects.create(pk=1, name='Linkin Park')
->>> album = band.album_set.create(name='Hybrid Theory')
+>>> album = band.album_set.create(name='Hybrid Theory', cover_art=r'albums\hybrid_theory.jpg')
+
+>>> w = AdminFileWidget()
+>>> print conditional_escape(w.render('test', album.cover_art))
+Currently: <a target="_blank" href="%(STORAGE_URL)salbums/hybrid_theory.jpg">albums\hybrid_theory.jpg</a> <br />Change: <input type="file" name="test" />
 
 >>> rel = Album._meta.get_field('band').rel
 >>> w = ForeignKeyRawIdWidget(rel)
@@ -81,5 +83,5 @@ True
 
 """ % {
     'ADMIN_MEDIA_PREFIX': settings.ADMIN_MEDIA_PREFIX,
-    'MEDIA_URL': settings.MEDIA_URL,
+    'STORAGE_URL': default_storage.url(''),
 }}

tests/regressiontests/bug639/models.py

 import tempfile
+
 from django.db import models
+from django.core.files.storage import FileSystemStorage
+
+temp_storage = FileSystemStorage(tempfile.gettempdir())
 
 class Photo(models.Model):
     title = models.CharField(max_length=30)
-    image = models.FileField(upload_to=tempfile.gettempdir())
+    image = models.FileField(storage=temp_storage, upload_to='tests')
     
     # Support code for the tests; this keeps track of how many times save() gets
     # called on each instance.
     def __init__(self, *args, **kwargs):
-       super(Photo, self).__init__(*args, **kwargs)
-       self._savecount = 0
+        super(Photo, self).__init__(*args, **kwargs)
+        self._savecount = 0
     
     def save(self):
         super(Photo, self).save()
-        self._savecount +=1
\ No newline at end of file
+        self._savecount += 1

tests/regressiontests/bug639/tests.py

@@ -36,4 +36,4 @@ class Bug639Test(unittest.TestCase):
         Make sure to delete the "uploaded" file to avoid clogging /tmp.
         """
         p = Photo.objects.get()
-        os.unlink(p.get_image_filename())
+        p.image.delete(save=False)

tests/regressiontests/file_storage/__init__.py

+

tests/regressiontests/file_storage/models.py

+import os
+import tempfile
+from django.db import models
+from django.core.files.storage import FileSystemStorage
+from django.core.files.base import ContentFile
+
+temp_storage = FileSystemStorage(tempfile.gettempdir())
+
+# Test for correct behavior of width_field/height_field.
+# Of course, we can't run this without PIL.
+
+try:
+    # Checking for the existence of Image is enough for CPython, but
+    # for PyPy, you need to check for the underlying modules
+    import Image, _imaging
+except ImportError:
+    Image = None
+
+# If we have PIL, do these tests
+if Image:
+    class Person(models.Model):
+        name = models.CharField(max_length=50)
+        mugshot = models.ImageField(storage=temp_storage, upload_to='tests', 
+                                    height_field='mug_height', 
+                                    width_field='mug_width')
+        mug_height = models.PositiveSmallIntegerField()
+        mug_width = models.PositiveSmallIntegerField()
+        
+    __test__ = {'API_TESTS': """
+
+>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png"), 'rb').read()
+>>> p = Person(name="Joe")
+>>> p.mugshot.save("mug", ContentFile(image_data))
+>>> p.mugshot.width
+16
+>>> p.mugshot.height
+16
+>>> p.mug_height
+16
+>>> p.mug_width
+16
+
+"""}
+    
\ No newline at end of file

tests/regressiontests/file_storage/tests.py

+"""
+Tests for the file storage mechanism
+
+>>> import tempfile
+>>> from django.core.files.storage import FileSystemStorage
+>>> from django.core.files.base import ContentFile
+
+>>> temp_storage = FileSystemStorage(location=tempfile.gettempdir())
+
+# Standard file access options are available, and work as expected.
+
+>>> temp_storage.exists('storage_test')
+False
+>>> file = temp_storage.open('storage_test', 'w')
+>>> file.write('storage contents')
+>>> file.close()
+
+>>> temp_storage.exists('storage_test')
+True
+>>> file = temp_storage.open('storage_test', 'r')
+>>> file.read()
+'storage contents'
+>>> file.close()
+
+>>> temp_storage.delete('storage_test')
+>>> temp_storage.exists('storage_test')
+False
+
+# Files can only be accessed if they're below the specified location.
+
+>>> temp_storage.exists('..')
+Traceback (most recent call last):
+...
+SuspiciousOperation: Attempted access to '..' denied.
+>>> temp_storage.open('/etc/passwd')
+Traceback (most recent call last):
+  ...
+SuspiciousOperation: Attempted access to '/etc/passwd' denied.
+
+# Custom storage systems can be created to customize behavior
+
+>>> class CustomStorage(FileSystemStorage):
+...     def get_available_name(self, name):
+...         # Append numbers to duplicate files rather than underscores, like Trac
+...
+...         parts = name.split('.')
+...         basename, ext = parts[0], parts[1:]
+...         number = 2
+...
+...         while self.exists(name):
+...             name = '.'.join([basename, str(number)] + ext)
+...             number += 1
+...
+...         return name
+>>> custom_storage = CustomStorage(tempfile.gettempdir())
+
+>>> first = custom_storage.save('custom_storage', ContentFile('custom contents'))
+>>> first
+u'custom_storage'
+>>> second = custom_storage.save('custom_storage', ContentFile('more contents'))
+>>> second
+u'custom_storage.2'
+
+>>> custom_storage.delete(first)
+>>> custom_storage.delete(second)
+"""

tests/regressiontests/file_uploads/models.py

 import tempfile
 import os
 from django.db import models
+from django.core.files.storage import FileSystemStorage
 
-UPLOAD_ROOT = tempfile.mkdtemp()
-UPLOAD_TO = os.path.join(UPLOAD_ROOT, 'test_upload')
+temp_storage = FileSystemStorage(tempfile.mkdtemp())
+UPLOAD_TO = os.path.join(temp_storage.location, 'test_upload')
 
 class FileModel(models.Model):
-    testfile = models.FileField(upload_to=UPLOAD_TO)
+    testfile = models.FileField(storage=temp_storage, upload_to='test_upload')

tests/regressiontests/file_uploads/tests.py

@@ -9,7 +9,7 @@ from django.test import TestCase, client
 from django.utils import simplejson
 from django.utils.hashcompat import sha_constructor
 
-from models import FileModel, UPLOAD_ROOT, UPLOAD_TO
+from models import FileModel, temp_storage, UPLOAD_TO
 
 class FileUploadTests(TestCase):
     def test_simple_upload(self):
@@ -194,22 +194,22 @@ class DirectoryCreationTests(unittest.TestCase):
     """
     def setUp(self):
         self.obj = FileModel()
-        if not os.path.isdir(UPLOAD_ROOT):
-            os.makedirs(UPLOAD_ROOT)
+        if not os.path.isdir(temp_storage.location):
+            os.makedirs(temp_storage.location)
 
     def tearDown(self):
-        os.chmod(UPLOAD_ROOT, 0700)
-        shutil.rmtree(UPLOAD_ROOT)
+        os.chmod(temp_storage.location, 0700)
+        shutil.rmtree(temp_storage.location)
 
     def test_readonly_root(self):
         """Permission errors are not swallowed"""
-        os.chmod(UPLOAD_ROOT, 0500)
+        os.chmod(temp_storage.location, 0500)
         try:
-            self.obj.save_testfile_file('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
+            self.obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
         except OSError, err:
             self.assertEquals(err.errno, errno.EACCES)
-        except:
-            self.fail("OSError [Errno %s] not raised" % errno.EACCES)
+        except Exception, err:
+            self.fail("OSError [Errno %s] not raised." % errno.EACCES)
 
     def test_not_a_directory(self):
         """The correct IOError is raised when the upload directory name exists but isn't a directory"""
@@ -217,11 +217,11 @@ class DirectoryCreationTests(unittest.TestCase):
         fd = open(UPLOAD_TO, 'w')
         fd.close()
         try:
-            self.obj.save_testfile_file('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
+            self.obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
         except IOError, err:
             # The test needs to be done on a specific string as IOError
             # is raised even without the patch (just not early enough)
             self.assertEquals(err.args[0],
-                              "%s exists and is not a directory" % UPLOAD_TO)
+                              "%s exists and is not a directory." % UPLOAD_TO)
         except:
             self.fail("IOError not raised")

tests/regressiontests/serializers_regress/models.py

@@ -157,8 +157,8 @@ class DecimalPKData(models.Model):
 class EmailPKData(models.Model):
     data = models.EmailField(primary_key=True)
 
-class FilePKData(models.Model):
-    data = models.FileField(primary_key=True, upload_to='/foo/bar')
+# class FilePKData(models.Model):
+#    data = models.FileField(primary_key=True, upload_to='/foo/bar')
 
 class FilePathPKData(models.Model):
     data = models.FilePathField(primary_key=True)

tests/regressiontests/serializers_regress/tests.py

@@ -144,7 +144,7 @@ test_data = [
     (data_obj, 41, EmailData, None),
     (data_obj, 42, EmailData, ""),
     (data_obj, 50, FileData, 'file:///foo/bar/whiz.txt'),
-    (data_obj, 51, FileData, None),
+#     (data_obj, 51, FileData, None),
     (data_obj, 52, FileData, ""),
     (data_obj, 60, FilePathData, "/foo/bar/whiz.txt"),
     (data_obj, 61, FilePathData, None),
@@ -242,7 +242,7 @@ The end."""),
 #     (pk_obj, 620, DatePKData, datetime.date(2006,6,16)),
 #     (pk_obj, 630, DateTimePKData, datetime.datetime(2006,6,16,10,42,37)),
     (pk_obj, 640, EmailPKData, "hovercraft@example.com"),
-    (pk_obj, 650, FilePKData, 'file:///foo/bar/whiz.txt'),
+#     (pk_obj, 650, FilePKData, 'file:///foo/bar/whiz.txt'),
     (pk_obj, 660, FilePathPKData, "/foo/bar/whiz.txt"),
     (pk_obj, 670, DecimalPKData, decimal.Decimal('12.345')),
     (pk_obj, 671, DecimalPKData, decimal.Decimal('-12.345')),