Added ManifestStaticFilesStorage to staticfiles contrib app.

It uses a static manifest file that is created when running
collectstatic in the JSON format.

django/contrib/staticfiles/storage.py

@@ -5,6 +5,7 @@ from importlib import import_module
 import os
 import posixpath
 import re
+import json
 
 from django.conf import settings
 from django.core.cache import (caches, InvalidCacheBackendError,
@@ -49,7 +50,7 @@ class StaticFilesStorage(FileSystemStorage):
         return super(StaticFilesStorage, self).path(name)
 
 
-class CachedFilesMixin(object):
+class HashedFilesMixin(object):
     default_template = """url("%s")"""
     patterns = (
         ("*.css", (
@@ -59,13 +60,9 @@ class CachedFilesMixin(object):
     )
 
     def __init__(self, *args, **kwargs):
-        super(CachedFilesMixin, self).__init__(*args, **kwargs)
-        try:
-            self.cache = caches['staticfiles']
-        except InvalidCacheBackendError:
-            # Use the default backend
-            self.cache = default_cache
+        super(HashedFilesMixin, self).__init__(*args, **kwargs)
         self._patterns = OrderedDict()
+        self.hashed_files = {}
         for extension, patterns in self.patterns:
             for pattern in patterns:
                 if isinstance(pattern, (tuple, list)):
@@ -119,9 +116,6 @@ class CachedFilesMixin(object):
             unparsed_name[2] += '?'
         return urlunsplit(unparsed_name)
 
-    def cache_key(self, name):
-        return 'staticfiles:%s' % hashlib.md5(force_bytes(name)).hexdigest()
-
     def url(self, name, force=False):
         """
         Returns the real URL in DEBUG mode.
@@ -133,15 +127,9 @@ class CachedFilesMixin(object):
             if urlsplit(clean_name).path.endswith('/'):  # don't hash paths
                 hashed_name = name
             else:
-                cache_key = self.cache_key(name)
-                hashed_name = self.cache.get(cache_key)
-                if hashed_name is None:
-                    hashed_name = self.hashed_name(clean_name).replace('\\', '/')
-                    # set the cache if there was a miss
-                    # (e.g. if cache server goes down)
-                    self.cache.set(cache_key, hashed_name)
+                hashed_name = self.stored_name(clean_name)
 
-        final_url = super(CachedFilesMixin, self).url(hashed_name)
+        final_url = super(HashedFilesMixin, self).url(hashed_name)
 
         # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
         # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
@@ -220,7 +208,7 @@ class CachedFilesMixin(object):
             return
 
         # where to store the new paths
-        hashed_paths = {}
+        hashed_files = OrderedDict()
 
         # build a list of adjustable files
         matches = lambda path: matches_patterns(path, self._patterns.keys())
@@ -261,7 +249,7 @@ class CachedFilesMixin(object):
                     # then save the processed result
                     content_file = ContentFile(force_bytes(content))
                     saved_name = self._save(hashed_name, content_file)
-                    hashed_name = force_text(saved_name.replace('\\', '/'))
+                    hashed_name = force_text(self.clean_name(saved_name))
                     processed = True
                 else:
                     # or handle the case in which neither processing nor
@@ -269,14 +257,114 @@ class CachedFilesMixin(object):
                     if not hashed_file_exists:
                         processed = True
                         saved_name = self._save(hashed_name, original_file)
-                        hashed_name = force_text(saved_name.replace('\\', '/'))
+                        hashed_name = force_text(self.clean_name(saved_name))
 
                 # and then set the cache accordingly
-                hashed_paths[self.cache_key(name.replace('\\', '/'))] = hashed_name
+                hashed_files[self.hash_key(name)] = hashed_name
                 yield name, hashed_name, processed
 
-        # Finally set the cache
-        self.cache.set_many(hashed_paths)
+        # Finally store the processed paths
+        self.hashed_files.update(hashed_files)
+
+    def clean_name(self, name):
+        return name.replace('\\', '/')
+
+    def hash_key(self, name):
+        return name
+
+    def stored_name(self, name):
+        hash_key = self.hash_key(name)
+        cache_name = self.hashed_files.get(hash_key)
+        if cache_name is None:
+            cache_name = self.clean_name(self.hashed_name(name))
+            # store the hashed name if there was a miss, e.g.
+            # when the files are still processed
+            self.hashed_files[hash_key] = cache_name
+        return cache_name
+
+
+class ManifestFilesMixin(HashedFilesMixin):
+    manifest_version = '1.0'  # the manifest format standard
+    manifest_name = 'staticfiles.json'
+
+    def __init__(self, *args, **kwargs):
+        super(ManifestFilesMixin, self).__init__(*args, **kwargs)
+        self.hashed_files = self.load_manifest()
+
+    def read_manifest(self):
+        try:
+            with self.open(self.manifest_name) as manifest:
+                return manifest.read()
+        except IOError:
+            return None
+
+    def load_manifest(self):
+        content = self.read_manifest()
+        if content is None:
+            return OrderedDict()
+        try:
+            stored = json.loads(content, object_pairs_hook=OrderedDict)
+        except ValueError:
+            pass
+        else:
+            version = stored.get('version', None)
+            if version == '1.0':
+                return stored.get('paths', OrderedDict())
+        raise ValueError("Couldn't load manifest '%s' (version %s)" %
+                         (self.manifest_name, self.manifest_version))
+
+    def post_process(self, *args, **kwargs):
+        all_post_processed = super(ManifestFilesMixin,
+                                   self).post_process(*args, **kwargs)
+        for post_processed in all_post_processed:
+            yield post_processed
+        payload = {'paths': self.hashed_files, 'version': self.manifest_version}
+        if self.exists(self.manifest_name):
+            self.delete(self.manifest_name)
+        self._save(self.manifest_name, ContentFile(json.dumps(payload)))
+
+
+class _MappingCache(object):
+    """
+    A small dict-like wrapper for a given cache backend instance.
+    """
+    def __init__(self, cache):
+        self.cache = cache
+
+    def __setitem__(self, key, value):
+        self.cache.set(key, value)
+
+    def __getitem__(self, key):
+        value = self.cache.get(key, None)
+        if value is None:
+            raise KeyError("Couldn't find a file name '%s'" % key)
+        return value
+
+    def clear(self):
+        self.cache.clear()
+
+    def update(self, data):
+        self.cache.set_many(data)
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+
+class CachedFilesMixin(HashedFilesMixin):
+    def __init__(self, *args, **kwargs):
+        super(CachedFilesMixin, self).__init__(*args, **kwargs)
+        try:
+            self.hashed_files = _MappingCache(caches['staticfiles'])
+        except InvalidCacheBackendError:
+            # Use the default backend
+            self.hashed_files = _MappingCache(default_cache)
+
+    def hash_key(self, name):
+        key = hashlib.md5(force_bytes(self.clean_name(name))).hexdigest()
+        return 'staticfiles:%s' % key
 
 
 class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
@@ -287,6 +375,14 @@ class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
     pass
 
 
+class ManifestStaticFilesStorage(ManifestFilesMixin, StaticFilesStorage):
+    """
+    A static file system storage backend which also saves
+    hashed copies of the files it saves.
+    """
+    pass
+
+
 class AppStaticStorage(FileSystemStorage):
     """
     A file system storage backend that takes an app module and works

docs/ref/contrib/staticfiles.txt

@@ -210,91 +210,107 @@ StaticFilesStorage
 
 .. class:: storage.StaticFilesStorage
 
-    A subclass of the :class:`~django.core.files.storage.FileSystemStorage`
-    storage backend that uses the :setting:`STATIC_ROOT` setting as the base
-    file system location and the :setting:`STATIC_URL` setting respectively
-    as the base URL.
+A subclass of the :class:`~django.core.files.storage.FileSystemStorage`
+storage backend that uses the :setting:`STATIC_ROOT` setting as the base
+file system location and the :setting:`STATIC_URL` setting respectively
+as the base URL.
 
-    .. method:: post_process(paths, **options)
+.. method:: post_process(paths, **options)
 
-    This method is called by the :djadmin:`collectstatic` management command
-    after each run and gets passed the local storages and paths of found
-    files as a dictionary, as well as the command line options.
+This method is called by the :djadmin:`collectstatic` management command
+after each run and gets passed the local storages and paths of found
+files as a dictionary, as well as the command line options.
 
-    The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
-    uses this behind the scenes to replace the paths with their hashed
-    counterparts and update the cache appropriately.
+The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
+uses this behind the scenes to replace the paths with their hashed
+counterparts and update the cache appropriately.
 
-CachedStaticFilesStorage
-------------------------
+ManifestStaticFilesStorage
+--------------------------
 
-.. class:: storage.CachedStaticFilesStorage
+.. versionadded:: 1.7
+
+.. class:: storage.ManifestStaticFilesStorage
+
+A subclass of the :class:`~django.contrib.staticfiles.storage.StaticFilesStorage`
+storage backend which stores the file names it handles by appending the MD5
+hash of the file's content to the filename. For example, the file
+``css/styles.css`` would also be saved as ``css/styles.55e7cbb9ba48.css``.
+
+The purpose of this storage is to keep serving the old files in case some
+pages still refer to those files, e.g. because they are cached by you or
+a 3rd party proxy server. Additionally, it's very helpful if you want to
+apply `far future Expires headers`_ to the deployed files to speed up the
+load time for subsequent page visits.
+
+The storage backend automatically replaces the paths found in the saved
+files matching other saved files with the path of the cached copy (using
+the :meth:`~django.contrib.staticfiles.storage.StaticFilesStorage.post_process`
+method). The regular expressions used to find those paths
+(``django.contrib.staticfiles.storage.HashedFilesMixin.patterns``)
+by default covers the `@import`_ rule and `url()`_ statement of `Cascading
+Style Sheets`_. For example, the ``'css/styles.css'`` file with the
+content
+
+.. code-block:: css+django
+
+    @import url("../admin/css/base.css");
+
+would be replaced by calling the :meth:`~django.core.files.storage.Storage.url`
+method of the ``ManifestStaticFilesStorage`` storage backend, ultimately
+saving a ``'css/styles.55e7cbb9ba48.css'`` file with the following
+content:
+
+.. code-block:: css+django
 
-    A subclass of the :class:`~django.contrib.staticfiles.storage.StaticFilesStorage`
-    storage backend which caches the files it saves by appending the MD5 hash
-    of the file's content to the filename. For example, the file
-    ``css/styles.css`` would also be saved as ``css/styles.55e7cbb9ba48.css``.
-
-    The purpose of this storage is to keep serving the old files in case some
-    pages still refer to those files, e.g. because they are cached by you or
-    a 3rd party proxy server. Additionally, it's very helpful if you want to
-    apply `far future Expires headers`_ to the deployed files to speed up the
-    load time for subsequent page visits.
-
-    The storage backend automatically replaces the paths found in the saved
-    files matching other saved files with the path of the cached copy (using
-    the :meth:`~django.contrib.staticfiles.storage.StaticFilesStorage.post_process`
-    method). The regular expressions used to find those paths
-    (``django.contrib.staticfiles.storage.CachedStaticFilesStorage.cached_patterns``)
-    by default cover the `@import`_ rule and `url()`_ statement of `Cascading
-    Style Sheets`_. For example, the ``'css/styles.css'`` file with the
-    content
-
-    .. code-block:: css+django
-
-        @import url("../admin/css/base.css");
-
-    would be replaced by calling the
-    :meth:`~django.core.files.storage.Storage.url`
-    method of the ``CachedStaticFilesStorage`` storage backend, ultimately
-    saving a ``'css/styles.55e7cbb9ba48.css'`` file with the following
-    content:
-
-    .. code-block:: css+django
-
-        @import url("../admin/css/base.27e20196a850.css");
-
-    To enable the ``CachedStaticFilesStorage`` you have to make sure the
-    following requirements are met:
-
-    * the :setting:`STATICFILES_STORAGE` setting is set to
-      ``'django.contrib.staticfiles.storage.CachedStaticFilesStorage'``
-    * the :setting:`DEBUG` setting is set to ``False``
-    * you use the ``staticfiles`` :ttag:`static<staticfiles-static>` template
-      tag to refer to your static files in your templates
-    * you've collected all your static files by using the
-      :djadmin:`collectstatic` management command
-
-    Since creating the MD5 hash can be a performance burden to your website
-    during runtime, ``staticfiles`` will automatically try to cache the
-    hashed name for each file path using Django's :doc:`caching
-    framework</topics/cache>`. If you want to override certain options of the
-    cache backend the storage uses, simply specify a custom entry in the
-    :setting:`CACHES` setting named ``'staticfiles'``. It falls back to using
-    the ``'default'`` cache backend.
-
-    .. method:: file_hash(name, content=None)
-
-    The method that is used when creating the hashed name of a file.
-    Needs to return a hash for the given file name and content.
-    By default it calculates a MD5 hash from the content's chunks as
-    mentioned above.
+    @import url("../admin/css/base.27e20196a850.css");
+
+To enable the ``ManifestStaticFilesStorage`` you have to make sure the
+following requirements are met:
+
+* the :setting:`STATICFILES_STORAGE` setting is set to
+  ``'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'``
+* the :setting:`DEBUG` setting is set to ``False``
+* you use the ``staticfiles`` :ttag:`static<staticfiles-static>` template
+  tag to refer to your static files in your templates
+* you've collected all your static files by using the
+  :djadmin:`collectstatic` management command
+
+Since creating the MD5 hash can be a performance burden to your website
+during runtime, ``staticfiles`` will automatically store the mapping with
+hashed names for all processed files in a file called ``staticfiles.json``.
+This happens once when you run the :djadmin:`collectstatic` management
+command.
+
+.. method:: file_hash(name, content=None)
+
+The method that is used when creating the hashed name of a file.
+Needs to return a hash for the given file name and content.
+By default it calculates a MD5 hash from the content's chunks as
+mentioned above. Feel free to override this method to use your own
+hashing algorithm.
 
 .. _`far future Expires headers`: http://developer.yahoo.com/performance/rules.html#expires
 .. _`@import`: http://www.w3.org/TR/CSS2/cascade.html#at-import
 .. _`url()`: http://www.w3.org/TR/CSS2/syndata.html#uri
 .. _`Cascading Style Sheets`: http://www.w3.org/Style/CSS/
 
+CachedStaticFilesStorage
+------------------------
+
+.. class:: storage.CachedStaticFilesStorage
+
+``CachedStaticFilesStorage`` is a similar class like the
+:class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage` class
+but uses Django's :doc:`caching framework</topics/cache>` for storing the
+hashed names of processed files instead of a static manifest file called
+``staticfiles.json``. This is mostly useful for situations in which you don't
+have accesss to the file system.
+
+If you want to override certain options of the cache backend the storage uses,
+simply specify a custom entry in the :setting:`CACHES` setting named
+``'staticfiles'``. It falls back to using the ``'default'`` cache backend.
+
 .. currentmodule:: django.contrib.staticfiles.templatetags.staticfiles
 
 Template tags

docs/releases/1.7.txt

@@ -340,6 +340,19 @@ Minor features
   and :attr:`~django.core.files.storage.FileSystemStorage.directory_permissions_mode`
   parameters. See :djadmin:`collectstatic` for example usage.
 
+* The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
+  backend gets a sibling class called
+  :class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage`
+  that doesn't use the cache system at all but instead a JSON file called
+  ``staticfiles.json`` for storing the mapping between the original file name
+  (e.g. ``css/styles.css``) and the hashed file name (e.g.
+  ``css/styles.55e7cbb9ba48.css``. The ``staticfiles.json`` file is created
+  when running the :djadmin:`collectstatic` management command and should
+  be a less expensive alternative for remote storages such as Amazon S3.
+
+  See the :class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage`
+  docs for more information.
+
 :mod:`django.contrib.syndication`
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^