Fixed #3566 -- Added support for aggregation to the ORM. See the documentation…

Fixed #3566 -- Added support for aggregation to the ORM. See the documentation for details on usage.

Many thanks to:
 * Nicolas Lara, who worked on this feature during the 2008 Google Summer of Code.
 * Alex Gaynor for his help debugging and fixing a number of issues.
 * Justin Bronn for his help integrating with contrib.gis.
 * Karen Tracey for her help with cross-platform testing.
 * Ian Kelly for his help testing and fixing Oracle support.
 * Malcolm Tredinnick for his invaluable review notes.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@9742 bcc190cf-cafb-0310-a4f2-bffc1f526a37

AUTHORS

@@ -31,6 +31,7 @@ answer newbie questions, and generally made Django that much better:
     AgarFu <heaven@croasanaso.sytes.net>
     Dagur Páll Ammendrup <dagurp@gmail.com>
     Collin Anderson <cmawebsite@gmail.com>
+    Nicolas Lara <nicolaslara@gmail.com>
     Jeff Anderson <jefferya@programmerq.net>
     Marian Andre <django@andre.sk>
     Andreas

django/contrib/gis/db/models/aggregates.py

+from django.db.models import Aggregate
+
+class Extent(Aggregate):
+    name = 'Extent'
+
+class MakeLine(Aggregate):
+    name = 'MakeLine'
+
+class Union(Aggregate):
+    name = 'Union'

django/contrib/gis/db/models/sql/aggregates.py

+from django.db.models.sql.aggregates import *
+
+from django.contrib.gis.db.models.fields import GeometryField
+from django.contrib.gis.db.backend import SpatialBackend
+
+if SpatialBackend.oracle:
+    geo_template = '%(function)s(SDOAGGRTYPE(%(field)s,%(tolerance)s))'
+else:
+    geo_template = '%(function)s(%(field)s)'
+
+class GeoAggregate(Aggregate):
+    # Overriding the SQL template with the geographic one.
+    sql_template = geo_template
+
+    is_extent = False
+
+    def __init__(self, col, source=None, is_summary=False, **extra):
+        super(GeoAggregate, self).__init__(col, source, is_summary, **extra)
+
+        # Can't use geographic aggregates on non-geometry fields.
+        if not isinstance(self.source, GeometryField):
+            raise ValueError('Geospatial aggregates only allowed on geometry fields.')
+
+        # Making sure the SQL function is available for this spatial backend.
+        if not self.sql_function:
+            raise NotImplementedError('This aggregate functionality not implemented for your spatial backend.')
+
+class Extent(GeoAggregate):
+    is_extent = True
+    sql_function = SpatialBackend.extent
+
+class MakeLine(GeoAggregate):
+    sql_function = SpatialBackend.make_line
+
+class Union(GeoAggregate):
+    sql_function = SpatialBackend.unionagg

django/db/backends/__init__.py

@@ -10,6 +10,12 @@ except NameError:
     # Python 2.3 compat
     from sets import Set as set
 
+try:
+    import decimal
+except ImportError:
+    # Python 2.3 fallback
+    from django.utils import _decimal as decimal
+
 from django.db.backends import util
 from django.utils import datetime_safe
 
@@ -62,6 +68,7 @@ class BaseDatabaseWrapper(local):
         return util.CursorDebugWrapper(cursor, self)
 
 class BaseDatabaseFeatures(object):
+    allows_group_by_pk = False
     # True if django.db.backend.utils.typecast_timestamp is used on values
     # returned from dates() calls.
     needs_datetime_string_cast = True
@@ -376,6 +383,22 @@ class BaseDatabaseOperations(object):
         """
         return self.year_lookup_bounds(value)
 
+    def convert_values(self, value, field):
+        """Coerce the value returned by the database backend into a consistent type that
+        is compatible with the field type.
+        """
+        internal_type = field.get_internal_type()
+        if internal_type == 'DecimalField':
+            return value
+        elif internal_type and internal_type.endswith('IntegerField') or internal_type == 'AutoField':
+            return int(value)
+        elif internal_type in ('DateField', 'DateTimeField', 'TimeField'):
+            return value
+        # No field, or the field isn't known to be a decimal or integer
+        # Default to a float
+        return float(value)
+
+
 class BaseDatabaseIntrospection(object):
     """
     This class encapsulates all backend-specific introspection utilities

django/db/backends/mysql/base.py

@@ -110,6 +110,7 @@ class CursorWrapper(object):
 class DatabaseFeatures(BaseDatabaseFeatures):
     empty_fetchmany_value = ()
     update_can_self_select = False
+    allows_group_by_pk = True
     related_fields_match_type = True
 
 class DatabaseOperations(BaseDatabaseOperations):

django/db/backends/oracle/query.py

@@ -53,21 +53,23 @@ def query_class(QueryClass, Database):
             return values
 
         def convert_values(self, value, field):
-            from django.db.models.fields import DateField, DateTimeField, \
-                 TimeField, BooleanField, NullBooleanField, DecimalField, Field
+            from django.db.models.fields import Field
             if isinstance(value, Database.LOB):
                 value = value.read()
             # Oracle stores empty strings as null. We need to undo this in
             # order to adhere to the Django convention of using the empty
             # string instead of null, but only if the field accepts the
             # empty string.
-            if value is None and isinstance(field, Field) and field.empty_strings_allowed:
+            if value is None and field and field.empty_strings_allowed:
                 value = u''
             # Convert 1 or 0 to True or False
-            elif value in (1, 0) and isinstance(field, (BooleanField, NullBooleanField)):
+            elif value in (1, 0) and field and field.get_internal_type() in ('BooleanField', 'NullBooleanField'):
                 value = bool(value)
+            # Force floats to the correct type
+            elif value is not None and field and field.get_internal_type() == 'FloatField':
+                value = float(value)
             # Convert floats to decimals
-            elif value is not None and isinstance(field, DecimalField):
+            elif value is not None and field and field.get_internal_type() == 'DecimalField':
                 value = util.typecast_decimal(field.format_number(value))
             # cx_Oracle always returns datetime.datetime objects for
             # DATE and TIMESTAMP columns, but Django wants to see a
@@ -86,13 +88,9 @@ def query_class(QueryClass, Database):
                     value = datetime.datetime(value.year, value.month,
                             value.day, value.hour, value.minute, value.second,
                             value.fsecond)
-                if isinstance(field, DateTimeField):
-                    # DateTimeField subclasses DateField so must be checked
-                    # first.
-                    pass
-                elif isinstance(field, DateField):
+                if field and field.get_internal_type() == 'DateField':
                     value = value.date()
-                elif isinstance(field, TimeField) or (value.year == 1900 and value.month == value.day == 1):
+                elif field and field.get_internal_type() == 'TimeField' or (value.year == 1900 and value.month == value.day == 1):
                     value = value.time()
                 elif value.hour == value.minute == value.second == value.microsecond == 0:
                     value = value.date()

django/db/backends/sqlite3/base.py

@@ -10,7 +10,7 @@ from django.db.backends import *
 from django.db.backends.sqlite3.client import DatabaseClient
 from django.db.backends.sqlite3.creation import DatabaseCreation
 from django.db.backends.sqlite3.introspection import DatabaseIntrospection
-from django.utils.safestring import SafeString                                                           
+from django.utils.safestring import SafeString
 
 try:
     try:
@@ -102,6 +102,26 @@ class DatabaseOperations(BaseDatabaseOperations):
         second = '%s-12-31 23:59:59.999999'
         return [first % value, second % value]
 
+    def convert_values(self, value, field):
+        """SQLite returns floats when it should be returning decimals,
+        and gets dates and datetimes wrong.
+        For consistency with other backends, coerce when required.
+        """
+        internal_type = field.get_internal_type()
+        if internal_type == 'DecimalField':
+            return util.typecast_decimal(field.format_number(value))
+        elif internal_type and internal_type.endswith('IntegerField') or internal_type == 'AutoField':
+            return int(value)
+        elif internal_type == 'DateField':
+            return util.typecast_date(value)
+        elif internal_type == 'DateTimeField':
+            return util.typecast_timestamp(value)
+        elif internal_type == 'TimeField':
+            return util.typecast_time(value)
+
+        # No field, or the field isn't known to be a decimal or integer
+        return value
+
 class DatabaseWrapper(BaseDatabaseWrapper):
 
     # SQLite requires LIKE statements to include an ESCAPE clause if the value

django/db/models/__init__.py

@@ -5,6 +5,7 @@ from django.db.models.loading import get_apps, get_app, get_models, get_model, r
 from django.db.models.query import Q
 from django.db.models.manager import Manager
 from django.db.models.base import Model
+from django.db.models.aggregates import *
 from django.db.models.fields import *
 from django.db.models.fields.subclassing import SubfieldBase
 from django.db.models.fields.files import FileField, ImageField

django/db/models/aggregates.py

+"""
+Classes to represent the definitions of aggregate functions.
+"""
+
+class Aggregate(object):
+    """
+    Default Aggregate definition.
+    """
+    def __init__(self, lookup, **extra):
+        """Instantiate a new aggregate.
+
+         * lookup is the field on which the aggregate operates.
+         * extra is a dictionary of additional data to provide for the
+           aggregate definition
+
+        Also utilizes the class variables:
+         * name, the identifier for this aggregate function.
+        """
+        self.lookup = lookup
+        self.extra = extra
+
+    def _default_alias(self):
+        return '%s__%s' % (self.lookup, self.name.lower())
+    default_alias = property(_default_alias)
+
+    def add_to_query(self, query, alias, col, source, is_summary):
+        """Add the aggregate to the nominated query.
+
+        This method is used to convert the generic Aggregate definition into a
+        backend-specific definition.
+
+         * query is the backend-specific query instance to which the aggregate
+           is to be added.
+         * col is a column reference describing the subject field
+           of the aggregate. It can be an alias, or a tuple describing
+           a table and column name.
+         * source is the underlying field or aggregate definition for
+           the column reference. If the aggregate is not an ordinal or
+           computed type, this reference is used to determine the coerced
+           output type of the aggregate.
+         * is_summary is a boolean that is set True if the aggregate is a
+           summary value rather than an annotation.
+        """
+        aggregate = getattr(query.aggregates_module, self.name)
+        query.aggregate_select[alias] = aggregate(col, source=source, is_summary=is_summary, **self.extra)
+
+class Avg(Aggregate):
+    name = 'Avg'
+
+class Count(Aggregate):
+    name = 'Count'
+
+class Max(Aggregate):
+    name = 'Max'
+
+class Min(Aggregate):
+    name = 'Min'
+
+class StdDev(Aggregate):
+    name = 'StdDev'
+
+class Sum(Aggregate):
+    name = 'Sum'
+
+class Variance(Aggregate):
+    name = 'Variance'

django/db/models/manager.py

@@ -101,6 +101,12 @@ class Manager(object):
     def filter(self, *args, **kwargs):
         return self.get_query_set().filter(*args, **kwargs)
 
+    def aggregate(self, *args, **kwargs):
+        return self.get_query_set().aggregate(*args, **kwargs)
+
+    def annotate(self, *args, **kwargs):
+        return self.get_query_set().annotate(*args, **kwargs)
+
     def complex_filter(self, *args, **kwargs):
         return self.get_query_set().complex_filter(*args, **kwargs)
 

django/db/models/query.py

@@ -4,6 +4,7 @@ except NameError:
     from sets import Set as set     # Python 2.3 fallback
 
 from django.db import connection, transaction, IntegrityError
+from django.db.models.aggregates import Aggregate
 from django.db.models.fields import DateField
 from django.db.models.query_utils import Q, select_related_descend
 from django.db.models import signals, sql
@@ -270,18 +271,47 @@ class QuerySet(object):
         else:
             requested = None
         max_depth = self.query.max_depth
+
         extra_select = self.query.extra_select.keys()
+        aggregate_select = self.query.aggregate_select.keys()
+
         index_start = len(extra_select)
+        aggregate_start = index_start + len(self.model._meta.fields)
+
         for row in self.query.results_iter():
             if fill_cache:
-                obj, _ = get_cached_row(self.model, row, index_start,
-                        max_depth, requested=requested)
+                obj, aggregate_start = get_cached_row(self.model, row,
+                                    index_start, max_depth, requested=requested)
             else:
-                obj = self.model(*row[index_start:])
+                # omit aggregates in object creation
+                obj = self.model(*row[index_start:aggregate_start])
+
             for i, k in enumerate(extra_select):
                 setattr(obj, k, row[i])
+
+            # Add the aggregates to the model
+            for i, aggregate in enumerate(aggregate_select):
+                setattr(obj, aggregate, row[i+aggregate_start])
+
             yield obj
 
+    def aggregate(self, *args, **kwargs):
+        """
+        Returns a dictionary containing the calculations (aggregation)
+        over the current queryset
+
+        If args is present the expression is passed as a kwarg ussing
+        the Aggregate object's default alias.
+        """
+        for arg in args:
+            kwargs[arg.default_alias] = arg
+
+        for (alias, aggregate_expr) in kwargs.items():
+            self.query.add_aggregate(aggregate_expr, self.model, alias,
+                is_summary=True)
+
+        return self.query.get_aggregation()
+
     def count(self):
         """
         Performs a SELECT COUNT() and returns the number of records as an
@@ -553,6 +583,25 @@ class QuerySet(object):
         """
         self.query.select_related = other.query.select_related
 
+    def annotate(self, *args, **kwargs):
+        """
+        Return a query set in which the returned objects have been annotated
+        with data aggregated from related fields.
+        """
+        for arg in args:
+            kwargs[arg.default_alias] = arg
+
+        obj = self._clone()
+
+        obj._setup_aggregate_query()
+
+        # Add the aggregates to the query
+        for (alias, aggregate_expr) in kwargs.items():
+            obj.query.add_aggregate(aggregate_expr, self.model, alias,
+                is_summary=False)
+
+        return obj
+
     def order_by(self, *field_names):
         """
         Returns a new QuerySet instance with the ordering changed.
@@ -641,6 +690,16 @@ class QuerySet(object):
         """
         pass
 
+    def _setup_aggregate_query(self):
+        """
+        Prepare the query for computing a result that contains aggregate annotations.
+        """
+        opts = self.model._meta
+        if not self.query.group_by:
+            field_names = [f.attname for f in opts.fields]
+            self.query.add_fields(field_names, False)
+            self.query.set_group_by()
+
     def as_sql(self):
         """
         Returns the internal query's SQL and parameters (as a tuple).
@@ -669,6 +728,8 @@ class ValuesQuerySet(QuerySet):
             len(self.field_names) != len(self.model._meta.fields)):
             self.query.trim_extra_select(self.extra_names)
         names = self.query.extra_select.keys() + self.field_names
+        names.extend(self.query.aggregate_select.keys())
+
         for row in self.query.results_iter():
             yield dict(zip(names, row))
 
@@ -682,20 +743,25 @@ class ValuesQuerySet(QuerySet):
         """
         self.query.clear_select_fields()
         self.extra_names = []
+        self.aggregate_names = []
+
         if self._fields:
-            if not self.query.extra_select:
+            if not self.query.extra_select and not self.query.aggregate_select:
                 field_names = list(self._fields)
             else:
                 field_names = []
                 for f in self._fields:
                     if self.query.extra_select.has_key(f):
                         self.extra_names.append(f)
+                    elif self.query.aggregate_select.has_key(f):
+                        self.aggregate_names.append(f)
                     else:
                         field_names.append(f)
         else:
             # Default to all fields.
             field_names = [f.attname for f in self.model._meta.fields]
 
+        self.query.select = []
         self.query.add_fields(field_names, False)
         self.query.default_cols = False
         self.field_names = field_names
@@ -711,6 +777,7 @@ class ValuesQuerySet(QuerySet):
             c._fields = self._fields[:]
         c.field_names = self.field_names
         c.extra_names = self.extra_names
+        c.aggregate_names = self.aggregate_names
         if setup and hasattr(c, '_setup_query'):
             c._setup_query()
         return c
@@ -718,10 +785,18 @@ class ValuesQuerySet(QuerySet):
     def _merge_sanity_check(self, other):
         super(ValuesQuerySet, self)._merge_sanity_check(other)
         if (set(self.extra_names) != set(other.extra_names) or
-                set(self.field_names) != set(other.field_names)):
+                set(self.field_names) != set(other.field_names) or
+                self.aggregate_names != other.aggregate_names):
             raise TypeError("Merging '%s' classes must involve the same values in each case."
                     % self.__class__.__name__)
 
+    def _setup_aggregate_query(self):
+        """
+        Prepare the query for computing a result that contains aggregate annotations.
+        """
+        self.query.set_group_by()
+
+        super(ValuesQuerySet, self)._setup_aggregate_query()
 
 class ValuesListQuerySet(ValuesQuerySet):
     def iterator(self):
@@ -729,14 +804,14 @@ class ValuesListQuerySet(ValuesQuerySet):
         if self.flat and len(self._fields) == 1:
             for row in self.query.results_iter():
                 yield row[0]
-        elif not self.query.extra_select:
+        elif not self.query.extra_select and not self.query.aggregate_select:
             for row in self.query.results_iter():
                 yield tuple(row)
         else:
             # When extra(select=...) is involved, the extra cols come are
             # always at the start of the row, so we need to reorder the fields
             # to match the order in self._fields.
-            names = self.query.extra_select.keys() + self.field_names
+            names = self.query.extra_select.keys() + self.field_names + self.query.aggregate_select.keys()
             for row in self.query.results_iter():
                 data = dict(zip(names, row))
                 yield tuple([data[f] for f in self._fields])

django/db/models/query_utils.py

@@ -64,4 +64,3 @@ def select_related_descend(field, restricted, requested):
     if not restricted and field.null:
         return False
     return True
-

django/db/models/sql/aggregates.py

+"""
+Classes to represent the default SQL aggregate functions
+"""
+
+class AggregateField(object):
+    """An internal field mockup used to identify aggregates in the
+    data-conversion parts of the database backend.
+    """
+    def __init__(self, internal_type):
+        self.internal_type = internal_type
+    def get_internal_type(self):
+        return self.internal_type
+
+ordinal_aggregate_field = AggregateField('IntegerField')
+computed_aggregate_field = AggregateField('FloatField')
+
+class Aggregate(object):
+    """
+    Default SQL Aggregate.
+    """
+    is_ordinal = False
+    is_computed = False
+    sql_template = '%(function)s(%(field)s)'
+
+    def __init__(self, col, source=None, is_summary=False, **extra):
+        """Instantiate an SQL aggregate
+
+         * col is a column reference describing the subject field
+           of the aggregate. It can be an alias, or a tuple describing
+           a table and column name.
+         * source is the underlying field or aggregate definition for
+           the column reference. If the aggregate is not an ordinal or
+           computed type, this reference is used to determine the coerced
+           output type of the aggregate.
+         * extra is a dictionary of additional data to provide for the
+           aggregate definition
+
+        Also utilizes the class variables:
+         * sql_function, the name of the SQL function that implements the
+           aggregate.
+         * sql_template, a template string that is used to render the
+           aggregate into SQL.
+         * is_ordinal, a boolean indicating if the output of this aggregate
+           is an integer (e.g., a count)
+         * is_computed, a boolean indicating if this output of this aggregate
+           is a computed float (e.g., an average), regardless of the input
+           type.
+
+        """
+        self.col = col
+        self.source = source
+        self.is_summary = is_summary
+        self.extra = extra
+
+        # Follow the chain of aggregate sources back until you find an
+        # actual field, or an aggregate that forces a particular output
+        # type. This type of this field will be used to coerce values
+        # retrieved from the database.
+        tmp = self
+
+        while tmp and isinstance(tmp, Aggregate):
+            if getattr(tmp, 'is_ordinal', False):
+                tmp = ordinal_aggregate_field
+            elif getattr(tmp, 'is_computed', False):
+                tmp = computed_aggregate_field
+            else:
+                tmp = tmp.source
+
+        self.field = tmp
+
+    def relabel_aliases(self, change_map):
+        if isinstance(self.col, (list, tuple)):
+            self.col = (change_map.get(self.col[0], self.col[0]), self.col[1])
+
+    def as_sql(self, quote_func=None):
+        "Return the aggregate, rendered as SQL."
+        if not quote_func:
+            quote_func = lambda x: x
+
+        if hasattr(self.col, 'as_sql'):
+            field_name = self.col.as_sql(quote_func)
+        elif isinstance(self.col, (list, tuple)):
+            field_name = '.'.join([quote_func(c) for c in self.col])
+        else:
+            field_name = self.col
+
+        params = {
+            'function': self.sql_function,
+            'field': field_name
+        }
+        params.update(self.extra)
+
+        return self.sql_template % params
+
+
+class Avg(Aggregate):
+    is_computed = True
+    sql_function = 'AVG'
+
+class Count(Aggregate):
+    is_ordinal = True
+    sql_function = 'COUNT'
+    sql_template = '%(function)s(%(distinct)s%(field)s)'
+
+    def __init__(self, col, distinct=False, **extra):
+        super(Count, self).__init__(col, distinct=distinct and 'DISTINCT ' or '', **extra)
+
+class Max(Aggregate):
+    sql_function = 'MAX'
+
+class Min(Aggregate):
+    sql_function = 'MIN'
+
+class StdDev(Aggregate):
+    is_computed = True
+
+    def __init__(self, col, sample=False, **extra):
+        super(StdDev, self).__init__(col, **extra)
+        self.sql_function = sample and 'STDDEV_SAMP' or 'STDDEV_POP'
+
+class Sum(Aggregate):
+    sql_function = 'SUM'
+
+class Variance(Aggregate):
+    is_computed = True
+
+    def __init__(self, col, sample=False, **extra):
+        super(Variance, self).__init__(col, **extra)
+        self.sql_function = sample and 'VAR_SAMP' or 'VAR_POP'
+

django/db/models/sql/datastructures.py

@@ -25,59 +25,6 @@ class RawValue(object):
     def __init__(self, value):
         self.value = value
 
-class Aggregate(object):
-    """
-    Base class for all aggregate-related classes (min, max, avg, count, sum).
-    """
-    def relabel_aliases(self, change_map):
-        """
-        Relabel the column alias, if necessary. Must be implemented by
-        subclasses.
-        """
-        raise NotImplementedError
-
-    def as_sql(self, quote_func=None):
-        """
-        Returns the SQL string fragment for this object.
-
-        The quote_func function is used to quote the column components. If
-        None, it defaults to doing nothing.
-
-        Must be implemented by subclasses.
-        """
-        raise NotImplementedError
-
-class Count(Aggregate):
-    """
-    Perform a count on the given column.
-    """
-    def __init__(self, col='*', distinct=False):
-        """
-        Set the column to count on (defaults to '*') and set whether the count
-        should be distinct or not.
-        """
-        self.col = col
-        self.distinct = distinct
-
-    def relabel_aliases(self, change_map):
-        c = self.col
-        if isinstance(c, (list, tuple)):
-            self.col = (change_map.get(c[0], c[0]), c[1])
-
-    def as_sql(self, quote_func=None):
-        if not quote_func:
-            quote_func = lambda x: x
-        if isinstance(self.col, (list, tuple)):
-            col = ('%s.%s' % tuple([quote_func(c) for c in self.col]))
-        elif hasattr(self.col, 'as_sql'):
-            col = self.col.as_sql(quote_func)
-        else:
-            col = self.col
-        if self.distinct:
-            return 'COUNT(DISTINCT %s)' % col
-        else:
-            return 'COUNT(%s)' % col
-
 class Date(object):
     """
     Add a date selection column.

django/db/models/sql/subqueries.py

@@ -9,7 +9,7 @@ from django.db.models.sql.query import Query
 from django.db.models.sql.where import AND, Constraint
 
 __all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'DateQuery',
-        'CountQuery']
+        'AggregateQuery']
 
 class DeleteQuery(Query):
     """
@@ -400,15 +400,25 @@ class DateQuery(Query):
         self.distinct = True
         self.order_by = order == 'ASC' and [1] or [-1]
 
-class CountQuery(Query):
+class AggregateQuery(Query):
     """
-    A CountQuery knows how to take a normal query which would select over
-    multiple distinct columns and turn it into SQL that can be used on a
-    variety of backends (it requires a select in the FROM clause).
+    An AggregateQuery takes another query as a parameter to the FROM
+    clause and only selects the elements in the provided list.
     """
-    def get_from_clause(self):
-        result, params = self._query.as_sql()
-        return ['(%s) A1' % result], params
+    def add_subquery(self, query):
+        self.subquery, self.sub_params = query.as_sql(with_col_aliases=True)
 
-    def get_ordering(self):
-        return ()
+    def as_sql(self, quote_func=None):
+        """
+        Creates the SQL for this query. Returns the SQL string and list of
+        parameters.
+        """
+        sql = ('SELECT %s FROM (%s) subquery' % (
+            ', '.join([
+                aggregate.as_sql()
+                for aggregate in self.aggregate_select.values()
+            ]),
+            self.subquery)
+        )
+        params = self.sub_params
+        return (sql, params)

django/test/testcases.py

@@ -14,6 +14,7 @@ from django.test.client import Client
 from django.utils import simplejson
 
 normalize_long_ints = lambda s: re.sub(r'(?<![\w])(\d+)L(?![\w])', '\\1', s)
+normalize_decimals = lambda s: re.sub(r"Decimal\('(\d+(\.\d*)?)'\)", lambda m: "Decimal(\"%s\")" % m.groups()[0], s)
 
 def to_list(value):
     """
@@ -31,7 +32,7 @@ class OutputChecker(doctest.OutputChecker):
     def check_output(self, want, got, optionflags):
         "The entry method for doctest output checking. Defers to a sequence of child checkers"
         checks = (self.check_output_default,
-                  self.check_output_long,
+                  self.check_output_numeric,
                   self.check_output_xml,
                   self.check_output_json)
         for check in checks:
@@ -43,19 +44,23 @@ class OutputChecker(doctest.OutputChecker):
         "The default comparator provided by doctest - not perfect, but good for most purposes"
         return doctest.OutputChecker.check_output(self, want, got, optionflags)
 
-    def check_output_long(self, want, got, optionflags):
-        """Doctest does an exact string comparison of output, which means long
-        integers aren't equal to normal integers ("22L" vs. "22"). The
-        following code normalizes long integers so that they equal normal
-        integers.
+    def check_output_numeric(self, want, got, optionflags):
+        """Doctest does an exact string comparison of output, which means that
+        some numerically equivalent values aren't equal. This check normalizes
+         * long integers (22L) so that they equal normal integers. (22)
+         * Decimals so that they are comparable, regardless of the change
+           made to __repr__ in Python 2.6.
         """
-        return normalize_long_ints(want) == normalize_long_ints(got)
+        return doctest.OutputChecker.check_output(self,
+            normalize_decimals(normalize_long_ints(want)),
+            normalize_decimals(normalize_long_ints(got)),
+            optionflags)
 
     def check_output_xml(self, want, got, optionsflags):
         """Tries to do a 'xml-comparision' of want and got.  Plain string
         comparision doesn't always work because, for example, attribute
         ordering should not be important.
-        
+
         Based on http://codespeak.net/svn/lxml/trunk/src/lxml/doctestcompare.py
         """
         _norm_whitespace_re = re.compile(r'[ \t\n][ \t\n]+')
@@ -102,7 +107,7 @@ class OutputChecker(doctest.OutputChecker):
             wrapper = '<root>%s</root>'
             want = wrapper % want
             got = wrapper % got
-            
+
         # Parse the want and got strings, and compare the parsings.
         try:
             want_root = parseString(want).firstChild
@@ -174,7 +179,7 @@ class TestCase(unittest.TestCase):
         """Performs any pre-test setup. This includes:
 
             * Flushing the database.
-            * If the Test Case class has a 'fixtures' member, installing the 
+            * If the Test Case class has a 'fixtures' member, installing the
               named fixtures.
             * If the Test Case class has a 'urls' member, replace the
               ROOT_URLCONF with it.

docs/index.txt

@@ -42,7 +42,7 @@ The model layer
     * **Models:** :ref:`Model syntax <topics-db-models>` | :ref:`Field types <ref-models-fields>` | :ref:`Meta options <ref-models-options>`
     * **QuerySets:** :ref:`Executing queries <topics-db-queries>` | :ref:`QuerySet method reference <ref-models-querysets>`
     * **Model instances:** :ref:`Instance methods <ref-models-instances>` | :ref:`Accessing related objects <ref-models-relations>`
-    * **Advanced:** :ref:`Managers <topics-db-managers>` | :ref:`Raw SQL <topics-db-sql>` | :ref:`Transactions <topics-db-transactions>` | :ref:`Custom fields <howto-custom-model-fields>`
+    * **Advanced:** :ref:`Managers <topics-db-managers>` | :ref:`Raw SQL <topics-db-sql>` | :ref:`Transactions <topics-db-transactions>` | :ref:`Aggregation <topics-db-aggregation>` | :ref:`Custom fields <howto-custom-model-fields>`
     * **Other:** :ref:`Supported databases <ref-databases>` | :ref:`Legacy databases <howto-legacy-databases>` | :ref:`Providing initial data <howto-initial-data>`
 
 The template layer

docs/ref/models/index.txt

@@ -7,7 +7,7 @@ Model API reference. For introductory material, see :ref:`topics-db-models`.
 
 .. toctree::
    :maxdepth: 1
-   
+
    fields
    relations
    options

docs/ref/models/querysets.txt

@@ -158,6 +158,48 @@ In SQL terms, that evaluates to::
 
 Note the second example is more restrictive.
 
+``annotate(*args, **kwargs)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.1
+
+Annotates each object in the ``QuerySet`` with the provided list of
+aggregate values (averages, sums, etc) that have been computed over
+the objects that are related to the objects in the ``QuerySet``.
+Each argument to ``annotate()`` is an annotation that will be added
+to each object in the ``QuerySet`` that is returned.
+
+The aggregation functions that are provided by Django are described
+in `Aggregation Functions`_ below.
+
+Annotations specified using keyword arguments will use the keyword as
+the alias for the annotation. Anonymous arguments will have an alias
+generated for them based upon the name of the aggregate function and
+the model field that is being aggregated.
+
+For example, if you were manipulating a list of blogs, you may want
+to determine how many entries have been made in each blog::
+
+    >>> q = Blog.objects.annotate(Count('entry'))
+    # The name of the first blog
+    >>> q[0].name
+    'Blogasaurus'
+    # The number of entries on the first blog
+    >>> q[0].entry__count
+    42
+
+The ``Blog`` model doesn't define an ``entry_count`` attribute by itself,
+but by using a keyword argument to specify the aggregate function, you can
+control the name of the annotation::
+
+    >>> q = Blog.objects.annotate(number_of_entries=Count('entry'))
+    # The number of entries on the first blog, using the name provided
+    >>> q[0].number_of_entries
+    42
+
+For an in-depth discussion of aggregation, see :ref:`the topic guide on
+Aggregation <topics-db-aggregation>`.
+
 ``order_by(*fields)``
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -931,6 +973,38 @@ exist with the given parameters.
 
 Note ``latest()`` exists purely for convenience and readability.
 
+``aggregate(*args, **kwargs)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.1
+
+Returns a dictionary of aggregate values (averages, sums, etc) calculated
+over the ``QuerySet``. Each argument to ``aggregate()`` specifies
+a value that will be included in the dictionary that is returned.
+
+The aggregation functions that are provided by Django are described
+in `Aggregation Functions`_ below.
+
+Aggregates specified using keyword arguments will use the keyword as
+the name for the annotation. Anonymous arguments will have an name
+generated for them based upon the name of the aggregate function and
+the model field that is being aggregated.
+
+For example, if you were manipulating blog entries, you may want to know
+the average number of authors contributing to blog entries::
+
+    >>> q = Blog.objects.aggregate(Count('entry'))
+    {'entry__count': 16}
+
+By using a keyword argument to specify the aggregate function, you can
+control the name of the aggregation value that is returned::
+
+    >>> q = Blog.objects.aggregate(number_of_entries=Count('entry'))
+    {'number_of_entries': 2.34}
+
+For an in-depth discussion of aggregation, see :ref:`the topic guide on
+Aggregation <topics-db-aggregation>`.
+
 .. _field-lookups:
 
 Field lookups
@@ -1326,3 +1400,115 @@ SQL equivalents::
 
     SELECT ... WHERE title REGEXP '(?i)^(an?|the) +'; -- SQLite
 
+.. _aggregation-functions:
+
+Aggregation Functions
+---------------------
+
+.. versionadded:: 1.1
+
+Django provides the following aggregation functions in the
+``django.db.models`` module.
+
+``Avg``
+~~~~~~~
+
+.. class:: Avg(field)
+
+Returns the mean value of the given field.
+
+    * Default alias: ``<field>__avg``
+    * Return type: float
+
+``Count``
+~~~~~~~~~
+
+.. class:: Count(field, distinct=False)
+
+Returns the number of objects that are related through the provided field.
+
+    * Default alias: ``<field>__count``
+    * Return type: integer
+
+Has one optional argument:
+
+.. attribute:: distinct
+
+    If distinct=True, the count will only include unique instances. This has
+    the SQL equivalent of ``COUNT(DISTINCT field)``. Default value is ``False``.
+
+``Max``
+~~~~~~~
+
+.. class:: Max(field)
+
+Returns the maximum value of the given field.
+
+    * Default alias: ``<field>__max``
+    * Return type: same as input field
+
+``Min``
+~~~~~~~
+
+.. class:: Min(field)
+
+Returns the minimum value of the given field.
+
+    * Default alias: ``<field>__min``
+    * Return type: same as input field
+
+``StdDev``
+~~~~~~~~~
+
+.. class:: StdDev(field, sample=False)
+
+Returns the standard deviation of the data in the provided field.
+
+    * Default alias: ``<field>__stddev``
+    * Return type: float
+
+Has one optional argument:
+
+.. attribute:: sample
+
+    By default, ``StdDev`` returns the population standard deviation. However,
+    if ``sample=True``, the return value will be the sample standard deviation.
+
+.. admonition:: SQLite
+
+    SQLite doesn't provide ``StdDev`` out of the box. An implementation is
+    available as an extension module for SQLite. Consult the SQlite
+    documentation for instructions on obtaining and installing this extension.
+
+``Sum``
+~~~~~~~
+
+.. class:: Sum(field)
+
+Computes the sum of all values of the given field.
+
+    * Default alias: ``<field>__sum``
+    * Return type: same as input field
+
+``Variance``
+~~~~~~~~~
+
+.. class:: Variance(field, sample=False)
+
+Returns the variance of the data in the provided field.
+
+    * Default alias: ``<field>__variance``
+    * Return type: float
+
+Has one optional argument:
+
+.. attribute:: sample
+
+    By default, ``Variance`` returns the population variance. However,
+    if ``sample=True``, the return value will be the sample variance.
+
+.. admonition:: SQLite
+
+    SQLite doesn't provide ``Variance`` out of the box. An implementation is
+    available as an extension module for SQLite. Consult the SQlite
+    documentation for instructions on obtaining and installing this extension.

docs/topics/db/index.txt

@@ -12,6 +12,7 @@ model maps to a single database table.
 
    models
    queries
+   aggregation
    managers
    sql
    transactions

tests/modeltests/aggregation/fixtures/initial_data.json

+[
+    {
+        "pk": 1,
+        "model": "aggregation.publisher",
+        "fields": {
+            "name": "Apress",
+            "num_awards": 3
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation.publisher",
+        "fields": {
+            "name": "Sams",
+            "num_awards": 1
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation.publisher",
+        "fields": {
+            "name": "Prentice Hall",
+            "num_awards": 7
+        }
+    },
+    {
+        "pk": 4,
+        "model": "aggregation.publisher",
+        "fields": {
+            "name": "Morgan Kaufmann",
+            "num_awards": 9
+        }
+    },
+    {
+        "pk": 1,
+        "model": "aggregation.book",
+        "fields": {
+            "publisher": 1,
+            "isbn": "159059725",
+            "name": "The Definitive Guide to Django: Web Development Done Right",
+            "price": "30.00",
+            "rating": 4.5,
+            "authors": [1, 2],
+            "pages": 447,
+            "pubdate": "2007-12-6"
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation.book",
+        "fields": {
+            "publisher": 2,
+            "isbn": "067232959",
+            "name": "Sams Teach Yourself Django in 24 Hours",
+            "price": "23.09",
+            "rating": 3.0,
+            "authors": [3],
+            "pages": 528,
+            "pubdate": "2008-3-3"
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation.book",
+        "fields": {
+            "publisher": 1,
+            "isbn": "159059996",
+            "name": "Practical Django Projects",
+            "price": "29.69",
+            "rating": 4.0,
+            "authors": [4],
+            "pages": 300,
+            "pubdate": "2008-6-23"
+        }
+    },
+    {
+        "pk": 4,
+        "model": "aggregation.book",
+        "fields": {
+            "publisher": 3,
+            "isbn": "013235613",
+            "name": "Python Web Development with Django",
+            "price": "29.69",
+            "rating": 4.0,
+            "authors": [5, 6, 7],
+            "pages": 350,
+            "pubdate": "2008-11-3"
+        }
+    },
+    {
+        "pk": 5,
+        "model": "aggregation.book",
+        "fields": {
+            "publisher": 3,
+            "isbn": "013790395",
+            "name": "Artificial Intelligence: A Modern Approach",
+            "price": "82.80",
+            "rating": 4.0,
+            "authors": [8, 9],
+            "pages": 1132,
+            "pubdate": "1995-1-15"
+        }
+    },
+    {
+        "pk": 6,
+        "model": "aggregation.book",
+        "fields": {
+            "publisher": 4,
+            "isbn": "155860191",
+            "name": "Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp",
+            "price": "75.00",
+            "rating": 5.0,
+            "authors": [8],
+            "pages": 946,
+            "pubdate": "1991-10-15"
+        }
+    },
+    {
+        "pk": 1,
+        "model": "aggregation.store",
+        "fields": {
+            "books": [1, 2, 3, 4, 5, 6],
+            "name": "Amazon.com",
+            "original_opening": "1994-4-23 9:17:42",
+            "friday_night_closing": "23:59:59"
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation.store",
+        "fields": {
+            "books": [1, 3, 5, 6],
+            "name": "Books.com",
+            "original_opening": "2001-3-15 11:23:37",
+            "friday_night_closing": "23:59:59"
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation.store",
+        "fields": {
+            "books": [3, 4, 6],
+            "name": "Mamma and Pappa's Books",
+            "original_opening": "1945-4-25 16:24:14",
+            "friday_night_closing": "21:30:00"
+        }
+    },
+    {
+        "pk": 1,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 34,
+            "friends": [2, 4],
+            "name": "Adrian Holovaty"
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 35,
+            "friends": [1, 7],
+            "name": "Jacob Kaplan-Moss"
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 45,
+            "friends": [],
+            "name": "Brad Dayley"
+        }
+    },
+    {
+        "pk": 4,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 29,
+            "friends": [1],
+            "name": "James Bennett"
+        }
+    },
+    {
+        "pk": 5,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 37,
+            "friends": [6, 7],
+            "name": "Jeffrey Forcier "
+        }
+    },
+    {
+        "pk": 6,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 29,
+            "friends": [5, 7],
+            "name": "Paul Bissex"
+        }
+    },
+    {
+        "pk": 7,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 25,
+            "friends": [2, 5, 6],
+            "name": "Wesley J. Chun"
+        }
+    },
+    {
+        "pk": 8,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 57,
+            "friends": [9],
+            "name": "Peter Norvig"
+        }
+    },
+    {
+        "pk": 9,
+        "model": "aggregation.author",
+        "fields": {
+            "age": 46,
+            "friends": [8],
+            "name": "Stuart Russell"
+        }
+    }
+]

tests/regressiontests/aggregation_regress/fixtures/initial_data.json

+[
+    {
+        "pk": 1,
+        "model": "aggregation_regress.publisher",
+        "fields": {
+            "name": "Apress",
+            "num_awards": 3
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation_regress.publisher",
+        "fields": {
+            "name": "Sams",
+            "num_awards": 1
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation_regress.publisher",
+        "fields": {
+            "name": "Prentice Hall",
+            "num_awards": 7
+        }
+    },
+    {
+        "pk": 4,
+        "model": "aggregation_regress.publisher",
+        "fields": {
+            "name": "Morgan Kaufmann",
+            "num_awards": 9
+        }
+    },
+    {
+        "pk": 1,
+        "model": "aggregation_regress.book",
+        "fields": {
+            "publisher": 1,
+            "isbn": "159059725",
+            "name": "The Definitive Guide to Django: Web Development Done Right",
+            "price": "30.00",
+            "rating": 4.5,
+            "authors": [1, 2],
+            "pages": 447,
+            "pubdate": "2007-12-6"
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation_regress.book",
+        "fields": {
+            "publisher": 2,
+            "isbn": "067232959",
+            "name": "Sams Teach Yourself Django in 24 Hours",
+            "price": "23.09",
+            "rating": 3.0,
+            "authors": [3],
+            "pages": 528,
+            "pubdate": "2008-3-3"
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation_regress.book",
+        "fields": {
+            "publisher": 1,
+            "isbn": "159059996",
+            "name": "Practical Django Projects",
+            "price": "29.69",
+            "rating": 4.0,
+            "authors": [4],
+            "pages": 300,
+            "pubdate": "2008-6-23"
+        }
+    },
+    {
+        "pk": 4,
+        "model": "aggregation_regress.book",
+        "fields": {
+            "publisher": 3,
+            "isbn": "013235613",
+            "name": "Python Web Development with Django",
+            "price": "29.69",
+            "rating": 4.0,
+            "authors": [5, 6, 7],
+            "pages": 350,
+            "pubdate": "2008-11-3"
+        }
+    },
+    {
+        "pk": 5,
+        "model": "aggregation_regress.book",
+        "fields": {
+            "publisher": 3,
+            "isbn": "013790395",
+            "name": "Artificial Intelligence: A Modern Approach",
+            "price": "82.80",
+            "rating": 4.0,
+            "authors": [8, 9],
+            "pages": 1132,
+            "pubdate": "1995-1-15"
+        }
+    },
+    {
+        "pk": 6,
+        "model": "aggregation_regress.book",
+        "fields": {
+            "publisher": 4,
+            "isbn": "155860191",
+            "name": "Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp",
+            "price": "75.00",
+            "rating": 5.0,
+            "authors": [8],
+            "pages": 946,
+            "pubdate": "1991-10-15"
+        }
+    },
+    {
+        "pk": 1,
+        "model": "aggregation_regress.store",
+        "fields": {
+            "books": [1, 2, 3, 4, 5, 6],
+            "name": "Amazon.com",
+            "original_opening": "1994-4-23 9:17:42",
+            "friday_night_closing": "23:59:59"
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation_regress.store",
+        "fields": {
+            "books": [1, 3, 5, 6],
+            "name": "Books.com",
+            "original_opening": "2001-3-15 11:23:37",
+            "friday_night_closing": "23:59:59"
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation_regress.store",
+        "fields": {
+            "books": [3, 4, 6],
+            "name": "Mamma and Pappa's Books",
+            "original_opening": "1945-4-25 16:24:14",
+            "friday_night_closing": "21:30:00"
+        }
+    },
+    {
+        "pk": 1,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 34,
+            "friends": [2, 4],
+            "name": "Adrian Holovaty"
+        }
+    },
+    {
+        "pk": 2,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 35,
+            "friends": [1, 7],
+            "name": "Jacob Kaplan-Moss"
+        }
+    },
+    {
+        "pk": 3,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 45,
+            "friends": [],
+            "name": "Brad Dayley"
+        }
+    },
+    {
+        "pk": 4,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 29,
+            "friends": [1],
+            "name": "James Bennett"
+        }
+    },
+    {
+        "pk": 5,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 37,
+            "friends": [6, 7],
+            "name": "Jeffrey Forcier "
+        }
+    },
+    {
+        "pk": 6,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 29,
+            "friends": [5, 7],
+            "name": "Paul Bissex"
+        }
+    },
+    {
+        "pk": 7,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 25,
+            "friends": [2, 5, 6],
+            "name": "Wesley J. Chun"
+        }
+    },
+    {
+        "pk": 8,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 57,
+            "friends": [9],
+            "name": "Peter Norvig"
+        }
+    },
+    {
+        "pk": 9,
+        "model": "aggregation_regress.author",
+        "fields": {
+            "age": 46,
+            "friends": [8],
+            "name": "Stuart Russell"
+        }
+    }
+]

tests/regressiontests/aggregation_regress/models.py

+# coding: utf-8
+from django.db import models
+from django.conf import settings
+
+try:
+    sorted
+except NameError:
+    from django.utils.itercompat import sorted      # For Python 2.3
+
+class Author(models.Model):
+   name = models.CharField(max_length=100)
+   age = models.IntegerField()
+   friends = models.ManyToManyField('self', blank=True)
+
+   def __unicode__(self):
+      return self.name
+
+class Publisher(models.Model):
+   name = models.CharField(max_length=300)
+   num_awards = models.IntegerField()
+
+   def __unicode__(self):
+      return self.name
+
+class Book(models.Model):
+   isbn = models.CharField(max_length=9)
+   name = models.CharField(max_length=300)
+   pages = models.IntegerField()
+   rating = models.FloatField()
+   price = models.DecimalField(decimal_places=2, max_digits=6)
+   authors = models.ManyToManyField(Author)
+   publisher = models.ForeignKey(Publisher)
+   pubdate = models.DateField()
+
+   class Meta:
+       ordering = ('name',)
+
+   def __unicode__(self):
+      return self.name
+
+class Store(models.Model):
+   name = models.CharField(max_length=300)
+   books = models.ManyToManyField(Book)
+   original_opening = models.DateTimeField()
+   friday_night_closing = models.TimeField()
+
+   def __unicode__(self):
+      return self.name
+
+#Extra does not play well with values. Modify the tests if/when this is fixed.
+__test__ = {'API_TESTS': """
+>>> from django.core import management
+>>> from django.db.models import get_app
+
+# Reset the database representation of this app.
+# This will return the database to a clean initial state.
+>>> management.call_command('flush', verbosity=0, interactive=False)
+
+>>> from django.db.models import Avg, Sum, Count, Max, Min, StdDev, Variance
+
+# Ordering requests are ignored
+>>> Author.objects.all().order_by('name').aggregate(Avg('age'))
+{'age__avg': 37.4...}
+
+# Implicit ordering is also ignored
+>>> Book.objects.all().aggregate(Sum('pages'))
+{'pages__sum': 3703}
+
+# Baseline results
+>>> Book.objects.all().aggregate(Sum('pages'), Avg('pages'))
+{'pages__sum': 3703, 'pages__avg': 617.1...}
+
+# Empty values query doesn't affect grouping or results
+>>> Book.objects.all().values().aggregate(Sum('pages'), Avg('pages'))
+{'pages__sum': 3703, 'pages__avg': 617.1...}
+
+# Aggregate overrides extra selected column
+>>> Book.objects.all().extra(select={'price_per_page' : 'price / pages'}).aggregate(Sum('pages'))
+{'pages__sum': 3703}
+
+# Annotations get combined with extra select clauses
+>>> sorted(Book.objects.all().annotate(mean_auth_age=Avg('authors__age')).extra(select={'manufacture_cost' : 'price * .5'}).get(pk=2).__dict__.items())
+[('id', 2), ('isbn', u'067232959'), ('manufacture_cost', ...11.545...), ('mean_auth_age', 45.0), ('name', u'Sams Teach Yourself Django in 24 Hours'), ('pages', 528), ('price', Decimal("23.09")), ('pubdate', datetime.date(2008, 3, 3)), ('publisher_id', 2), ('rating', 3.0)]
+
+# Order of the annotate/extra in the query doesn't matter
+>>> sorted(Book.objects.all().extra(select={'manufacture_cost' : 'price * .5'}).annotate(mean_auth_age=Avg('authors__age')).get(pk=2).__dict__.items())
+[('id', 2), ('isbn', u'067232959'), ('manufacture_cost', ...11.545...), ('mean_auth_age', 45.0), ('name', u'Sams Teach Yourself Django in 24 Hours'), ('pages', 528), ('price', Decimal("23.09")), ('pubdate', datetime.date(2008, 3, 3)), ('publisher_id', 2), ('rating', 3.0)]
+
+# Values queries can be combined with annotate and extra
+>>> sorted(Book.objects.all().annotate(mean_auth_age=Avg('authors__age')).extra(select={'manufacture_cost' : 'price * .5'}).values().get(pk=2).items())
+[('id', 2), ('isbn', u'067232959'), ('manufacture_cost', ...11.545...), ('mean_auth_age', 45.0), ('name', u'Sams Teach Yourself Django in 24 Hours'), ('pages', 528), ('price', Decimal("23.09")), ('pubdate', datetime.date(2008, 3, 3)), ('publisher_id', 2), ('rating', 3.0)]
+
+# The order of the values, annotate and extra clauses doesn't matter
+>>> sorted(Book.objects.all().values().annotate(mean_auth_age=Avg('authors__age')).extra(select={'manufacture_cost' : 'price * .5'}).get(pk=2).items())
+[('id', 2), ('isbn', u'067232959'), ('manufacture_cost', ...11.545...), ('mean_auth_age', 45.0), ('name', u'Sams Teach Yourself Django in 24 Hours'), ('pages', 528), ('price', Decimal("23.09")), ('pubdate', datetime.date(2008, 3, 3)), ('publisher_id', 2), ('rating', 3.0)]
+
+# A values query that selects specific columns reduces the output
+>>> sorted(Book.objects.all().annotate(mean_auth_age=Avg('authors__age')).extra(select={'price_per_page' : 'price / pages'}).values('name').get(pk=1).items())
+[('mean_auth_age', 34.5), ('name', u'The Definitive Guide to Django: Web Development Done Right')]
+
+# The annotations are added to values output if values() precedes annotate()
+>>> sorted(Book.objects.all().values('name').annotate(mean_auth_age=Avg('authors__age')).extra(select={'price_per_page' : 'price / pages'}).get(pk=1).items())
+[('mean_auth_age', 34.5), ('name', u'The Definitive Guide to Django: Web Development Done Right')]
+
+# Check that all of the objects are getting counted (allow_nulls) and that values respects the amount of objects
+>>> len(Author.objects.all().annotate(Avg('friends__age')).values())
+9
+
+# Check that consecutive calls to annotate accumulate in the query
+>>> Book.objects.values('price').annotate(oldest=Max('authors__age')).order_by('oldest', 'price').annotate(Max('publisher__num_awards'))
+[{'price': Decimal("30..."), 'oldest': 35, 'publisher__num_awards__max': 3}, {'price': Decimal("29.69"), 'oldest': 37, 'publisher__num_awards__max': 7}, {'price': Decimal("23.09"), 'oldest': 45, 'publisher__num_awards__max': 1}, {'price': Decimal("75..."), 'oldest': 57, 'publisher__num_awards__max': 9}, {'price': Decimal("82.8..."), 'oldest': 57, 'publisher__num_awards__max': 7}]
+
+# Aggregates can be composed over annotations.
+# The return type is derived from the composed aggregate
+>>> Book.objects.all().annotate(num_authors=Count('authors__id')).aggregate(Max('pages'), Max('price'), Sum('num_authors'), Avg('num_authors'))
+{'num_authors__sum': 10, 'num_authors__avg': 1.66..., 'pages__max': 1132, 'price__max': Decimal("82.80")}
+
+# Bad field requests in aggregates are caught and reported
+>>> Book.objects.all().aggregate(num_authors=Count('foo'))
+Traceback (most recent call last):
+...
+FieldError: Cannot resolve keyword 'foo' into field. Choices are: authors, id, isbn, name, pages, price, pubdate, publisher, rating, store
+
+>>> Book.objects.all().annotate(num_authors=Count('foo'))
+Traceback (most recent call last):
+...
+FieldError: Cannot resolve keyword 'foo' into field. Choices are: authors, id, isbn, name, pages, price, pubdate, publisher, rating, store
+
+>>> Book.objects.all().annotate(num_authors=Count('authors__id')).aggregate(Max('foo'))
+Traceback (most recent call last):
+...
+FieldError: Cannot resolve keyword 'foo' into field. Choices are: authors, id, isbn, name, pages, price, pubdate, publisher, rating, store, num_authors
+
+# Old-style count aggregations can be mixed with new-style
+>>> Book.objects.annotate(num_authors=Count('authors')).count()
+6
+
+# Non-ordinal, non-computed Aggregates over annotations correctly inherit
+# the annotation's internal type if the annotation is ordinal or computed
+>>> Book.objects.annotate(num_authors=Count('authors')).aggregate(Max('num_authors'))
+{'num_authors__max': 3}
+
+>>> Publisher.objects.annotate(avg_price=Avg('book__price')).aggregate(Max('avg_price'))
+{'avg_price__max': 75.0...}
+
+# Aliases are quoted to protected aliases that might be reserved names
+>>> Book.objects.aggregate(number=Max('pages'), select=Max('pages'))
+{'number': 1132, 'select': 1132}
+
+
+"""
+}
+
+if settings.DATABASE_ENGINE != 'sqlite3':
+    __test__['API_TESTS'] += """
+# Stddev and Variance are not guaranteed to be available for SQLite.
+
+>>> Book.objects.aggregate(StdDev('pages'))
+{'pages__stddev': 311.46...}
+
+>>> Book.objects.aggregate(StdDev('rating'))
+{'rating__stddev': 0.60...}
+
+>>> Book.objects.aggregate(StdDev('price'))
+{'price__stddev': 24.16...}
+
+
+>>> Book.objects.aggregate(StdDev('pages', sample=True))
+{'pages__stddev': 341.19...}
+
+>>> Book.objects.aggregate(StdDev('rating', sample=True))
+{'rating__stddev': 0.66...}
+
+>>> Book.objects.aggregate(StdDev('price', sample=True))
+{'price__stddev': 26.46...}
+
+
+>>> Book.objects.aggregate(Variance('pages'))
+{'pages__variance': 97010.80...}
+
+>>> Book.objects.aggregate(Variance('rating'))
+{'rating__variance': 0.36...}
+
+>>> Book.objects.aggregate(Variance('price'))
+{'price__variance': 583.77...}
+
+
+>>> Book.objects.aggregate(Variance('pages', sample=True))
+{'pages__variance': 116412.96...}
+
+>>> Book.objects.aggregate(Variance('rating', sample=True))
+{'rating__variance': 0.44...}
+
+>>> Book.objects.aggregate(Variance('price', sample=True))
+{'price__variance': 700.53...}
+
+
+"""
+