shelve.py 8.04 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
"""Manage shelves of pickled objects.

A "shelf" is a persistent, dictionary-like object.  The difference
with dbm databases is that the values (not the keys!) in a shelf can
be essentially arbitrary Python objects -- anything that the "pickle"
module can handle.  This includes most class instances, recursive data
types, and objects containing lots of shared sub-objects.  The keys
are ordinary strings.

To summarize the interface (key is a string, data is an arbitrary
object):

13 14
        import shelve
        d = shelve.open(filename) # open, with (g)dbm filename -- no suffix
15

16 17
        d[key] = data   # store data at key (overwrites old data if
                        # using an existing key)
Tim Peters's avatar
Tim Peters committed
18
        data = d[key]   # retrieve a COPY of the data at key (raise
19 20
                        # KeyError if no such key) -- NOTE that this
                        # access returns a *copy* of the entry!
21 22
        del d[key]      # delete data stored at key (raises KeyError
                        # if no such key)
23
        flag = key in d # true if the key exists
24
        list = d.keys() # a list of all existing keys (slow!)
25

26
        d.close()       # close it
27 28 29

Dependent on the implementation, closing a persistent dictionary may
or may not be necessary to flush changes to disk.
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

Normally, d[key] returns a COPY of the entry.  This needs care when
mutable entries are mutated: for example, if d[key] is a list,
        d[key].append(anitem)
does NOT modify the entry d[key] itself, as stored in the persistent
mapping -- it only modifies the copy, which is then immediately
discarded, so that the append has NO effect whatsoever.  To append an
item to d[key] in a way that will affect the persistent mapping, use:
        data = d[key]
        data.append(anitem)
        d[key] = data

To avoid the problem with mutable entries, you may pass the keyword
argument writeback=True in the call to shelve.open.  When you use:
        d = shelve.open(filename, writeback=True)
then d keeps a cache of all entries you access, and writes them all back
to the persistent mapping when you call d.close().  This ensures that
such usage as d[key].append(anitem) works as intended.

However, using keyword argument writeback=True may consume vast amount
of memory for the cache, and it may make d.close() very slow, if you
access many of d's entries after opening it in this way: d has no way to
check which of the entries you access are mutable and/or which ones you
actually mutate, so it must cache, and write back at close, all of the
entries that you access.  You can call d.sync() to write back all the
entries in the cache, and empty the cache (d.sync() also synchronizes
the persistent dictionary on disk, if feasible).
57
"""
58

59
from pickle import Pickler, Unpickler
60
from io import BytesIO
61

62
import collections
63

64
__all__ = ["Shelf","BsdDbShelf","DbfilenameShelf","open"]
65

66 67 68 69 70 71 72 73 74 75
class _ClosedDict(collections.MutableMapping):
    'Marker for a closed dict.  Access attempts raise a ValueError.'

    def closed(self, *args):
        raise ValueError('invalid operation on closed shelf')
    __iter__ = __len__ = __getitem__ = __setitem__ = __delitem__ = keys = closed

    def __repr__(self):
        return '<Closed Dictionary>'

76
class Shelf(collections.MutableMapping):
Tim Peters's avatar
Tim Peters committed
77 78 79 80 81 82
    """Base class for shelf implementations.

    This is initialized with a dictionary-like object.
    See the module's __doc__ string for an overview of the interface.
    """

83 84
    def __init__(self, dict, protocol=None, writeback=False,
                 keyencoding="utf-8"):
Tim Peters's avatar
Tim Peters committed
85
        self.dict = dict
86
        if protocol is None:
87
            protocol = 3
88 89 90
        self._protocol = protocol
        self.writeback = writeback
        self.cache = {}
91
        self.keyencoding = "utf-8"
Tim Peters's avatar
Tim Peters committed
92

93
    def __iter__(self):
94 95
        for k in self.dict.keys():
            yield k.decode(self.keyencoding)
Tim Peters's avatar
Tim Peters committed
96 97 98 99

    def __len__(self):
        return len(self.dict)

100
    def __contains__(self, key):
101
        return key.encode(self.keyencoding) in self.dict
102

Tim Peters's avatar
Tim Peters committed
103
    def get(self, key, default=None):
104
        if key.encode(self.keyencoding) in self.dict:
Tim Peters's avatar
Tim Peters committed
105 106 107 108
            return self[key]
        return default

    def __getitem__(self, key):
109 110 111
        try:
            value = self.cache[key]
        except KeyError:
112
            f = BytesIO(self.dict[key.encode(self.keyencoding)])
113 114 115 116
            value = Unpickler(f).load()
            if self.writeback:
                self.cache[key] = value
        return value
Tim Peters's avatar
Tim Peters committed
117 118

    def __setitem__(self, key, value):
119 120
        if self.writeback:
            self.cache[key] = value
121
        f = BytesIO()
122
        p = Pickler(f, self._protocol)
Tim Peters's avatar
Tim Peters committed
123
        p.dump(value)
124
        self.dict[key.encode(self.keyencoding)] = f.getvalue()
Tim Peters's avatar
Tim Peters committed
125 126

    def __delitem__(self, key):
127
        del self.dict[key.encode(self.keyencoding)]
128 129 130 131
        try:
            del self.cache[key]
        except KeyError:
            pass
Tim Peters's avatar
Tim Peters committed
132 133

    def close(self):
134
        self.sync()
Tim Peters's avatar
Tim Peters committed
135 136
        try:
            self.dict.close()
137
        except AttributeError:
Tim Peters's avatar
Tim Peters committed
138
            pass
139 140 141
        # Catch errors that may happen when close is called from __del__
        # because CPython is in interpreter shutdown.
        try:
142
            self.dict = _ClosedDict()
143 144
        except (NameError, TypeError):
            self.dict = None
Tim Peters's avatar
Tim Peters committed
145 146

    def __del__(self):
147 148 149
        if not hasattr(self, 'writeback'):
            # __init__ didn't succeed, so don't bother closing
            return
Tim Peters's avatar
Tim Peters committed
150 151 152
        self.close()

    def sync(self):
153 154
        if self.writeback and self.cache:
            self.writeback = False
155
            for key, entry in self.cache.items():
156 157 158
                self[key] = entry
            self.writeback = True
            self.cache = {}
Tim Peters's avatar
Tim Peters committed
159 160 161
        if hasattr(self.dict, 'sync'):
            self.dict.sync()

162

163
class BsdDbShelf(Shelf):
Tim Peters's avatar
Tim Peters committed
164
    """Shelf implementation using the "BSD" db interface.
165

Tim Peters's avatar
Tim Peters committed
166 167
    This adds methods first(), next(), previous(), last() and
    set_location() that have no counterpart in [g]dbm databases.
168

Tim Peters's avatar
Tim Peters committed
169 170 171
    The actual database must be opened using one of the "bsddb"
    modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or
    bsddb.rnopen) and passed to the constructor.
172

Tim Peters's avatar
Tim Peters committed
173 174
    See the module's __doc__ string for an overview of the interface.
    """
175

176 177 178
    def __init__(self, dict, protocol=None, writeback=False,
                 keyencoding="utf-8"):
        Shelf.__init__(self, dict, protocol, writeback, keyencoding)
179

Tim Peters's avatar
Tim Peters committed
180 181
    def set_location(self, key):
        (key, value) = self.dict.set_location(key)
182
        f = BytesIO(value)
183
        return (key.decode(self.keyencoding), Unpickler(f).load())
184

Tim Peters's avatar
Tim Peters committed
185
    def next(self):
186
        (key, value) = next(self.dict)
187
        f = BytesIO(value)
188
        return (key.decode(self.keyencoding), Unpickler(f).load())
189

Tim Peters's avatar
Tim Peters committed
190 191
    def previous(self):
        (key, value) = self.dict.previous()
192
        f = BytesIO(value)
193
        return (key.decode(self.keyencoding), Unpickler(f).load())
194

Tim Peters's avatar
Tim Peters committed
195 196
    def first(self):
        (key, value) = self.dict.first()
197
        f = BytesIO(value)
198
        return (key.decode(self.keyencoding), Unpickler(f).load())
199

Tim Peters's avatar
Tim Peters committed
200 201
    def last(self):
        (key, value) = self.dict.last()
202
        f = BytesIO(value)
203
        return (key.decode(self.keyencoding), Unpickler(f).load())
204 205 206


class DbfilenameShelf(Shelf):
207
    """Shelf implementation using the "dbm" generic dbm interface.
Tim Peters's avatar
Tim Peters committed
208 209 210 211

    This is initialized with the filename for the dbm database.
    See the module's __doc__ string for an overview of the interface.
    """
212

213
    def __init__(self, filename, flag='c', protocol=None, writeback=False):
214 215
        import dbm
        Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
216

217

218
def open(filename, flag='c', protocol=None, writeback=False):
Tim Peters's avatar
Tim Peters committed
219 220
    """Open a persistent dictionary for reading and writing.

221 222 223 224
    The filename parameter is the base filename for the underlying
    database.  As a side-effect, an extension may be added to the
    filename and more than one file may be created.  The optional flag
    parameter has the same interpretation as the flag parameter of
225
    dbm.open(). The optional protocol parameter specifies the
226 227
    version of the pickle protocol (0, 1, or 2).

Tim Peters's avatar
Tim Peters committed
228 229
    See the module's __doc__ string for an overview of the interface.
    """
230

231
    return DbfilenameShelf(filename, flag, protocol, writeback)