dumb.py 10.3 KB
Newer Older
Guido van Rossum's avatar
Guido van Rossum committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
"""A dumb and slow but simple dbm clone.

For database spam, spam.dir contains the index (a text file),
spam.bak *may* contain a backup of the index (also a text file),
while spam.dat contains the data (a binary file).

XXX TO DO:

- seems to contain a bug when updating...

- reclaim free space (currently, space once occupied by deleted or expanded
items is never reused)

- support concurrent access (currently, if two processes take turns making
updates, they can mess up the index)

- support efficient access to large databases (currently, the whole index
is read when the database is opened, and some updates rewrite the whole index)

- support opening for read-only (flag = 'm')

"""

24
import io as _io
25
import os as _os
26
import collections
Guido van Rossum's avatar
Guido van Rossum committed
27

28 29
__all__ = ["error", "open"]

Guido van Rossum's avatar
Guido van Rossum committed
30 31
_BLOCKSIZE = 512

32
error = OSError
33

34
class _Database(collections.MutableMapping):
Guido van Rossum's avatar
Guido van Rossum committed
35

36 37 38 39 40 41
    # The on-disk directory and data files can remain in mutually
    # inconsistent states for an arbitrarily long time (see comments
    # at the end of __setitem__).  This is only repaired when _commit()
    # gets called.  One place _commit() gets called is from __del__(),
    # and if that occurs at program shutdown time, module globals may
    # already have gotten rebound to None.  Since it's crucial that
Tim Peters's avatar
Tim Peters committed
42
    # _commit() finish successfully, we can't ignore shutdown races
43 44
    # here, and _commit() must not reference any globals.
    _os = _os       # for _commit()
45
    _io = _io       # for _commit()
46

47
    def __init__(self, filebasename, mode):
48
        self._mode = mode
49 50 51 52 53 54

        # The directory file is a text file.  Each line looks like
        #    "%r, (%d, %d)\n" % (key, pos, siz)
        # where key is the string key, pos is the offset into the dat
        # file of the associated value's first byte, and siz is the number
        # of bytes in the associated value.
Skip Montanaro's avatar
Skip Montanaro committed
55
        self._dirfile = filebasename + '.dir'
56 57 58 59 60

        # The data file is a binary file pointed into by the directory
        # file, and holds the values associated with keys.  Each value
        # begins at a _BLOCKSIZE-aligned byte offset, and is a raw
        # binary 8-bit string value.
Skip Montanaro's avatar
Skip Montanaro committed
61 62
        self._datfile = filebasename + '.dat'
        self._bakfile = filebasename + '.bak'
63 64 65 66

        # The index is an in-memory dict, mirroring the directory file.
        self._index = None  # maps keys to (pos, siz) pairs

Tim Peters's avatar
Tim Peters committed
67 68
        # Mod by Jack: create data file if needed
        try:
69
            f = _io.open(self._datfile, 'r', encoding="Latin-1")
70
        except OSError:
71 72 73 74
            with _io.open(self._datfile, 'w', encoding="Latin-1") as f:
                self._chmod(self._datfile)
        else:
            f.close()
Tim Peters's avatar
Tim Peters committed
75 76
        self._update()

77
    # Read directory file into the in-memory index dict.
Tim Peters's avatar
Tim Peters committed
78 79 80
    def _update(self):
        self._index = {}
        try:
81
            f = _io.open(self._dirfile, 'r', encoding="Latin-1")
82
        except OSError:
Tim Peters's avatar
Tim Peters committed
83 84
            pass
        else:
85 86 87 88 89 90
            with f:
                for line in f:
                    line = line.rstrip()
                    key, pos_and_siz_pair = eval(line)
                    key = key.encode('Latin-1')
                    self._index[key] = pos_and_siz_pair
Tim Peters's avatar
Tim Peters committed
91

92 93 94
    # Write the index dict to the directory file.  The original directory
    # file (if any) is renamed with a .bak extension first.  If a .bak
    # file currently exists, it's deleted.
Tim Peters's avatar
Tim Peters committed
95
    def _commit(self):
96 97 98
        # CAUTION:  It's vital that _commit() succeed, and _commit() can
        # be called from __del__().  Therefore we must never reference a
        # global in this routine.
99 100 101
        if self._index is None:
            return  # nothing to do

102
        try:
103
            self._os.unlink(self._bakfile)
104
        except OSError:
105 106 107
            pass

        try:
108
            self._os.rename(self._dirfile, self._bakfile)
109
        except OSError:
110 111
            pass

112 113 114 115 116 117 118
        with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f:
            self._chmod(self._dirfile)
            for key, pos_and_siz_pair in self._index.items():
                # Use Latin-1 since it has no qualms with any value in any
                # position; UTF-8, though, does care sometimes.
                entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair)
                f.write(entry)
Tim Peters's avatar
Tim Peters committed
119

120 121
    sync = _commit

122 123 124 125
    def _verify_open(self):
        if self._index is None:
            raise error('DBM object has already been closed')

Tim Peters's avatar
Tim Peters committed
126
    def __getitem__(self, key):
127 128
        if isinstance(key, str):
            key = key.encode('utf-8')
129
        self._verify_open()
Tim Peters's avatar
Tim Peters committed
130
        pos, siz = self._index[key]     # may raise KeyError
131 132 133
        with _io.open(self._datfile, 'rb') as f:
            f.seek(pos)
            dat = f.read(siz)
Tim Peters's avatar
Tim Peters committed
134 135
        return dat

136 137 138 139
    # Append val to the data file, starting at a _BLOCKSIZE-aligned
    # offset.  The data file is first padded with NUL bytes (if needed)
    # to get to an aligned offset.  Return pair
    #     (starting offset of val, len(val))
Tim Peters's avatar
Tim Peters committed
140
    def _addval(self, val):
141 142 143 144 145 146 147
        with _io.open(self._datfile, 'rb+') as f:
            f.seek(0, 2)
            pos = int(f.tell())
            npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
            f.write(b'\0'*(npos-pos))
            pos = npos
            f.write(val)
Tim Peters's avatar
Tim Peters committed
148 149
        return (pos, len(val))

150 151 152 153
    # Write val to the data file, starting at offset pos.  The caller
    # is responsible for ensuring that there's enough room starting at
    # pos to hold val, without overwriting some other value.  Return
    # pair (pos, len(val)).
Tim Peters's avatar
Tim Peters committed
154
    def _setval(self, pos, val):
155 156 157
        with _io.open(self._datfile, 'rb+') as f:
            f.seek(pos)
            f.write(val)
Tim Peters's avatar
Tim Peters committed
158 159
        return (pos, len(val))

160
    # key is a new key whose associated value starts in the data file
161 162
    # at offset pos and with length siz.  Add an index record to
    # the in-memory index dict, and append one to the directory file.
163 164
    def _addkey(self, key, pos_and_siz_pair):
        self._index[key] = pos_and_siz_pair
165 166 167
        with _io.open(self._dirfile, 'a', encoding="Latin-1") as f:
            self._chmod(self._dirfile)
            f.write("%r, %r\n" % (key.decode("Latin-1"), pos_and_siz_pair))
Tim Peters's avatar
Tim Peters committed
168 169

    def __setitem__(self, key, val):
170 171 172 173
        if isinstance(key, str):
            key = key.encode('utf-8')
        elif not isinstance(key, (bytes, bytearray)):
            raise TypeError("keys must be bytes or strings")
174 175 176 177
        if isinstance(val, str):
            val = val.encode('utf-8')
        elif not isinstance(val, (bytes, bytearray)):
            raise TypeError("values must be bytes or strings")
178
        self._verify_open()
179 180
        if key not in self._index:
            self._addkey(key, self._addval(val))
Tim Peters's avatar
Tim Peters committed
181
        else:
182 183
            # See whether the new value is small enough to fit in the
            # (padded) space currently occupied by the old value.
Tim Peters's avatar
Tim Peters committed
184
            pos, siz = self._index[key]
185 186
            oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE
            newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE
Tim Peters's avatar
Tim Peters committed
187
            if newblocks <= oldblocks:
188
                self._index[key] = self._setval(pos, val)
Tim Peters's avatar
Tim Peters committed
189
            else:
190 191 192 193 194 195 196
                # The new value doesn't fit in the (padded) space used
                # by the old value.  The blocks used by the old value are
                # forever lost.
                self._index[key] = self._addval(val)

            # Note that _index may be out of synch with the directory
            # file now:  _setval() and _addval() don't update the directory
197 198 199 200 201
            # file.  This also means that the on-disk directory and data
            # files are in a mutually inconsistent state, and they'll
            # remain that way until _commit() is called.  Note that this
            # is a disaster (for the database) if the program crashes
            # (so that _commit() never gets called).
Tim Peters's avatar
Tim Peters committed
202 203

    def __delitem__(self, key):
204 205
        if isinstance(key, str):
            key = key.encode('utf-8')
206
        self._verify_open()
207
        # The blocks used by the associated value are lost.
Tim Peters's avatar
Tim Peters committed
208
        del self._index[key]
209
        # XXX It's unclear why we do a _commit() here (the code always
210
        # XXX has, so I'm not changing it).  __setitem__ doesn't try to
211 212
        # XXX keep the directory file in synch.  Why should we?  Or
        # XXX why shouldn't __setitem__?
Tim Peters's avatar
Tim Peters committed
213 214 215
        self._commit()

    def keys(self):
216 217 218 219
        try:
            return list(self._index)
        except TypeError:
            raise error('DBM object has already been closed') from None
220 221

    def items(self):
222
        self._verify_open()
223
        return [(key, self[key]) for key in self._index.keys()]
Tim Peters's avatar
Tim Peters committed
224

225
    def __contains__(self, key):
226 227
        if isinstance(key, str):
            key = key.encode('utf-8')
228 229 230 231 232 233 234
        try:
            return key in self._index
        except TypeError:
            if self._index is None:
                raise error('DBM object has already been closed') from None
            else:
                raise
235 236

    def iterkeys(self):
237 238 239 240
        try:
            return iter(self._index)
        except TypeError:
            raise error('DBM object has already been closed') from None
241 242
    __iter__ = iterkeys

Tim Peters's avatar
Tim Peters committed
243
    def __len__(self):
244 245 246 247
        try:
            return len(self._index)
        except TypeError:
            raise error('DBM object has already been closed') from None
Tim Peters's avatar
Tim Peters committed
248 249

    def close(self):
250
        self._commit()
251
        self._index = self._datfile = self._dirfile = self._bakfile = None
Guido van Rossum's avatar
Guido van Rossum committed
252

253
    __del__ = close
Tim Peters's avatar
Tim Peters committed
254

255
    def _chmod(self, file):
256 257
        if hasattr(self._os, 'chmod'):
            self._os.chmod(file, self._mode)
258

259 260 261 262 263 264
    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

Guido van Rossum's avatar
Guido van Rossum committed
265

266
def open(file, flag=None, mode=0o666):
267 268 269
    """Open the database file, filename, and return corresponding object.

    The flag argument, used to control how the database is opened in the
270
    other DBM implementations, is ignored in the dbm.dumb module; the
271 272 273 274
    database is always opened for update, and will be created if it does
    not exist.

    The optional mode argument is the UNIX mode of the file, used only when
275
    the database has to be created.  It defaults to octal code 0o666 (and
276 277 278
    will be modified by the prevailing umask).

    """
279
    # flag argument is currently ignored
280 281 282 283 284 285 286 287 288 289

    # Modify mode depending on the umask
    try:
        um = _os.umask(0)
        _os.umask(um)
    except AttributeError:
        pass
    else:
        # Turn off any bits that are set in the umask
        mode = mode & (~um)
290

291
    return _Database(file, mode)