Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r65209 | raymond.hettinger | 2008-07-23 19:08:18 -0500 (Wed, 23 Jul 2008) | 1 line

  Finish-up the partial conversion from int to Py_ssize_t for deque indices and length.
........
  r65210 | raymond.hettinger | 2008-07-23 19:53:49 -0500 (Wed, 23 Jul 2008) | 1 line

  Parse to the correct datatype.
........
  r65211 | benjamin.peterson | 2008-07-23 21:27:46 -0500 (Wed, 23 Jul 2008) | 1 line

  fix spacing
........
  r65212 | benjamin.peterson | 2008-07-23 21:31:28 -0500 (Wed, 23 Jul 2008) | 1 line

  fix markup
........
  r65213 | benjamin.peterson | 2008-07-23 21:45:37 -0500 (Wed, 23 Jul 2008) | 1 line

  add some documentation for 2to3
........
  r65214 | raymond.hettinger | 2008-07-24 00:38:48 -0500 (Thu, 24 Jul 2008) | 1 line

  Finish conversion from int to Py_ssize_t.
........
  r65215 | raymond.hettinger | 2008-07-24 02:04:55 -0500 (Thu, 24 Jul 2008) | 1 line

  Convert from long to Py_ssize_t.
........
  r65216 | georg.brandl | 2008-07-24 02:09:21 -0500 (Thu, 24 Jul 2008) | 2 lines

  Fix indentation.
........
  r65225 | benjamin.peterson | 2008-07-25 11:55:37 -0500 (Fri, 25 Jul 2008) | 1 line

  teach .bzrignore about doc tools
........
  r65226 | benjamin.peterson | 2008-07-25 12:02:11 -0500 (Fri, 25 Jul 2008) | 1 line

  document default value for fillvalue
........
  r65233 | raymond.hettinger | 2008-07-25 13:43:33 -0500 (Fri, 25 Jul 2008) | 1 line

  Issue 1592:  Better error reporting for operations on closed shelves.
........
  r65239 | benjamin.peterson | 2008-07-25 16:59:53 -0500 (Fri, 25 Jul 2008) | 1 line

  fix indentation
........
  r65246 | andrew.kuchling | 2008-07-26 08:08:19 -0500 (Sat, 26 Jul 2008) | 1 line

  This sentence continues to bug me; rewrite it for the second time
........
  r65247 | andrew.kuchling | 2008-07-26 08:09:06 -0500 (Sat, 26 Jul 2008) | 1 line

  Remove extra words
........
  r65255 | skip.montanaro | 2008-07-26 19:49:02 -0500 (Sat, 26 Jul 2008) | 3 lines

  Close issue 3437 - missing state change when Allow lines are processed.
  Adds test cases which use Allow: as well.
........
  r65256 | skip.montanaro | 2008-07-26 19:50:41 -0500 (Sat, 26 Jul 2008) | 2 lines

  note robotparser bug fix.
........

.bzrignore

@@ -35,11 +35,14 @@ gb-18030-2000.xml
 tags
 TAGS
 .gdb_history
+Doc/tools/sphinx
+Doc/tools/jinja
+Doc/tools/pygments
+Doc/tools/docutils
 Modules/Setup
 Modules/Setup.config
 Modules/Setup.local
 Modules/config.c
 Parser/pgen
-Lib/plat-mac/errors.rsrc.df.rsrc
 Lib/lib2to3/Grammar*.pickle
 Lib/lib2to3/PatternGrammar*.pickle

Doc/glossary.rst

@@ -16,6 +16,15 @@ Glossary
       The typical Python prompt of the interactive shell when entering code for
       an indented code block.
 
+   2to3
+      A tool that tries to convert Python 2.x code to Python 3.x code by
+      handling most of the incompatibilites that can be detected by parsing the
+      source and traversing the parse tree.
+
+      2to3 is available in the standard library as :mod:`lib2to3`; a standalone
+      entry point is provided as :file:`Tools/scripts/2to3`.  See
+      :ref:`2to3-reference`.
+
    abstract base class
       Abstract Base Classes (abbreviated ABCs) complement :term:`duck-typing` by
       providing a way to define interfaces when other techniques like :func:`hasattr`

Doc/library/2to3.rst

+.. _2to3-reference:
+
+2to3 - Automated Python 2 to 3 code translation
+===============================================
+
+.. sectionauthor:: Benjamin Peterson
+
+2to3 is a Python program that reads your Python 2.x source code and applies a
+series of *fixers* to transform it into valid Python 3.x code.
+
+
+Using 2to3
+----------
+
+2to3 can be run with a list of files to transform or a directory to recursively
+traverse looking for files with the ``.py`` extension.
+
+Here is a sample Python 2.x source file, :file:`example.py`::
+
+   def greet(name):
+       print "Hello, {0}!".format(name)
+   print "What's your name?"
+   name = raw_input()
+   greet(name)
+
+It can be converted to Python 3.x code via 2to3 on the command line::
+
+   $ 2to3 example.py
+
+A diff against the original source file will be printed.  2to3 can also write
+the needed modifications right back to the source file.  (A backup of the
+original file will also be made.)  This is done with the :option:`-w` flag::
+
+   $ 2to3 -w example.py
+
+:file:`example.py` will now look like this::
+
+   def greet(name):
+       print("Hello, {0}!".format(name))
+   print("What's your name?")
+   name = input()
+   greet(name)
+
+Comments and and exact indentation will be preserved throughout the translation
+process.
+
+By default, 2to3 will run a set of predefined fixers.  The :option:`-l` flag
+lists all avaible fixers.  An explicit set of fixers to run can be given by use
+of the :option:`-f` flag.  The following example runs only the ``imports`` and
+``has_key`` fixers::
+
+   $ 2to3 -f imports -f has_key example.py
+
+Some fixers are *explicit*, meaning they aren't run be default and must be
+listed on the command line.  Here, in addition to the default fixers, the
+``idioms`` fixer is run::
+
+   $ 2to3 -f all -f idioms example.py
+
+Notice how ``all`` enables all default fixers.
+
+Sometimes 2to3 will find will find a place in your source code that needs to be
+changed, but 2to3 cannot fix automatically.  In this case, 2to3 will print a
+warning beneath the diff for a file.
+
+
+:mod:`lib2to3` - 2to3's library
+-------------------------------
+
+.. module:: lib2to3
+   :synopsis: the 2to3 library
+.. moduleauthor:: Guido van Rossum
+.. moduleauthor:: Collin Winter
+
+.. XXX What is the public interface anyway?

Doc/library/development.rst

@@ -9,7 +9,8 @@ The modules described in this chapter help you write software.  For example, the
 :mod:`pydoc` module takes a module and generates documentation based on the
 module's contents.  The :mod:`doctest` and :mod:`unittest` modules contains
 frameworks for writing unit tests that automatically exercise code and verify
-that the expected output  is produced.
+that the expected output is produced.  :program:`2to3` can translate Python 2.x
+source code into valid Python 3.x code.
 
 The list of modules described in this chapter is:
 
@@ -19,4 +20,5 @@ The list of modules described in this chapter is:
    pydoc.rst
    doctest.rst
    unittest.rst
+   2to3.rst
    test.rst

Doc/library/email.message.rst

@@ -90,7 +90,7 @@ Here are the methods of the :class:`Message` class:
 
    .. method:: get_payload([i[, decode]])
 
-      Return a reference the current payload, which will be a list of
+      Return the current payload, which will be a list of
       :class:`Message` objects when :meth:`is_multipart` is ``True``, or a
       string when :meth:`is_multipart` is ``False``.  If the payload is a list
       and you mutate the list object, you modify the message's payload in place.

Doc/library/itertools.rst

@@ -295,9 +295,10 @@ loops that truncate the stream.
           except IndexError:
               pass
 
-   If one of the iterables is potentially infinite, then the :func:`zip_longest`
-   function should be wrapped with something that limits the number of calls (for
-   example :func:`islice` or :func:`takewhile`).
+   If one of the iterables is potentially infinite, then the
+   :func:`izip_longest` function should be wrapped with something that limits
+   the number of calls (for example :func:`islice` or :func:`takewhile`).  If
+   not specified, *fillvalue* defaults to ``None``.
 
 
 .. function:: permutations(iterable[, r])

Doc/whatsnew/2.6.rst

@@ -1144,7 +1144,7 @@ The :meth:`register`  method is useful when you've written a new
 ABC that can describe an existing type or class, or if you want
 to declare that some third-party class implements an ABC.
 For example, if you defined a :class:`PrintableType` ABC,
-it's legal to do:
+it's legal to do::
 
   # Register Python's types
   PrintableType.register(int)

Lib/shelve.py

@@ -64,6 +64,16 @@ import warnings
 
 __all__ = ["Shelf","BsdDbShelf","DbfilenameShelf","open"]
 
+class _ClosedDict(collections.MutableMapping):
+    'Marker for a closed dict.  Access attempts raise a ValueError.'
+
+    def closed(self, *args):
+        raise ValueError('invalid operation on closed shelf')
+    __iter__ = __len__ = __getitem__ = __setitem__ = __delitem__ = keys = closed
+
+    def __repr__(self):
+        return '<Closed Dictionary>'
+
 class Shelf(collections.MutableMapping):
     """Base class for shelf implementations.
 
@@ -127,7 +137,7 @@ class Shelf(collections.MutableMapping):
             self.dict.close()
         except AttributeError:
             pass
-        self.dict = 0
+        self.dict = _ClosedDict()
 
     def __del__(self):
         if not hasattr(self, 'writeback'):

Lib/test/test_robotparser.py

@@ -136,6 +136,75 @@ bad = [] # Bug report says "/" should be denied, but that is not in the RFC
 
 RobotTest(7, doc, good, bad)
 
+# From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364
+
+# 8.
+doc = """
+User-agent: Googlebot
+Allow: /folder1/myfile.html
+Disallow: /folder1/
+"""
+
+good = ['/folder1/myfile.html']
+bad = ['/folder1/anotherfile.html']
+
+RobotTest(8, doc, good, bad, agent="Googlebot")
+
+# 9.  This file is incorrect because "Googlebot" is a substring of
+#     "Googlebot-Mobile", so test 10 works just like test 9.
+doc = """
+User-agent: Googlebot
+Disallow: /
+
+User-agent: Googlebot-Mobile
+Allow: /
+"""
+
+good = []
+bad = ['/something.jpg']
+
+RobotTest(9, doc, good, bad, agent="Googlebot")
+
+good = []
+bad = ['/something.jpg']
+
+RobotTest(10, doc, good, bad, agent="Googlebot-Mobile")
+
+# 11.  Get the order correct.
+doc = """
+User-agent: Googlebot-Mobile
+Allow: /
+
+User-agent: Googlebot
+Disallow: /
+"""
+
+good = []
+bad = ['/something.jpg']
+
+RobotTest(11, doc, good, bad, agent="Googlebot")
+
+good = ['/something.jpg']
+bad = []
+
+RobotTest(12, doc, good, bad, agent="Googlebot-Mobile")
+
+
+# 13.  Google also got the order wrong in #8.  You need to specify the
+#      URLs from more specific to more general.
+doc = """
+User-agent: Googlebot
+Allow: /folder1/myfile.html
+Disallow: /folder1/
+"""
+
+good = ['/folder1/myfile.html']
+bad = ['/folder1/anotherfile.html']
+
+RobotTest(13, doc, good, bad, agent="googlebot")
+
+
+
 class NetworkTestCase(unittest.TestCase):
 
     def testPasswordProtectedSite(self):

Lib/test/test_shelve.py

@@ -47,6 +47,21 @@ class TestCase(unittest.TestCase):
         for f in glob.glob(self.fn+"*"):
             support.unlink(f)
 
+    def test_close(self):
+        d1 = {}
+        s = shelve.Shelf(d1, protocol=2, writeback=False)
+        s['key1'] = [1,2,3,4]
+        self.assertEqual(s['key1'], [1,2,3,4])
+        self.assertEqual(len(s), 1)
+        s.close()
+        self.assertRaises(ValueError, len, s)
+        try:
+            s['key1']
+        except ValueError:
+            pass
+        else:
+            self.fail('Closed shelf should not find a key')
+
     def test_ascii_file_shelf(self):
         s = shelve.open(self.fn, protocol=0)
         try:

Lib/urllib/robotparser.py

@@ -76,6 +76,10 @@ class RobotFileParser:
         We allow that a user-agent: line is not preceded by
         one or more blank lines.
         """
+        # states:
+        #   0: start state
+        #   1: saw user-agent line
+        #   2: saw an allow or disallow line
         state = 0
         entry = Entry()
 
@@ -112,6 +116,7 @@ class RobotFileParser:
                 elif line[0] == "allow":
                     if state != 0:
                         entry.rulelines.append(RuleLine(line[1], True))
+                        state = 2
         if state == 2:
             self.entries.append(entry)
 

Modules/_bisectmodule.c

@@ -5,7 +5,7 @@ Converted to C by Dmitry Vasiliev (dima at hlabs.spb.ru).
 
 #include "Python.h"
 
-static int
+static Py_ssize_t
 internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t hi)
 {
 	PyObject *litem;
@@ -41,18 +41,18 @@ static PyObject *
 bisect_right(PyObject *self, PyObject *args, PyObject *kw)
 {
 	PyObject *list, *item;
-	int lo = 0;
-	int hi = -1;
-	int index;
+	Py_ssize_t lo = 0;
+	Py_ssize_t hi = -1;
+	Py_ssize_t index;
 	static char *keywords[] = {"a", "x", "lo", "hi", NULL};
 
-	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|ii:bisect_right",
+	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|nn:bisect_right",
 		keywords, &list, &item, &lo, &hi))
 		return NULL;
 	index = internal_bisect_right(list, item, lo, hi);
 	if (index < 0)
 		return NULL;
-	return PyLong_FromLong(index);
+	return PyLong_FromSsize_t(index);
 }
 
 PyDoc_STRVAR(bisect_right_doc,
@@ -71,12 +71,12 @@ static PyObject *
 insort_right(PyObject *self, PyObject *args, PyObject *kw)
 {
 	PyObject *list, *item, *result;
-	int lo = 0;
-	int hi = -1;
-	int index;
+	Py_ssize_t lo = 0;
+	Py_ssize_t hi = -1;
+	Py_ssize_t index;
 	static char *keywords[] = {"a", "x", "lo", "hi", NULL};
 
-	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|ii:insort_right",
+	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|nn:insort_right",
 		keywords, &list, &item, &lo, &hi))
 		return NULL;
 	index = internal_bisect_right(list, item, lo, hi);
@@ -86,7 +86,7 @@ insort_right(PyObject *self, PyObject *args, PyObject *kw)
 		if (PyList_Insert(list, index, item) < 0)
 			return NULL;
 	} else {
-		result = PyObject_CallMethod(list, "insert", "iO",
+		result = PyObject_CallMethod(list, "insert", "nO",
 					     index, item);
 		if (result == NULL)
 			return NULL;
@@ -106,11 +106,11 @@ If x is already in a, insert it to the right of the rightmost x.\n\
 Optional args lo (default 0) and hi (default len(a)) bound the\n\
 slice of a to be searched.\n");
 
-static int
-internal_bisect_left(PyObject *list, PyObject *item, int lo, int hi)
+static Py_ssize_t
+internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t hi)
 {
 	PyObject *litem;
-	int mid, res;
+	Py_ssize_t mid, res;
 
 	if (lo < 0) {
 		PyErr_SetString(PyExc_ValueError, "lo must be non-negative");
@@ -142,18 +142,18 @@ static PyObject *
 bisect_left(PyObject *self, PyObject *args, PyObject *kw)
 {
 	PyObject *list, *item;
-	int lo = 0;
-	int hi = -1;
-	int index;
+	Py_ssize_t lo = 0;
+	Py_ssize_t hi = -1;
+	Py_ssize_t index;
 	static char *keywords[] = {"a", "x", "lo", "hi", NULL};
 
-	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|ii:bisect_left",
+	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|nn:bisect_left",
 		keywords, &list, &item, &lo, &hi))
 		return NULL;
 	index = internal_bisect_left(list, item, lo, hi);
 	if (index < 0)
 		return NULL;
-	return PyLong_FromLong(index);
+	return PyLong_FromSsize_t(index);
 }
 
 PyDoc_STRVAR(bisect_left_doc,
@@ -172,12 +172,12 @@ static PyObject *
 insort_left(PyObject *self, PyObject *args, PyObject *kw)
 {
 	PyObject *list, *item, *result;
-	int lo = 0;
-	int hi = -1;
-	int index;
+	Py_ssize_t lo = 0;
+	Py_ssize_t hi = -1;
+	Py_ssize_t index;
 	static char *keywords[] = {"a", "x", "lo", "hi", NULL};
 
-	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|ii:insort_left",
+	if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|nn:insort_left",
 		keywords, &list, &item, &lo, &hi))
 		return NULL;
 	index = internal_bisect_left(list, item, lo, hi);

Modules/_collectionsmodule.c

@@ -52,20 +52,20 @@ typedef struct BLOCK {
 } block;
 
 #define MAXFREEBLOCKS 10
-static int numfreeblocks = 0;
+static Py_ssize_t numfreeblocks = 0;
 static block *freeblocks[MAXFREEBLOCKS];
 
 static block *
-newblock(block *leftlink, block *rightlink, int len) {
+newblock(block *leftlink, block *rightlink, Py_ssize_t len) {
 	block *b;
-	/* To prevent len from overflowing INT_MAX on 64-bit machines, we
+	/* To prevent len from overflowing PY_SSIZE_T_MAX on 64-bit machines, we
 	 * refuse to allocate new blocks if the current len is dangerously
 	 * close.  There is some extra margin to prevent spurious arithmetic
 	 * overflows at various places.  The following check ensures that
 	 * the blocks allocated to the deque, in the worst case, can only
-	 * have INT_MAX-2 entries in total.
+	 * have PY_SSIZE_T_MAX-2 entries in total.
 	 */
-	if (len >= INT_MAX - 2*BLOCKLEN) {
+	if (len >= PY_SSIZE_T_MAX - 2*BLOCKLEN) {
 		PyErr_SetString(PyExc_OverflowError,
 				"cannot add more blocks to the deque");
 		return NULL;
@@ -100,10 +100,10 @@ typedef struct {
 	PyObject_HEAD
 	block *leftblock;
 	block *rightblock;
-	int leftindex;	/* in range(BLOCKLEN) */
-	int rightindex;	/* in range(BLOCKLEN) */
-	int len;
-	int maxlen;
+	Py_ssize_t leftindex;	/* in range(BLOCKLEN) */
+	Py_ssize_t rightindex;	/* in range(BLOCKLEN) */
+	Py_ssize_t len;
+	Py_ssize_t maxlen;
 	long state;	/* incremented whenever the indices move */
 	PyObject *weakreflist; /* List of weak references */
 } dequeobject;
@@ -355,7 +355,7 @@ PyDoc_STRVAR(extendleft_doc,
 static int
 _deque_rotate(dequeobject *deque, Py_ssize_t n)
 {
-	int i, len=deque->len, halflen=(len+1)>>1;
+	Py_ssize_t i, len=deque->len, halflen=(len+1)>>1;
 	PyObject *item, *rv;
 
 	if (len == 0)
@@ -392,9 +392,9 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n)
 static PyObject *
 deque_rotate(dequeobject *deque, PyObject *args)
 {
-	int n=1;
+	Py_ssize_t n=1;
 
-	if (!PyArg_ParseTuple(args, "|i:rotate", &n))
+	if (!PyArg_ParseTuple(args, "|n:rotate", &n))
 		return NULL;
 	if (_deque_rotate(deque, n) == 0)
 		Py_RETURN_NONE;
@@ -462,11 +462,11 @@ deque_clear(dequeobject *deque)
 }
 
 static PyObject *
-deque_item(dequeobject *deque, int i)
+deque_item(dequeobject *deque, Py_ssize_t i)
 {
 	block *b;
 	PyObject *item;
-	int n, index=i;
+	Py_ssize_t n, index=i;
 
 	if (i < 0 || i >= deque->len) {
 		PyErr_SetString(PyExc_IndexError,
@@ -591,11 +591,11 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg)
 {
 	block *b;
 	PyObject *item;
-	int index;
-	int indexlo = deque->leftindex;
+	Py_ssize_t index;
+	Py_ssize_t indexlo = deque->leftindex;
 
 	for (b = deque->leftblock; b != NULL; b = b->rightlink) {
-		const int indexhi = b == deque->rightblock ?
+		const Py_ssize_t indexhi = b == deque->rightblock ?
 					 deque->rightindex :
 				    	 BLOCKLEN - 1;
 
@@ -637,12 +637,12 @@ deque_reduce(dequeobject *deque)
 		if (deque->maxlen == -1)
 			result = Py_BuildValue("O(O)", Py_TYPE(deque), aslist);
 		else
-			result = Py_BuildValue("O(Oi)", Py_TYPE(deque), aslist, deque->maxlen);
+			result = Py_BuildValue("O(On)", Py_TYPE(deque), aslist, deque->maxlen);
 	} else {
 		if (deque->maxlen == -1)
 			result = Py_BuildValue("O(OO)O", Py_TYPE(deque), aslist, Py_None, dict);
 		else
-			result = Py_BuildValue("O(Oi)O", Py_TYPE(deque), aslist, deque->maxlen, dict);
+			result = Py_BuildValue("O(On)O", Py_TYPE(deque), aslist, deque->maxlen, dict);
 	}
 	Py_XDECREF(dict);
 	Py_DECREF(aslist);
@@ -683,7 +683,8 @@ static PyObject *
 deque_richcompare(PyObject *v, PyObject *w, int op)
 {
 	PyObject *it1=NULL, *it2=NULL, *x, *y;
-	int b, vs, ws, cmp=-1;
+	Py_ssize_t vs, ws;
+	int b, cmp=-1;
 
 	if (!PyObject_TypeCheck(v, &deque_type) ||
 	    !PyObject_TypeCheck(w, &deque_type)) {
@@ -762,13 +763,13 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs)
 {
 	PyObject *iterable = NULL;
 	PyObject *maxlenobj = NULL;
-	int maxlen = -1;
+	Py_ssize_t maxlen = -1;
 	char *kwlist[] = {"iterable", "maxlen", 0};
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, &iterable, &maxlenobj))
 		return -1;
 	if (maxlenobj != NULL && maxlenobj != Py_None) {
-		maxlen = PyLong_AsLong(maxlenobj);
+		maxlen = PyLong_AsSsize_t(maxlenobj);
 		if (maxlen == -1 && PyErr_Occurred())
 			return -1;
 		if (maxlen < 0) {
@@ -884,11 +885,11 @@ static PyTypeObject deque_type = {
 
 typedef struct {
 	PyObject_HEAD
-	int index;
+	Py_ssize_t index;
 	block *b;
 	dequeobject *deque;
 	long state;	/* state when the iterator is created */
-	int counter;    /* number of items remaining for iteration */
+	Py_ssize_t counter;    /* number of items remaining for iteration */
 } dequeiterobject;
 
 static PyTypeObject dequeiter_type;

Objects/enumobject.c

@@ -31,6 +31,7 @@ enum_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 			Py_DECREF(en);
 			return NULL;
 		}
+		assert(PyLong_Check(start));
 		en->en_index = PyLong_AsSsize_t(start);
 		if (en->en_index == -1 && PyErr_Occurred()) {
 			PyErr_Clear();