Simplified tuple returned by string._formatter_parser to only have

4 elements.  No need for old is_markup element, the same information
is indicated by literal_string being None.

Factored string.Formatter class to make subclasses easier to write.

Lib/string.py

@@ -208,49 +208,83 @@ class Formatter:
         return self.vformat(format_string, args, kwargs)
 
     def vformat(self, format_string, args, kwargs):
-        used_args = set()
+        used_args = self.get_empty_used_args()
         result = []
-        for (is_markup, literal, field_name, format_spec, conversion) in \
-                format_string._formatter_parser():
-            if is_markup:
-                # given the field_name, find the object it references
-
-                # split it into the first part, and and iterator that
-                #  looks over the rest
-                first, rest = field_name._formatter_field_name_split()
-
-                used_args.add(first)
-                obj = self.get_value(first, args, kwargs)
+        for literal_text, field_name, format_spec, conversion in \
+                self.parse(format_string):
+            if literal_text is None:
+                # this is some markup, find the object and do
+                #  the formatting
 
-                # loop through the rest of the field_name, doing
-                #  getattr or getitem as needed
-                for is_attr, i in rest:
-                    if is_attr:
-                        obj = getattr(obj, i)
-                    else:
-                        obj = obj[i]
+                # given the field_name, find the object it references
+                obj = self.get_field(field_name, args, kwargs, used_args)
 
                 # do any conversion on the resulting object
-                if conversion == 'r':
-                    obj = repr(obj)
-                elif conversion == 's':
-                    obj = str(obj)
+                obj = self.convert_field(obj, conversion)
 
                 # format the object and append to the result
                 result.append(self.format_field(obj, format_spec))
             else:
-                result.append(literal)
+                # this is literal text, use it directly
+                result.append(literal_text)
         self.check_unused_args(used_args, args, kwargs)
         return ''.join(result)
 
+
+    def get_empty_used_args(self):
+        return set()
+
+
     def get_value(self, key, args, kwargs):
         if isinstance(key, int):
             return args[key]
         else:
             return kwargs[key]
 
+
     def check_unused_args(self, used_args, args, kwargs):
         pass
 
+
     def format_field(self, value, format_spec):
         return format(value, format_spec)
+
+
+    def convert_field(self, value, conversion):
+        # do any conversion on the resulting object
+        if conversion == 'r':
+            return repr(value)
+        elif conversion == 's':
+            return str(value)
+        else:
+            assert conversion is None
+            return value
+
+
+    # returns an iterable that contains tuples of the form:
+    # (literal_text, field_name, format_spec, conversion)
+    def parse(self, format_string):
+        return format_string._formatter_parser()
+
+
+    # given a field_name, find the object it references.
+    #  field_name:   the field being looked up, e.g. "0.name"
+    #                 or "lookup[3]"
+    #  used_args:    a set of which args have been used
+    #  args, kwargs: as passed in to vformat
+    # also, mark it as used in 'used_args'
+    def get_field(self, field_name, args, kwargs, used_args):
+        first, rest = field_name._formatter_field_name_split()
+
+        used_args.add(first)
+        obj = self.get_value(first, args, kwargs)
+
+        # loop through the rest of the field_name, doing
+        #  getattr or getitem as needed
+        for is_attr, i in rest:
+            if is_attr:
+                obj = getattr(obj, i)
+            else:
+                obj = obj[i]
+
+        return obj

Objects/stringlib/string_format.h

@@ -979,17 +979,12 @@ formatteriter_next(formatteriterobject *it)
         /* if 0, error has already been set, if 1, iterator is empty */
         return NULL;
     else {
-        PyObject *is_markup_bool = NULL;
         PyObject *literal_str = NULL;
         PyObject *field_name_str = NULL;
         PyObject *format_spec_str = NULL;
         PyObject *conversion_str = NULL;
         PyObject *tuple = NULL;
 
-        is_markup_bool = PyBool_FromLong(is_markup);
-        if (!is_markup_bool)
-            return NULL;
-
         if (is_markup) {
             /* field_name, format_spec, and conversion are returned */
             literal_str = Py_None;
@@ -1030,11 +1025,9 @@ formatteriter_next(formatteriterobject *it)
             Py_INCREF(format_spec_str);
             Py_INCREF(conversion_str);
         }
-        tuple = PyTuple_Pack(5, is_markup_bool, literal_str,
-                             field_name_str, format_spec_str,
+        tuple = PyTuple_Pack(4, literal_str, field_name_str, format_spec_str,
                              conversion_str);
     error:
-        Py_XDECREF(is_markup_bool);
         Py_XDECREF(literal_str);
         Py_XDECREF(field_name_str);
         Py_XDECREF(format_spec_str);