bpo-34822: Simplify AST for subscription. (GH-9605)

* Remove the slice type.
* Make Slice a kind of the expr type instead of the slice type.
* Replace ExtSlice(slices) with Tuple(slices, Load()).
* Replace Index(value) with a value itself.

All non-terminal nodes in AST for expressions are now of the expr type.

Author: serhiy-storchaka

Doc/library/ast.rst

@@ -120,13 +120,26 @@ Node classes
 
    Class :class:`ast.Constant` is now used for all constants.
 
+.. versionchanged:: 3.9
+
+   Simple indices are represented by their value, extended slices are
+   represented as tuples.
+
 .. deprecated:: 3.8
 
    Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`,
    :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available,
-   but they will be removed in future Python releases.  In the meanwhile,
+   but they will be removed in future Python releases.  In the meantime,
    instantiating them will return an instance of a different class.
 
+.. deprecated:: 3.9
+
+   Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still
+   available, but they will be removed in future Python releases.
+   In the meantime, instantiating them will return an instance of
+   a different class.
+
+
 Literals
 ^^^^^^^^
 
@@ -552,30 +565,33 @@ Subscripting
 
 .. class:: Subscript(value, slice, ctx)
 
-   A subscript, such as ``l[1]``. ``value`` is the object, often a
-   :class:`Name`. ``slice`` is one of :class:`Index`, :class:`Slice` or
-   :class:`ExtSlice`. ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del`
+   A subscript, such as ``l[1]``. ``value`` is the subscripted object
+   (usually sequence or mapping). ``slice`` is an index, slice or key.
+   It can be a :class:`Tuple` and contain a :class:`Slice`.
+   ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del`
    according to the action performed with the subscript.
 
-
-.. class:: Index(value)
-
-   Simple subscripting with a single value
-
    .. doctest::
 
-        >>> print(ast.dump(ast.parse('l[1]', mode='eval'), indent=4))
+        >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4))
         Expression(
             body=Subscript(
                 value=Name(id='l', ctx=Load()),
-                slice=Index(
-                    value=Constant(value=1)),
+                slice=Tuple(
+                    elts=[
+                        Slice(
+                            lower=Constant(value=1),
+                            upper=Constant(value=2)),
+                        Constant(value=3)],
+                    ctx=Load()),
                 ctx=Load()))
 
 
 .. class:: Slice(lower, upper, step)
 
-   Regular slicing (on the form x:y).
+   Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``).
+   Can occur only inside the *slice* field of :class:`Subscript`, either
+   directly or as an element of :class:`Tuple`.
 
    .. doctest::
 
@@ -589,27 +605,6 @@ Subscripting
                 ctx=Load()))
 
 
-.. class:: ExtSlice(dims)
-
-   Advanced slicing. ``dims`` holds a list of :class:`Slice` and
-   :class:`Index` nodes
-
-   .. doctest::
-
-        >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4))
-        Expression(
-            body=Subscript(
-                value=Name(id='l', ctx=Load()),
-                slice=ExtSlice(
-                    dims=[
-                        Slice(
-                            lower=Constant(value=1),
-                            upper=Constant(value=2)),
-                        Index(
-                            value=Constant(value=3))]),
-                ctx=Load()))
-
-
 Comprehensions
 ~~~~~~~~~~~~~~
 
@@ -823,8 +818,7 @@ Statements
                 AnnAssign(
                     target=Subscript(
                         value=Name(id='a', ctx=Load()),
-                        slice=Index(
-                            value=Constant(value=1)),
+                        slice=Constant(value=1),
                         ctx=Store()),
                     annotation=Name(id='int', ctx=Load()),
                     simple=0)],
@@ -1708,7 +1702,7 @@ and classes for traversing abstract syntax trees:
           def visit_Name(self, node):
               return Subscript(
                   value=Name(id='data', ctx=Load()),
-                  slice=Index(value=Constant(value=node.id)),
+                  slice=Constant(value=node.id),
                   ctx=node.ctx
               ), node)
 

Doc/tools/susp-ignored.csv

@@ -108,6 +108,8 @@ howto/pyporting,,::,Programming Language :: Python :: 3
 howto/regex,,::,
 howto/regex,,:foo,(?:foo)
 howto/urllib2,,:password,"""joe:password@example.com"""
+library/ast,,:upper,lower:upper
+library/ast,,:step,lower:upper:step
 library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)],"
 library/bisect,32,:hi,all(val >= x for val in a[i:hi])
 library/bisect,42,:hi,all(val > x for val in a[i:hi])

Doc/whatsnew/3.9.rst

@@ -535,6 +535,12 @@ Deprecated
 
   (Contributed by Victor Stinner in :issue:`39353`.)
 
+* :mod:`ast` classes ``Index`` and ``ExtSlice`` are considered deprecated
+  and will be removed in future Python versions.  ``value`` itself should be
+  used instead of ``Index(value)``.  ``Tuple(slices, Load())`` should be
+  used instead of ``ExtSlice(slices)``.
+  (Contributed by Serhiy Storchaka in :issue:`32892`.)
+
 * The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized`
   functions are now deprecated and will be removed in Python 3.11. Calling
   :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized
@@ -667,10 +673,17 @@ Changes in the Python API
   since the *buffering* parameter has been removed.
   (Contributed by Victor Stinner in :issue:`39357`.)
 
+* Simplified AST for subscription. Simple indices will be represented by
+  their value, extended slices will be represented as tuples.
+  ``Index(value)`` will return a ``value`` itself, ``ExtSlice(slices)``
+  will return ``Tuple(slices, Load())``.
+  (Contributed by Serhiy Storchaka in :issue:`34822`.)
+
 * The :mod:`importlib` module now ignores the :envvar:`PYTHONCASEOK`
   environment variable when the :option:`-E` or :option:`-I` command line
   options are being used.
 
+
 CPython bytecode changes
 ------------------------
 

Include/Python-ast.h

@@ -445,7 +445,7 @@ class RewriteName(NodeTransformer):
            def visit_Name(self, node):
                return copy_location(Subscript(
                    value=Name(id='data', ctx=Load()),
-                   slice=Index(value=Str(s=node.id)),
+                   slice=Constant(value=node.id),
                    ctx=node.ctx
                ), node)
 
@@ -552,6 +552,7 @@ def __new__(cls, *args, **kwargs):
 _const_types_not = {
     Num: (bool,),
 }
+
 _const_node_type_names = {
     bool: 'NameConstant',  # should be before int
     type(None): 'NameConstant',
@@ -563,6 +564,23 @@ def __new__(cls, *args, **kwargs):
     type(...): 'Ellipsis',
 }
 
+class Index(AST):
+    def __new__(cls, value, **kwargs):
+        return value
+
+class ExtSlice(AST):
+    def __new__(cls, dims=(), **kwargs):
+        return Tuple(list(dims), Load(), **kwargs)
+
+def _dims_getter(self):
+    return self.elts
+
+def _dims_setter(self, value):
+    self.elts = value
+
+Tuple.dims = property(_dims_getter, _dims_setter)
+
+
 # Large float and imaginary literals get turned into infinities in the AST.
 # We unparse those infinities to INFSTR.
 _INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
@@ -1268,10 +1286,8 @@ def visit_Subscript(self, node):
         self.set_precedence(_Precedence.ATOM, node.value)
         self.traverse(node.value)
         with self.delimit("[", "]"):
-            if (isinstance(node.slice, Index)
-                    and isinstance(node.slice.value, Tuple)
-                    and node.slice.value.elts):
-                self.items_view(self.traverse, node.slice.value.elts)
+            if isinstance(node.slice, Tuple) and node.slice.elts:
+                self.items_view(self.traverse, node.slice.elts)
             else:
                 self.traverse(node.slice)
 
@@ -1283,10 +1299,6 @@ def visit_Starred(self, node):
     def visit_Ellipsis(self, node):
         self.write("...")
 
-    def visit_Index(self, node):
-        self.set_precedence(_Precedence.TUPLE, node.value)
-        self.traverse(node.value)
-
     def visit_Slice(self, node):
         if node.lower:
             self.traverse(node.lower)
@@ -1297,9 +1309,6 @@ def visit_Slice(self, node):
             self.write(":")
             self.traverse(node.step)
 
-    def visit_ExtSlice(self, node):
-        self.items_view(self.traverse, node.dims)
-
     def visit_arg(self, node):
         self.write(node.arg)
         if node.annotation:

Lib/ast.py

@@ -343,6 +343,10 @@ def test_base_classes(self):
     def test_field_attr_existence(self):
         for name, item in ast.__dict__.items():
             if self._is_ast_node(name, item):
+                if name == 'Index':
+                    # Index(value) just returns value now.
+                    # The argument is required.
+                    continue
                 x = item()
                 if isinstance(x, ast.AST):
                     self.assertEqual(type(x._fields), tuple)
@@ -1308,21 +1312,21 @@ def test_attribute(self):
         self.expr(attr, "must have Load context")
 
     def test_subscript(self):
-        sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Index(ast.Num(3)),
+        sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Num(3),
                             ast.Load())
         self.expr(sub, "must have Load context")
         x = ast.Name("x", ast.Load())
-        sub = ast.Subscript(x, ast.Index(ast.Name("y", ast.Store())),
+        sub = ast.Subscript(x, ast.Name("y", ast.Store()),
                             ast.Load())
         self.expr(sub, "must have Load context")
         s = ast.Name("x", ast.Store())
         for args in (s, None, None), (None, s, None), (None, None, s):
             sl = ast.Slice(*args)
             self.expr(ast.Subscript(x, sl, ast.Load()),
                       "must have Load context")
-        sl = ast.ExtSlice([])
-        self.expr(ast.Subscript(x, sl, ast.Load()), "empty dims on ExtSlice")
-        sl = ast.ExtSlice([ast.Index(s)])
+        sl = ast.Tuple([], ast.Load())
+        self.expr(ast.Subscript(x, sl, ast.Load()))
+        sl = ast.Tuple([s], ast.Load())
         self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context")
 
     def test_starred(self):
@@ -1664,11 +1668,11 @@ def test_slices(self):
         ''').strip()
         i1, i2, im = map(self._parse_value, (s1, s2, sm))
         self._check_content(s1, i1.value, 'f()[1, 2]')
-        self._check_content(s1, i1.value.slice.value, '1, 2')
+        self._check_content(s1, i1.value.slice, '1, 2')
         self._check_content(s2, i2.slice.lower, 'a.b')
         self._check_content(s2, i2.slice.upper, 'c.d')
-        self._check_content(sm, im.slice.dims[0].upper, 'f ()')
-        self._check_content(sm, im.slice.dims[1].lower, 'g ()')
+        self._check_content(sm, im.slice.elts[0].upper, 'f ()')
+        self._check_content(sm, im.slice.elts[1].lower, 'g ()')
         self._check_end_pos(im, 3, 3)
 
     def test_binop(self):
@@ -1989,13 +1993,13 @@ def main():
 ('Expression', ('Constant', (1, 0, 1, 2), 10, None)),
 ('Expression', ('Constant', (1, 0, 1, 8), 'string', None)),
 ('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))),
-('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))),
+('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))),
 ('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))),
 ('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))),
 ('Expression', ('List', (1, 0, 1, 2), [], ('Load',))),
 ('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))),
 ('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))),
 ('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))),
-('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])),
+('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])),
 ]
 main()