gh-108469: Update ast.unparse for unescaped quote support from PEP701 [3.12] (#108553)

Co-authored-by: sunmy2019 <59365878+sunmy2019@users.noreply.github.com>

Author: tonybaloney

Lib/ast.py

@@ -1225,17 +1225,7 @@ def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES):
 
     def visit_JoinedStr(self, node):
         self.write("f")
-        if self._avoid_backslashes:
-            with self.buffered() as buffer:
-                self._write_fstring_inner(node)
-            return self._write_str_avoiding_backslashes("".join(buffer))
-
-        # If we don't need to avoid backslashes globally (i.e., we only need
-        # to avoid them inside FormattedValues), it's cosmetically preferred
-        # to use escaped whitespace. That is, it's preferred to use backslashes
-        # for cases like: f"{x}\n". To accomplish this, we keep track of what
-        # in our buffer corresponds to FormattedValues and what corresponds to
-        # Constant parts of the f-string, and allow escapes accordingly.
+
         fstring_parts = []
         for value in node.values:
             with self.buffered() as buffer:
@@ -1247,11 +1237,14 @@ def visit_JoinedStr(self, node):
         new_fstring_parts = []
         quote_types = list(_ALL_QUOTES)
         for value, is_constant in fstring_parts:
-            value, quote_types = self._str_literal_helper(
-                value,
-                quote_types=quote_types,
-                escape_special_whitespace=is_constant,
-            )
+            if is_constant:
+                value, quote_types = self._str_literal_helper(
+                    value,
+                    quote_types=quote_types,
+                    escape_special_whitespace=True,
+                )
+            elif "\n" in value:
+                quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
             new_fstring_parts.append(value)
 
         value = "".join(new_fstring_parts)
@@ -1273,16 +1266,12 @@ def _write_fstring_inner(self, node):
 
     def visit_FormattedValue(self, node):
         def unparse_inner(inner):
-            unparser = type(self)(_avoid_backslashes=True)
+            unparser = type(self)()
             unparser.set_precedence(_Precedence.TEST.next(), inner)
             return unparser.visit(inner)
 
         with self.delimit("{", "}"):
             expr = unparse_inner(node.value)
-            if "\\" in expr:
-                raise ValueError(
-                    "Unable to avoid backslash in f-string expression part"
-                )
             if expr.startswith("{"):
                 # Separate pair of opening brackets as "{ {"
                 self.write(" ")

Lib/test/test_tokenize.py

@@ -1860,7 +1860,7 @@ def test_random_files(self):
 
         testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py"))
 
-        # TODO: Remove this once we can unparse PEP 701 syntax
+        # TODO: Remove this once we can untokenize PEP 701 syntax
         testfiles.remove(os.path.join(tempdir, "test_fstring.py"))
 
         for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):

Lib/test/test_unparse.py

@@ -197,6 +197,10 @@ def test_fstrings_complicated(self):
         self.check_ast_roundtrip('''f"a\\r\\nb"''')
         self.check_ast_roundtrip('''f"\\u2028{'x'}"''')
 
+    def test_fstrings_pep701(self):
+        self.check_ast_roundtrip('f" something { my_dict["key"] } something else "')
+        self.check_ast_roundtrip('f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"')
+
     def test_strings(self):
         self.check_ast_roundtrip("u'foo'")
         self.check_ast_roundtrip("r'foo'")
@@ -378,8 +382,15 @@ def test_invalid_fstring_value(self):
             )
         )
 
-    def test_invalid_fstring_backslash(self):
-        self.check_invalid(ast.FormattedValue(value=ast.Constant(value="\\\\")))
+    def test_fstring_backslash(self):
+        # valid since Python 3.12
+        self.assertEqual(ast.unparse(
+                            ast.FormattedValue(
+                                value=ast.Constant(value="\\\\"),
+                                conversion=-1,
+                                format_spec=None,
+                            )
+                        ), "{'\\\\\\\\'}")
 
     def test_invalid_yield_from(self):
         self.check_invalid(ast.YieldFrom(value=None))
@@ -502,11 +513,11 @@ def test_class_bases_and_keywords(self):
         self.check_src_roundtrip("class X(*args, **kwargs):\n    pass")
 
     def test_fstrings(self):
-        self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''')
-        self.check_src_roundtrip('''f"\\u2028{'x'}"''')
+        self.check_src_roundtrip("f'-{f'*{f'+{f'.{x}.'}+'}*'}-'")
+        self.check_src_roundtrip("f'\\u2028{'x'}'")
         self.check_src_roundtrip(r"f'{x}\n'")
-        self.check_src_roundtrip('''f''\'{"""\n"""}\\n''\'''')
-        self.check_src_roundtrip('''f''\'{f"""{x}\n"""}\\n''\'''')
+        self.check_src_roundtrip("f'{'\\n'}\\n'")
+        self.check_src_roundtrip("f'{f'{x}\\n'}\\n'")
 
     def test_docstrings(self):
         docstrings = (

Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst

@@ -0,0 +1,3 @@
+:func:`ast.unparse` now supports new :term:`f-string` syntax introduced in
+Python 3.12. Note that the :term:`f-string` quotes are reselected for simplicity
+under the new syntax. (Patch by Steven Sun)