diff --git a/patches/pyink.patch b/patches/pyink.patch index 6c571ad0aaa..402acee5f34 100644 --- a/patches/pyink.patch +++ b/patches/pyink.patch @@ -2,7 +2,7 @@ # This file is provided so it's easier to see the actual differences between Black and Pyink. --- a/__init__.py +++ b/__init__.py -@@ -20,6 +20,8 @@ from pathlib import Path +@@ -19,6 +19,8 @@ from pathlib import Path from re import Pattern from typing import Any @@ -11,7 +11,7 @@ import click from click.core import ParameterSource from mypy_extensions import mypyc_attr -@@ -60,7 +62,13 @@ from pyink.linegen import LN, LineGenera +@@ -61,7 +63,13 @@ from pyink.linegen import LN, LineGenera from pyink.lines import EmptyLineTracker, LinesBlock from pyink.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature from pyink.mode import Mode as Mode # re-exported @@ -37,7 +37,7 @@ COMPILED = Path(__file__).suffix in (".pyd", ".so") -@@ -332,6 +339,61 @@ def validate_regex( +@@ -354,6 +361,61 @@ def validate_regex( ), ) @click.option( @@ -99,7 +99,7 @@ "--check", is_flag=True, help=( -@@ -532,6 +594,12 @@ def main( +@@ -618,6 +680,12 @@ def main( preview: bool, unstable: bool, enable_unstable_feature: list[Preview], @@ -112,7 +112,7 @@ quiet: bool, verbose: bool, required_version: str | None, -@@ -639,7 +707,16 @@ def main( +@@ -731,5 +731,14 @@ def main( preview=preview, unstable=unstable, python_cell_magics=set(python_cell_magics), @@ -128,9 +128,7 @@ + QuoteStyle.MAJORITY if pyink_use_majority_quotes else QuoteStyle.DOUBLE + ), ) - - lines: list[tuple[int, int]] = [] -@@ -1141,6 +1218,17 @@ def validate_metadata(nb: MutableMapping +@@ -1172,6 +1249,17 @@ def validate_metadata(nb: MutableMapping if language is not None and language != "python": raise NothingChanged from None @@ -148,7 +146,7 @@ def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: """Format Jupyter notebook. -@@ -1152,7 +1240,6 @@ def format_ipynb_string(src_contents: st +@@ -1183,7 +1271,6 @@ def format_ipynb_string(src_contents: st raise NothingChanged trailing_newline = src_contents[-1] == "\n" @@ -156,7 +154,7 @@ nb = json.loads(src_contents) validate_metadata(nb) for cell in nb["cells"]: -@@ -1164,14 +1251,17 @@ def format_ipynb_string(src_contents: st +@@ -1195,14 +1282,17 @@ def format_ipynb_string(src_contents: st pass else: cell["source"] = dst.splitlines(keepends=True) @@ -182,7 +180,7 @@ def format_str( -@@ -1242,6 +1332,8 @@ def _format_str_once( +@@ -1272,6 +1362,8 @@ def _format_str_once( future_imports = get_future_imports(src_node) versions = detect_target_versions(src_node, future_imports=future_imports) @@ -194,31 +192,32 @@ --- a/_width_table.py +++ b/_width_table.py @@ -3,7 +3,7 @@ - # Unicode 15.0.0 + # Unicode 17.0.0 from typing import Final -WIDTH_TABLE: Final[list[tuple[int, int, int]]] = [ +WIDTH_TABLE: Final[tuple[tuple[int, int, int], ...]] = ( - (0, 0, 0), - (1, 31, -1), - (127, 159, -1), -@@ -475,4 +475,4 @@ WIDTH_TABLE: Final[list[tuple[int, int, + (4352, 4447, 2), + (8986, 8987, 2), + (9001, 9002, 2), +@@ -129,4 +129,4 @@ WIDTH_TABLE: Final[list[tuple[int, int, + (129775, 129784, 2), (131072, 196605, 2), (196608, 262141, 2), - (917760, 917999, 0), -] +) --- a/comments.py +++ b/comments.py -@@ -4,6 +4,7 @@ from dataclasses import dataclass - from functools import lru_cache - from typing import Final, Union +@@ -20,6 +20,8 @@ from pyink.nodes import ( + from blib2to3.pgen2 import token + from blib2to3.pytree import Leaf, Node +from pyink import ink_comments - from pyink.mode import Mode, Preview - from pyink.nodes import ( - CLOSING_BRACKETS, -@@ -793,7 +794,7 @@ def children_contains_fmt_on(container: ++ + # types + LN = Union[Leaf, Node] + +@@ -781,7 +783,7 @@ def children_contains_fmt_on(container: return False @@ -227,7 +226,7 @@ """ Returns: True iff one of the comments in @comment_list is a pragma used by one -@@ -801,7 +802,7 @@ def contains_pragma_comment(comment_list +@@ -789,7 +791,7 @@ def contains_pragma_comment(comment_list pylint). """ for comment in comment_list: @@ -249,18 +248,19 @@ May raise: --- a/handle_ipynb_magics.py +++ b/handle_ipynb_magics.py -@@ -308,6 +308,8 @@ def unmask_cell(src: str, replacements: - """ +@@ -310,5 +310,7 @@ for replacement in replacements: - src = src.replace(replacement.mask, replacement.src) + if src.count(replacement.mask) != 1: + raise NothingChanged + src = src.replace(replacement.mask, replacement.src, 1) + # Strings in src might have been reformatted with single quotes. + src = src.replace(f"b'{replacement.mask[2:-1]}'", replacement.src) return src - - + + --- a/linegen.py +++ b/linegen.py -@@ -8,8 +8,9 @@ from collections.abc import Collection, +@@ -8,8 +8,9 @@ from collections.abc import Collection, from dataclasses import replace from enum import Enum, auto from functools import partial, wraps @@ -496,7 +496,7 @@ self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) self.visit_async_funcdef = self.visit_async_stmt self.visit_decorated = self.visit_decorators -@@ -713,14 +757,23 @@ def transform_line( +@@ -718,14 +762,23 @@ def transform_line( ll = mode.line_length sn = mode.string_normalization @@ -525,7 +525,7 @@ and not line.should_split_rhs and not line.magic_trailing_comma and ( -@@ -943,7 +996,6 @@ def _first_right_hand_split( +@@ -949,7 +1002,6 @@ def _first_right_hand_split( omit: Collection[LeafID] = (), ) -> RHSResult: """Split the line into head, body, tail starting with the last bracket pair. @@ -533,7 +533,7 @@ Note: this function should not have side effects. It's relied upon by _maybe_split_omitting_optional_parens to get an opinion whether to prefer splitting on the right side of an assignment statement. -@@ -1265,7 +1317,7 @@ def bracket_split_build_line( +@@ -1271,7 +1323,7 @@ def bracket_split_build_line( result = Line(mode=original.mode, depth=original.depth) if component is _BracketSplitComponent.body: result.inside_brackets = True @@ -542,7 +542,7 @@ if _ensure_trailing_comma(leaves, original, opening_bracket): for i in range(len(leaves) - 1, -1, -1): if leaves[i].type == STANDALONE_COMMENT: -@@ -1854,7 +1906,7 @@ def maybe_make_parens_invisible_in_atom( +@@ -1867,7 +1919,7 @@ def maybe_make_parens_invisible_in_atom( if ( # If the prefix of `middle` includes a type comment with # ignore annotation, then we do not remove the parentheses @@ -551,7 +551,7 @@ ): first.value = "" last.value = "" -@@ -1928,7 +1980,7 @@ def generate_trailers_to_omit(line: Line +@@ -1941,7 +1993,7 @@ def generate_trailers_to_omit(line: Line if not line.magic_trailing_comma: yield omit @@ -560,7 +560,7 @@ opening_bracket: Leaf | None = None closing_bracket: Leaf | None = None inner_brackets: set[LeafID] = set() -@@ -2013,7 +2065,7 @@ def run_transformer( +@@ -2026,7 +2078,7 @@ def run_transformer( or not line.bracket_tracker.invisible or any(bracket.value for bracket in line.bracket_tracker.invisible) or line.contains_multiline_strings() @@ -688,7 +688,7 @@ and bool(current_line.leaves) and "\f\n" in current_line.leaves[0].prefix ) -@@ -611,6 +632,8 @@ class EmptyLineTracker: +@@ -601,6 +622,8 @@ class EmptyLineTracker: or current_line.is_def ): return False @@ -697,7 +697,7 @@ while previous_block := previous_block.previous_block: if not previous_block.original_line.is_comment: return False -@@ -618,7 +641,7 @@ class EmptyLineTracker: +@@ -608,7 +631,7 @@ class EmptyLineTracker: def _maybe_empty_lines(self, current_line: Line) -> tuple[int, int]: max_allowed = 1 @@ -706,7 +706,7 @@ max_allowed = 1 if self.mode.is_pyi else 2 if current_line.leaves: -@@ -635,7 +658,7 @@ class EmptyLineTracker: +@@ -625,7 +648,7 @@ class EmptyLineTracker: # Mutate self.previous_defs, remainder of this function should be pure previous_def = None @@ -715,7 +715,7 @@ previous_def = self.previous_defs.pop() if current_line.is_def or current_line.is_class: self.previous_defs.append(current_line) -@@ -692,8 +715,8 @@ class EmptyLineTracker: +@@ -682,18 +705,34 @@ class EmptyLineTracker: if ( self.previous_line.is_import @@ -724,10 +724,11 @@ + and not self.previous_line.depth + and not current_line.depth and not current_line.is_import ++ and not current_line.is_comment and not current_line.is_fmt_pass_converted(first_leaf_matches=is_import) - and Preview.always_one_newline_after_import in self.mode -@@ -701,10 +724,25 @@ class EmptyLineTracker: - return 1, 0 + ): +- return 1, 0 ++ return (before or 1), 0 if ( - self.previous_line.is_import @@ -754,7 +755,7 @@ ): return (before or 1), 0 -@@ -721,8 +759,9 @@ class EmptyLineTracker: +@@ -710,8 +748,9 @@ class EmptyLineTracker: return 0, 1 return 0, 0 @@ -766,7 +767,7 @@ ): if self.mode.is_pyi: return 0, 0 -@@ -731,7 +770,7 @@ class EmptyLineTracker: +@@ -720,7 +759,7 @@ class EmptyLineTracker: comment_to_add_newlines: LinesBlock | None = None if ( self.previous_line.is_comment @@ -775,7 +776,7 @@ and before == 0 ): slc = self.semantic_leading_comment -@@ -748,9 +787,9 @@ class EmptyLineTracker: +@@ -737,9 +776,9 @@ class EmptyLineTracker: if self.mode.is_pyi: if current_line.is_class or self.previous_line.is_class: @@ -787,7 +788,7 @@ newlines = 1 elif current_line.is_stub_class and self.previous_line.is_stub_class: # No blank line between classes with an empty body -@@ -779,7 +818,11 @@ class EmptyLineTracker: +@@ -768,7 +807,11 @@ class EmptyLineTracker: newlines = 1 if current_line.depth else 2 # If a user has left no space after a dummy implementation, don't insert # new lines. This is useful for instance for @overload or Protocols. @@ -800,7 +801,30 @@ newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block -@@ -1054,7 +1097,7 @@ def can_omit_invisible_parens( +@@ -1003,11 +1003,11 @@ def can_omit_invisible_parens( + # conflict with type: ignore comments in the body + if head_comments: + has_type_ignore_in_head = any( +- is_type_ignore_comment(comment, mode=rhs.head.mode) ++ is_pragma_comment(comment, rhs.head.mode) + for comment in head_comments + ) + has_other_comment_in_head = any( +- not is_type_ignore_comment(comment, mode=rhs.head.mode) ++ not is_pragma_comment(comment, rhs.head.mode) + for comment in head_comments + ) + +@@ -1017,7 +1017,7 @@ def can_omit_invisible_parens( + has_other_comment_in_body = False + for leaf in rhs.body.leaves: + for comment in rhs.body.comments.get(id(leaf), []): +- if is_type_ignore_comment(comment, mode=rhs.body.mode): ++ if is_pragma_comment(comment, rhs.body.mode): + has_type_ignore_in_body = True + else: + has_other_comment_in_body = True +@@ -1074,7 +1117,7 @@ def can_omit_invisible_parens( def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" remainder = False @@ -809,7 +833,7 @@ _index = -1 for _index, leaf, leaf_length in line.enumerate_with_length(): if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first: -@@ -1078,7 +1121,7 @@ def _can_omit_opening_paren(line: Line, +@@ -1098,7 +1141,7 @@ def _can_omit_opening_paren(line: Line, def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" @@ -829,8 +853,8 @@ from pyink.const import DEFAULT_LINE_LENGTH -@@ -253,7 +253,31 @@ class Deprecated(UserWarning): - """Visible deprecation warning.""" +@@ -240,7 +240,31 @@ UNSTABLE_FEATURES: set[Preview] = { + } -_MAX_CACHE_KEY_PART_LENGTH: Final = 32 @@ -862,7 +886,7 @@ @dataclass -@@ -261,12 +285,21 @@ class Mode: +@@ -248,12 +272,21 @@ class Mode: target_versions: set[TargetVersion] = field(default_factory=set) line_length: int = DEFAULT_LINE_LENGTH string_normalization: bool = True @@ -884,19 +908,7 @@ unstable: bool = False enabled_features: set[Preview] = field(default_factory=set) -@@ -278,6 +311,11 @@ class Mode: - except those in UNSTABLE_FEATURES are enabled. Any features in - `self.enabled_features` are also enabled. - """ -+ # The following feature is temporarily disabled in Pyink because it is -+ # not compatible with range formatting. It's because format skipping in -+ # Black is broken. -+ if feature is Preview.always_one_newline_after_import and self.is_pyink: -+ return False - if self.unstable: - return True - if feature in self.enabled_features: -@@ -309,16 +347,32 @@ class Mode: +@@ -295,16 +333,32 @@ class Mode: version_str, str(self.line_length), str(int(self.string_normalization)), @@ -939,8 +951,8 @@ from pyink.cache import CACHE_DIR from pyink.mode import Mode, Preview from pyink.strings import get_string_prefix, has_triple_quotes -@@ -818,9 +819,13 @@ def is_function_or_class(node: Node) -> - return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} +@@ -842,9 +843,13 @@ def is_parent_function_or_class(node: No + return node.parent.type in {syms.funcdef, syms.classdef} -def is_stub_suite(node: Node) -> bool: @@ -955,13 +967,23 @@ return False # If there is a comment, we want to keep it. -@@ -947,12 +952,12 @@ def is_type_comment_string(value: str, m - return is_valid +@@ -960,7 +964,7 @@ def is_async_stmt_or_funcdef(leaf: Leaf) + def is_type_comment(leaf: Leaf, mode: Mode) -> bool: + """Return True if the given leaf is a type comment. This function should only + be used for general type comments (excluding ignore annotations, which should +- use `is_type_ignore_comment`). Note that general type comments are no longer ++ use `is_pragma_comment`). Note that general type comments are no longer + used in modern version of Python, this function may be deprecated in the future.""" + t = leaf.type + v = leaf.value +@@ -967,12 +972,12 @@ def is_type_comment_string(value: str, m + return value.startswith("#") and value[1:].lstrip().startswith("type:") -def is_type_ignore_comment(leaf: Leaf, mode: Mode) -> bool: +def is_pragma_comment(leaf: Leaf, mode: Mode) -> bool: - """Return True if the given leaf is a type comment with ignore annotation.""" +- """Return True if the given leaf is a type comment with ignore annotation.""" ++ """Return True if the given leaf is an annotation pragma (e.g., type: ignore, pylint, noqa).""" t = leaf.type v = leaf.value - return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string( @@ -969,30 +991,30 @@ + return t in {token.COMMENT, STANDALONE_COMMENT} and ( + ink_comments.comment_contains_pragma(v, mode) ) - - + + --- a/pyproject.toml +++ b/pyproject.toml -@@ -1,52 +1,24 @@ +@@ -1,64 +1,47 @@ -# Example configuration for Black. - -# NOTE: you have to use single-quoted strings in TOML for regular expressions. --# It's the equivalent of r-strings in Python. Multiline strings are treated as --# verbose regular expressions by Black. Use [ ] to denote a significant space --# character. +-# It's the equivalent of r-strings in Python. +-# Multiline strings are treated as verbose regular expressions by Black. +-# Use [ ] to denote a significant space character. - -[tool.black] +[tool.pyink] +# Yes, we use the _Black_ style to format _Pyink_ code. +pyink = false line-length = 88 - target-version = ['py310'] + target-version = ["py310"] include = '\.pyi?$' -extend-exclude = ''' -/( -- # The following are specific to Black, you probably don't want those. -- tests/data/ -- | profiling/ +- # The following are specific to Black, you probably don't want those. +- tests/data/ +- | profiling/ -) -''' -# We use the unstable style for formatting Black itself. If you @@ -1006,7 +1028,7 @@ -# NOTE: You don't need this in your own Black configuration. - [build-system] --requires = ["hatchling>=1.27.0", "hatch-vcs", "hatch-fancy-pypi-readme"] +-requires = ["hatch-fancy-pypi-readme", "hatch-vcs>=0.3.0", "hatchling>=1.27.0"] +requires = ["hatchling>=1.27.0", "hatch-vcs"] build-backend = "hatchling.build" @@ -1018,46 +1040,43 @@ license = "MIT" license-files = ["LICENSE"] requires-python = ">=3.10" --authors = [ -- { name = "Łukasz Langa", email = "lukasz@langa.pl" }, --] --keywords = [ -- "automation", -- "autopep8", -- "formatter", -- "gofmt", -- "pyfmt", -- "rustfmt", -- "yapf", --] +-authors = [{ name = "Łukasz Langa", email = "lukasz@langa.pl" }] +-keywords = ["automation", "autopep8", "formatter", "gofmt", "pyfmt", "rustfmt", "yapf"] +readme = "README.md" +authors = [{name = "The Pyink Maintainers", email = "pyink-maintainers@google.com"}] classifiers = [ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", -@@ -66,55 +38,43 @@ dependencies = [ - "click>=8.0.0", - "mypy_extensions>=0.4.3", - "packaging>=22.0", -- "pathspec>=0.9.0", -+ "pathspec>=0.9.0,<1.0.0", - "platformdirs>=2", - "pytokens>=0.3.0", - "tomli>=1.1.0; python_version < '3.11'", - "typing_extensions>=4.0.1; python_version < '3.11'", -+ "black==25.12.0", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", + ] + dependencies = [ + "click>=8.0.0", + "mypy-extensions>=0.4.3", + "packaging>=22.0", + "pathspec>=1.0.0", + "platformdirs>=2", + "pytokens~=0.4.0", + "tomli>=1.1.0; python_version<'3.11'", + "typing-extensions>=4.0.1; python_version<'3.11'", ++ "black==26.3.1", ] +@@ -65,3 +65,3 @@ -dynamic = ["readme", "version"] +dynamic = ["version"] [project.optional-dependencies] - colorama = ["colorama>=0.4.3"] - uvloop = ["uvloop>=0.15.2"] --d = ["aiohttp>=3.10"] - jupyter = [ - "ipython>=7.8.0", - "tokenize-rt>=3.2.0", - ] +@@ -106,34 +95,25 @@ diff-shades-comment = ["click>=8.1.7", " + width-table = ["wcwidth==0.2.14"] [project.scripts] -black = "black:patched_main" @@ -1076,10 +1095,7 @@ - -[tool.hatch.metadata.hooks.fancy-pypi-readme] -content-type = "text/markdown" --fragments = [ -- { path = "README.md" }, -- { path = "CHANGES.md" }, --] +-fragments = [{ path = "README.md" }, { path = "CHANGES.md" }] +Changelog = "https://github.com/google/pyink/blob/pyink/CHANGES.md" +Repository = "https://github.com/google/pyink" +Issues = "https://github.com/google/pyink/issues" @@ -1090,9 +1106,9 @@ [tool.hatch.build.hooks.vcs] -version-file = "src/_black_version.py" +version-file = "src/_pyink_version.py" - template = ''' + template = """ version = "{version}" - ''' + """ -[tool.hatch.build.targets.sdist] -exclude = ["/profiling"] @@ -1100,53 +1116,25 @@ [tool.hatch.build.targets.wheel] only-include = ["src"] sources = ["src"] -@@ -125,7 +85,6 @@ macos-max-compat = true +@@ -144,17 +124,12 @@ macos-max-compat = true # Option below requires `tests/optional.py` addopts = "--strict-config --strict-markers" optional-tests = [ -- "no_blackd: run when `d` extra NOT installed", - "no_jupyter: run when `jupyter` extra NOT installed", - ] - markers = [ -@@ -133,38 +92,3 @@ markers = [ +- "no_blackd: run when `d` extra NOT installed", + "no_jupyter: run when `jupyter` extra NOT installed", ] + markers = ["incompatible_with_mypyc: run when testing mypyc compiled black"] xfail_strict = true filterwarnings = ["error"] -[tool.coverage.report] --omit = [ -- "src/blib2to3/*", -- "tests/data/*", -- "*/site-packages/*", -- ".tox/*" --] +-omit = ["src/blib2to3/*", "tests/data/*", "*/site-packages/*", ".tox/*"] -[tool.coverage.run] -relative_files = true -branch = true -- --[tool.mypy] --# Specify the target platform details in config, so your developers are --# free to run mypy on Windows, Linux, or macOS and get consistent --# results. --python_version = "3.10" --mypy_path = "src" --strict = true --strict_bytes = true --local_partial_types = true --# Unreachable blocks have been an issue when compiling mypyc, let's try to avoid 'em in the first place. --warn_unreachable = true --implicit_reexport = true --show_error_codes = true --show_column_numbers = true -- --[[tool.mypy.overrides]] --module = ["pathspec.*", "IPython.*", "colorama.*", "tokenize_rt.*", "uvloop.*"] --ignore_missing_imports = true -- --# CI only checks src/, but in case users are running LSP or similar we explicitly ignore --# errors in test data files. --[[tool.mypy.overrides]] --module = ["tests.data.*"] --ignore_errors = true ++ + + [tool.mypy] + # Specify the target platform details in config, so your developers are --- a/resources/pyink.schema.json +++ b/resources/pyink.schema.json @@ -1,7 +1,7 @@ @@ -1212,9 +1200,23 @@ PRINT_FULL_TREE: bool = False PRINT_TREE_DIFF: bool = True +--- a/tests/data/cases/torture.py ++++ b/tests/data/cases/torture.py +@@ -57,9 +57,9 @@ 0 ^ 0 # + class A: + def foo(self): + for _ in range(10): +- aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( ++ aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( # pylint: disable=no-member + xxxxxxxxxxxx +- ) # pylint: disable=no-member ++ ) + + + def test(self, othr): --- a/tests/empty.toml +++ b/tests/empty.toml -@@ -1 +1,5 @@ +@@ -1,1 +1,5 @@ # Empty configuration file; used in tests to avoid interference from Black's own config. + +# Explicitly disable _Pyink_ mode so it's the same as the default _Black_ style. @@ -1231,8 +1233,8 @@ from pyink.output import color_diff, diff from pyink.parsing import ASTSafetyError from pyink.report import Report -@@ -2419,6 +2419,19 @@ class TestCaching: - {Preview.multiline_string_handling}, +@@ -2455,6 +2455,19 @@ class TestCaching: + {Preview.wrap_comprehension_in}, {Preview.string_processing}, ] + elif field.type is Quote: @@ -1251,7 +1253,7 @@ elif field.type is bool: values = [True, False] elif field.type is int: -@@ -2899,6 +2912,82 @@ class TestFileCollection: +@@ -3004,6 +3017,82 @@ class TestFileCollection: stdin_filename=stdin_filename, ) @@ -1334,9 +1336,33 @@ def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( self, ) -> None: +@@ -3222,7 +3222,7 @@ + future_target = TargetVersion[f"PY3{sys.version_info[1] + 1}"] + mode = Mode(target_versions={future_target}) + with patch.object( +- black, ++ pyink, + "assert_equivalent", + side_effect=ASTSafetyError("mocked parse failure"), + ): +--- a/tests/test_format.py ++++ b/tests/test_format.py +@@ -51,3 +51,11 @@ + @pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning") +-@pytest.mark.parametrize("filename", all_data_cases("cases")) ++@pytest.mark.parametrize( ++ "filename", ++ [ ++ pytest.param(name, marks=pytest.mark.skip(reason="Skipping to suppress black incompatibility.")) ++ if name in ["preview_comments7", "import_line_collapse"] ++ else name ++ for name in all_data_cases("cases") ++ ], ++) + def test_simple_format(filename: str) -> None: --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py -@@ -12,6 +12,7 @@ from click.testing import CliRunner +@@ -12,6 +12,7 @@ from pytest import MonkeyPatch from pyink import ( Mode, NothingChanged, @@ -1344,7 +1370,7 @@ format_cell, format_file_contents, format_file_in_place, -@@ -27,8 +28,15 @@ pytest.importorskip("IPython", reason="I +@@ -32,8 +33,15 @@ pytest.importorskip("IPython", reason="I pytest.importorskip("tokenize_rt", reason="tokenize-rt is an optional dependency") JUPYTER_MODE = Mode(is_ipynb=True) @@ -1360,7 +1386,7 @@ runner = CliRunner() -@@ -240,6 +248,13 @@ def test_cell_magic_with_custom_python_m +@@ -256,6 +264,13 @@ def test_cell_magic_with_custom_python_m format_cell(src, fast=True, mode=JUPYTER_MODE) @@ -1374,7 +1400,7 @@ def test_cell_magic_nested() -> None: src = "%%time\n%%time\n2+2" result = format_cell(src, fast=True, mode=JUPYTER_MODE) -@@ -413,6 +428,45 @@ def test_entire_notebook_no_trailing_new +@@ -429,6 +444,45 @@ def test_entire_notebook_no_trailing_new assert result == expected @@ -1420,7 +1446,7 @@ def test_entire_notebook_without_changes() -> None: content = read_jupyter_notebook("jupyter", "notebook_without_changes") with pytest.raises(NothingChanged): -@@ -464,6 +518,29 @@ def test_ipynb_diff_with_no_change() -> +@@ -480,6 +534,29 @@ def test_ipynb_diff_with_no_change() -> assert expected in result.output @@ -1484,22 +1510,6 @@ @contextmanager ---- a/tox.ini -+++ b/tox.ini -@@ -83,12 +83,4 @@ setenv = PYTHONPATH = {toxinidir}/src - skip_install = True - commands = - pip install -e . -- black --check {toxinidir}/src {toxinidir}/tests {toxinidir}/docs {toxinidir}/scripts -- --[testenv:generate_schema] --setenv = PYTHONWARNDEFAULTENCODING = --skip_install = True --deps = --commands = -- pip install -e . -- python {toxinidir}/scripts/generate_schema.py --outfile {toxinidir}/src/black/resources/black.schema.json -+ pyink --check {toxinidir}/src {toxinidir}/tests {toxinidir}/docs {toxinidir}/scripts --- a/trans.py +++ b/trans.py @@ -12,8 +12,8 @@ from typing import Any, ClassVar, Final, @@ -1513,7 +1523,7 @@ from pyink.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, -@@ -233,9 +233,18 @@ class StringTransformer(ABC): +@@ -237,9 +237,18 @@ class StringTransformer(ABC): # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with # `abc.ABC`. @@ -1533,7 +1543,7 @@ @abstractmethod def do_match(self, line: Line) -> TMatchResult: -@@ -715,7 +724,9 @@ class StringMerger(StringTransformer, Cu +@@ -719,7 +728,9 @@ class StringMerger(StringTransformer, Cu S_leaf = Leaf(token.STRING, S) if self.normalize_strings: @@ -1544,7 +1554,7 @@ # Fill the 'custom_splits' list with the appropriate CustomSplit objects. temp_string = S_leaf.value[len(prefix) + 1 : -1] -@@ -833,7 +844,7 @@ class StringMerger(StringTransformer, Cu +@@ -837,7 +848,7 @@ class StringMerger(StringTransformer, Cu if id(leaf) in line.comments: num_of_inline_string_comments += 1 @@ -1553,7 +1563,7 @@ return TErr("Cannot merge strings which have pragma comments.") if num_of_strings < 2: -@@ -977,7 +988,13 @@ class StringParenStripper(StringTransfor +@@ -981,7 +992,13 @@ class StringParenStripper(StringTransfor idx += 1 if string_indices: @@ -1568,7 +1578,7 @@ return TErr("This line has no strings wrapped in parens.") def do_transform( -@@ -1139,7 +1156,7 @@ class BaseStringSplitter(StringTransform +@@ -1143,7 +1160,7 @@ class BaseStringSplitter(StringTransform ) if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment( @@ -1577,7 +1587,7 @@ ): return TErr( "Line appears to end with an inline pragma comment. Splitting the line" -@@ -1181,7 +1198,7 @@ class BaseStringSplitter(StringTransform +@@ -1185,7 +1202,7 @@ class BaseStringSplitter(StringTransform # NN: The leaf that is after N. # WMA4 the whitespace at the beginning of the line. @@ -1586,7 +1596,7 @@ if is_valid_index(string_idx - 1): p_idx = string_idx - 1 -@@ -1535,7 +1552,7 @@ class StringSplitter(BaseStringSplitter, +@@ -1553,7 +1570,7 @@ class StringSplitter(BaseStringSplitter, characters expand to two columns). """ result = self.line_length @@ -1595,7 +1605,7 @@ result -= 1 if ends_with_comma else 0 result -= string_op_leaves_length return result -@@ -1546,11 +1563,11 @@ class StringSplitter(BaseStringSplitter, +@@ -1564,11 +1581,11 @@ class StringSplitter(BaseStringSplitter, # The last index of a string of length N is N-1. max_break_width -= 1 # Leading whitespace is not present in the string value (e.g. Leaf.value). @@ -1609,7 +1619,7 @@ ) return -@@ -1847,7 +1864,9 @@ class StringSplitter(BaseStringSplitter, +@@ -1865,7 +1882,9 @@ class StringSplitter(BaseStringSplitter, def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None: if self.normalize_strings: @@ -1620,7 +1630,7 @@ def _normalize_f_string(self, string: str, prefix: str) -> str: """ -@@ -1970,7 +1989,8 @@ class StringParenWrapper(BaseStringSplit +@@ -1993,7 +2012,8 @@ class StringParenWrapper(BaseStringSplit char == " " or char in SPLIT_SAFE_CHARS for char in string_value ): # And will still violate the line length limit when split... @@ -1630,7 +1640,7 @@ if str_width(string_value) > max_string_width: # And has no associated custom splits... if not self.has_custom_splits(string_value): -@@ -2216,7 +2236,7 @@ class StringParenWrapper(BaseStringSplit +@@ -2265,7 +2285,7 @@ class StringParenWrapper(BaseStringSplit string_value = LL[string_idx].value string_line = Line( mode=line.mode, @@ -1639,24 +1649,7 @@ inside_brackets=True, should_split_rhs=line.should_split_rhs, magic_trailing_comma=line.magic_trailing_comma, ---- a/tests/data/cases/torture.py -+++ b/tests/data/cases/torture.py -@@ -57,9 +57,9 @@ importA - class A: - def foo(self): - for _ in range(10): -- aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( -+ aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( # pylint: disable=no-member - xxxxxxxxxxxx -- ) # pylint: disable=no-member -+ ) + ) - def test(self, othr): ---- a/test_requirements.txt -+++ b/test_requirements.txt -@@ -1,3 +1,4 @@ -+click == 8.1.8 - coverage >= 5.3 - pre-commit - pytest >= 7 + def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: diff --git a/pyproject.toml b/pyproject.toml index 2bec046d936..7521612fc1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ # Yes, we use the _Black_ style to format _Pyink_ code. pyink = false line-length = 88 -target-version = ['py310'] +target-version = ["py310"] include = '\.pyi?$' extend-exclude = 'tests/data' unstable = true @@ -20,40 +20,73 @@ requires-python = ">=3.10" readme = "README.md" authors = [{name = "The Pyink Maintainers", email = "pyink-maintainers@google.com"}] classifiers = [ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development :: Quality Assurance", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", ] dependencies = [ - "click>=8.0.0", - "mypy_extensions>=0.4.3", - "packaging>=22.0", - "pathspec>=0.9.0,<1.0.0", - "platformdirs>=2", - "pytokens>=0.3.0", - "tomli>=1.1.0; python_version < '3.11'", - "typing_extensions>=4.0.1; python_version < '3.11'", - "black==25.12.0", + "click>=8.0.0", + "mypy-extensions>=0.4.3", + "packaging>=22.0", + "pathspec>=1.0.0", + "platformdirs>=2", + "pytokens~=0.4.0", + "tomli>=1.1.0; python_version<'3.11'", + "typing-extensions>=4.0.1; python_version<'3.11'", + "black==26.3.1", ] dynamic = ["version"] [project.optional-dependencies] colorama = ["colorama>=0.4.3"] -uvloop = ["uvloop>=0.15.2"] -jupyter = [ - "ipython>=7.8.0", - "tokenize-rt>=3.2.0", +uvloop = [ + "uvloop>=0.15.2; sys_platform != 'win32'", + "winloop>=0.5.0; sys_platform == 'win32'" ] +d = ["aiohttp>=3.10"] +jupyter = ["ipython>=7.8.0", "tokenize-rt>=3.2.0"] + +[dependency-groups] +build = ["hatch==1.15.1", "hatch-fancy-pypi-readme", "hatch-vcs>=0.3.0", "virtualenv<21.0.0"] +wheels = ["cibuildwheel==3.3.1", "pypyp"] +binary = ["pyinstaller", "wheel>=0.45.1"] + +dev = [{ include-group = "cov-tests" }, { include-group = "tox" }, "pre-commit"] +cov-tests = [ + { include-group = "coverage" }, + { include-group = "tests" }, + "pytest-cov>=4.1.0", +] +docs = [ + "docutils==0.21.2", + "furo==2025.12.19", + "myst-parser==4.0.1", + "sphinx-copybutton==0.5.2", + "sphinx==8.2.3", + "sphinxcontrib-programoutput==0.19", +] + +tox = ["tox>=4.22"] +tests = ["pytest>=7", "pytest-xdist>=3.0.2"] +coverage = ["coverage>=5.3"] + +fuzz = [{ include-group = "coverage" }, "hypothesis", "hypothesmith"] +diff-shades = [ + "diff-shades @ https://github.com/ichard26/diff-shades/archive/stable.zip", +] +diff-shades-comment = ["click>=8.1.7", "packaging>=22.0", "urllib3"] + +width-table = ["wcwidth==0.2.14"] [project.scripts] pyink = "pyink:patched_main" @@ -71,9 +104,9 @@ source = "vcs" [tool.hatch.build.hooks.vcs] version-file = "src/_pyink_version.py" -template = ''' +template = """ version = "{version}" -''' +""" [tool.hatch.build.targets.wheel] only-include = ["src"] @@ -85,10 +118,34 @@ macos-max-compat = true # Option below requires `tests/optional.py` addopts = "--strict-config --strict-markers" optional-tests = [ - "no_jupyter: run when `jupyter` extra NOT installed", -] -markers = [ - "incompatible_with_mypyc: run when testing mypyc compiled black" + "no_jupyter: run when `jupyter` extra NOT installed", ] +markers = ["incompatible_with_mypyc: run when testing mypyc compiled black"] xfail_strict = true filterwarnings = ["error"] + + +[tool.mypy] +# Specify the target platform details in config, so your developers are +# free to run mypy on Windows, Linux, or macOS and get consistent +# results. +python_version = "3.10" +mypy_path = "src" +strict = true +strict_bytes = true +local_partial_types = true +# Unreachable blocks have been an issue when compiling mypyc, let's try to avoid 'em in the first place. +warn_unreachable = true +implicit_reexport = true +show_error_codes = true +show_column_numbers = true + +[[tool.mypy.overrides]] +module = ["pathspec.*", "IPython.*", "colorama.*", "tokenize_rt.*", "uvloop.*"] +ignore_missing_imports = true + +# CI only checks src/, but in case users are running LSP or similar we explicitly ignore +# errors in test data files. +[[tool.mypy.overrides]] +module = ["tests.data.*"] +ignore_errors = true diff --git a/src/pyink/__init__.py b/src/pyink/__init__.py index c41796d4cb1..caf0a60a18c 100644 --- a/src/pyink/__init__.py +++ b/src/pyink/__init__.py @@ -25,8 +25,8 @@ import click from click.core import ParameterSource from mypy_extensions import mypyc_attr -from pathspec import PathSpec -from pathspec.patterns.gitwildmatch import GitWildMatchPatternError +from pathspec import GitIgnoreSpec +from pathspec.patterns.gitignore import GitIgnorePatternError from _pyink_version import version as __version__ from pyink.cache import Cache @@ -209,6 +209,27 @@ def target_version_option_callback( return [TargetVersion[val.upper()] for val in v] +def _target_versions_exceed_runtime( + target_versions: set[TargetVersion], +) -> bool: + if not target_versions: + return False + max_target_minor = max(tv.value for tv in target_versions) + return max_target_minor > sys.version_info[1] + + +def _version_mismatch_message(target_versions: set[TargetVersion]) -> str: + max_target = max(target_versions, key=lambda tv: tv.value) + runtime = f"{sys.version_info[0]}.{sys.version_info[1]}" + return ( + f"Python {runtime} cannot parse code formatted for" + f" {max_target.pretty()}. To fix this: run Black with" + f" {max_target.pretty()}, set --target-version to" + f" py3{sys.version_info[1]}, or use --fast to skip the safety" + " check." + ) + + def enable_unstable_feature_callback( c: click.Context, p: click.Option | click.Parameter, v: tuple[str, ...] ) -> list[Preview]: @@ -719,6 +740,14 @@ def main( ), ) + if not fast and _target_versions_exceed_runtime(versions): + err( + f"Warning: {_version_mismatch_message(versions)} Black's safety" + " check verifies equivalence by parsing the AST, which fails" + " when the running Python is older than the target version.", + fg="yellow", + ) + lines: list[tuple[int, int]] = [] if line_ranges: if ipynb: @@ -762,7 +791,7 @@ def main( report=report, stdin_filename=stdin_filename, ) - except GitWildMatchPatternError: + except GitIgnorePatternError: ctx.exit(1) if not sources: @@ -826,7 +855,7 @@ def get_sources( assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" using_default_exclude = exclude is None exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude - gitignore: dict[Path, PathSpec] | None = None + gitignore: dict[Path, GitIgnoreSpec] | None = None root_gitignore = get_gitignore(root) for s in src: @@ -1087,10 +1116,8 @@ def format_stdin_to_stdout( if content is None: src, encoding, newline = decode_bytes(sys.stdin.buffer.read(), mode) - elif Preview.normalize_cr_newlines in mode: - src, encoding, newline = content, "utf-8", "\n" else: - src, encoding, newline = content, "utf-8", "" + src, encoding, newline = content, "utf-8", "\n" dst = src try: @@ -1106,12 +1133,8 @@ def format_stdin_to_stdout( ) if write_back == WriteBack.YES: # Make sure there's a newline after the content - if Preview.normalize_cr_newlines in mode: - if dst and dst[-1] != "\n" and dst[-1] != "\r": - dst += newline - else: - if dst and dst[-1] != "\n": - dst += "\n" + if dst and dst[-1] != "\n" and dst[-1] != "\r": + dst += newline f.write(dst) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): now = datetime.now(timezone.utc) @@ -1138,7 +1161,15 @@ def check_stability_and_equivalence( equivalent, or if a second pass of the formatter would format the content differently. """ - assert_equivalent(src_contents, dst_contents) + try: + assert_equivalent(src_contents, dst_contents) + except ASTSafetyError: + if _target_versions_exceed_runtime(mode.target_versions): + raise ASTSafetyError( + "failed to verify equivalence of the formatted output:" + f" {_version_mismatch_message(mode.target_versions)}" + ) from None + raise assert_stable(src_contents, dst_contents, mode=mode, lines=lines) @@ -1314,16 +1345,15 @@ def f( def _format_str_once( src_contents: str, *, mode: Mode, lines: Collection[tuple[int, int]] = () ) -> str: - if Preview.normalize_cr_newlines in mode: - normalized_contents, _, newline_type = decode_bytes( - src_contents.encode("utf-8"), mode - ) + # Use the encoding overwrite since the src_contents may contain a different + # magic encoding comment than utf-8 + normalized_contents, _, newline_type = decode_bytes( + src_contents.encode("utf-8"), mode, encoding_overwrite="utf-8" + ) - src_node = lib2to3_parse( - normalized_contents.lstrip(), target_versions=mode.target_versions - ) - else: - src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) + src_node = lib2to3_parse( + normalized_contents.lstrip(), target_versions=mode.target_versions + ) dst_blocks: list[LinesBlock] = [] if mode.target_versions: @@ -1372,53 +1402,48 @@ def _format_str_once( for block in dst_blocks: dst_contents.extend(block.all_lines()) if not dst_contents: - if Preview.normalize_cr_newlines in mode: - if "\n" in normalized_contents: - return newline_type - else: - # Use decode_bytes to retrieve the correct source newline (CRLF or LF), - # and check if normalized_content has more than one line - normalized_content, _, newline = decode_bytes( - src_contents.encode("utf-8"), mode - ) - if "\n" in normalized_content: - return newline - return "" - if Preview.normalize_cr_newlines in mode: - return "".join(dst_contents).replace("\n", newline_type) - else: - return "".join(dst_contents) + if "\n" in normalized_contents: + return newline_type + return "".join(dst_contents).replace("\n", newline_type) -def decode_bytes(src: bytes, mode: Mode) -> tuple[FileContent, Encoding, NewLine]: +def decode_bytes( + src: bytes, mode: Mode, *, encoding_overwrite: str | None = None +) -> tuple[FileContent, Encoding, NewLine]: """Return a tuple of (decoded_contents, encoding, newline). - `newline` is either CRLF or LF but `decoded_contents` is decoded with + `newline` is either CRLF, LF, or CR, but `decoded_contents` is decoded with universal newlines (i.e. only contains LF). + + Use the keyword only encoding_overwrite argument if the bytes are encoded + differently to their possible encoding magic comment. """ srcbuf = io.BytesIO(src) + + # Still use detect encoding even if overrite set because otherwise lines + # might be different encoding, lines = tokenize.detect_encoding(srcbuf.readline) + if encoding_overwrite is not None: + encoding = encoding_overwrite + if not lines: return "", encoding, "\n" - if Preview.normalize_cr_newlines in mode: - if lines[0][-2:] == b"\r\n": - if b"\r" in lines[0][:-2]: - newline = "\r" - else: - newline = "\r\n" - elif lines[0][-1:] == b"\n": - if b"\r" in lines[0][:-1]: - newline = "\r" - else: - newline = "\n" + if lines[0][-2:] == b"\r\n": + if b"\r" in lines[0][:-2]: + newline = "\r" else: - if b"\r" in lines[0]: - newline = "\r" - else: - newline = "\n" + newline = "\r\n" + elif lines[0][-1:] == b"\n": + if b"\r" in lines[0][:-1]: + newline = "\r" + else: + newline = "\n" else: - newline = "\r\n" if lines[0][-2:] == b"\r\n" else "\n" + if b"\r" in lines[0]: + newline = "\r" + else: + newline = "\n" srcbuf.seek(0) with io.TextIOWrapper(srcbuf, encoding) as tiow: diff --git a/src/pyink/_width_table.py b/src/pyink/_width_table.py index 12fd15d6e6c..c2e70a25886 100644 --- a/src/pyink/_width_table.py +++ b/src/pyink/_width_table.py @@ -1,182 +1,10 @@ # Generated by make_width_table.py -# wcwidth 0.2.6 -# Unicode 15.0.0 +# wcwidth 0.2.14 +# Unicode 17.0.0 from typing import Final WIDTH_TABLE: Final[tuple[tuple[int, int, int], ...]] = ( - (0, 0, 0), - (1, 31, -1), - (127, 159, -1), - (768, 879, 0), - (1155, 1161, 0), - (1425, 1469, 0), - (1471, 1471, 0), - (1473, 1474, 0), - (1476, 1477, 0), - (1479, 1479, 0), - (1552, 1562, 0), - (1611, 1631, 0), - (1648, 1648, 0), - (1750, 1756, 0), - (1759, 1764, 0), - (1767, 1768, 0), - (1770, 1773, 0), - (1809, 1809, 0), - (1840, 1866, 0), - (1958, 1968, 0), - (2027, 2035, 0), - (2045, 2045, 0), - (2070, 2073, 0), - (2075, 2083, 0), - (2085, 2087, 0), - (2089, 2093, 0), - (2137, 2139, 0), - (2200, 2207, 0), - (2250, 2273, 0), - (2275, 2306, 0), - (2362, 2362, 0), - (2364, 2364, 0), - (2369, 2376, 0), - (2381, 2381, 0), - (2385, 2391, 0), - (2402, 2403, 0), - (2433, 2433, 0), - (2492, 2492, 0), - (2497, 2500, 0), - (2509, 2509, 0), - (2530, 2531, 0), - (2558, 2558, 0), - (2561, 2562, 0), - (2620, 2620, 0), - (2625, 2626, 0), - (2631, 2632, 0), - (2635, 2637, 0), - (2641, 2641, 0), - (2672, 2673, 0), - (2677, 2677, 0), - (2689, 2690, 0), - (2748, 2748, 0), - (2753, 2757, 0), - (2759, 2760, 0), - (2765, 2765, 0), - (2786, 2787, 0), - (2810, 2815, 0), - (2817, 2817, 0), - (2876, 2876, 0), - (2879, 2879, 0), - (2881, 2884, 0), - (2893, 2893, 0), - (2901, 2902, 0), - (2914, 2915, 0), - (2946, 2946, 0), - (3008, 3008, 0), - (3021, 3021, 0), - (3072, 3072, 0), - (3076, 3076, 0), - (3132, 3132, 0), - (3134, 3136, 0), - (3142, 3144, 0), - (3146, 3149, 0), - (3157, 3158, 0), - (3170, 3171, 0), - (3201, 3201, 0), - (3260, 3260, 0), - (3263, 3263, 0), - (3270, 3270, 0), - (3276, 3277, 0), - (3298, 3299, 0), - (3328, 3329, 0), - (3387, 3388, 0), - (3393, 3396, 0), - (3405, 3405, 0), - (3426, 3427, 0), - (3457, 3457, 0), - (3530, 3530, 0), - (3538, 3540, 0), - (3542, 3542, 0), - (3633, 3633, 0), - (3636, 3642, 0), - (3655, 3662, 0), - (3761, 3761, 0), - (3764, 3772, 0), - (3784, 3790, 0), - (3864, 3865, 0), - (3893, 3893, 0), - (3895, 3895, 0), - (3897, 3897, 0), - (3953, 3966, 0), - (3968, 3972, 0), - (3974, 3975, 0), - (3981, 3991, 0), - (3993, 4028, 0), - (4038, 4038, 0), - (4141, 4144, 0), - (4146, 4151, 0), - (4153, 4154, 0), - (4157, 4158, 0), - (4184, 4185, 0), - (4190, 4192, 0), - (4209, 4212, 0), - (4226, 4226, 0), - (4229, 4230, 0), - (4237, 4237, 0), - (4253, 4253, 0), (4352, 4447, 2), - (4957, 4959, 0), - (5906, 5908, 0), - (5938, 5939, 0), - (5970, 5971, 0), - (6002, 6003, 0), - (6068, 6069, 0), - (6071, 6077, 0), - (6086, 6086, 0), - (6089, 6099, 0), - (6109, 6109, 0), - (6155, 6157, 0), - (6159, 6159, 0), - (6277, 6278, 0), - (6313, 6313, 0), - (6432, 6434, 0), - (6439, 6440, 0), - (6450, 6450, 0), - (6457, 6459, 0), - (6679, 6680, 0), - (6683, 6683, 0), - (6742, 6742, 0), - (6744, 6750, 0), - (6752, 6752, 0), - (6754, 6754, 0), - (6757, 6764, 0), - (6771, 6780, 0), - (6783, 6783, 0), - (6832, 6862, 0), - (6912, 6915, 0), - (6964, 6964, 0), - (6966, 6970, 0), - (6972, 6972, 0), - (6978, 6978, 0), - (7019, 7027, 0), - (7040, 7041, 0), - (7074, 7077, 0), - (7080, 7081, 0), - (7083, 7085, 0), - (7142, 7142, 0), - (7144, 7145, 0), - (7149, 7149, 0), - (7151, 7153, 0), - (7212, 7219, 0), - (7222, 7223, 0), - (7376, 7378, 0), - (7380, 7392, 0), - (7394, 7400, 0), - (7405, 7405, 0), - (7412, 7412, 0), - (7416, 7417, 0), - (7616, 7679, 0), - (8203, 8207, 0), - (8232, 8238, 0), - (8288, 8291, 0), - (8400, 8432, 0), (8986, 8987, 2), (9001, 9002, 2), (9193, 9196, 2), @@ -184,8 +12,10 @@ (9203, 9203, 2), (9725, 9726, 2), (9748, 9749, 2), + (9776, 9783, 2), (9800, 9811, 2), (9855, 9855, 2), + (9866, 9871, 2), (9875, 9875, 2), (9889, 9889, 2), (9898, 9899, 2), @@ -211,186 +41,34 @@ (11035, 11036, 2), (11088, 11088, 2), (11093, 11093, 2), - (11503, 11505, 0), - (11647, 11647, 0), - (11744, 11775, 0), (11904, 11929, 2), (11931, 12019, 2), (12032, 12245, 2), - (12272, 12283, 2), - (12288, 12329, 2), - (12330, 12333, 0), - (12334, 12350, 2), + (12272, 12329, 2), + (12336, 12350, 2), (12353, 12438, 2), - (12441, 12442, 0), (12443, 12543, 2), (12549, 12591, 2), (12593, 12686, 2), - (12688, 12771, 2), - (12784, 12830, 2), + (12688, 12773, 2), + (12783, 12830, 2), (12832, 12871, 2), - (12880, 19903, 2), - (19968, 42124, 2), + (12880, 42124, 2), (42128, 42182, 2), - (42607, 42610, 0), - (42612, 42621, 0), - (42654, 42655, 0), - (42736, 42737, 0), - (43010, 43010, 0), - (43014, 43014, 0), - (43019, 43019, 0), - (43045, 43046, 0), - (43052, 43052, 0), - (43204, 43205, 0), - (43232, 43249, 0), - (43263, 43263, 0), - (43302, 43309, 0), - (43335, 43345, 0), (43360, 43388, 2), - (43392, 43394, 0), - (43443, 43443, 0), - (43446, 43449, 0), - (43452, 43453, 0), - (43493, 43493, 0), - (43561, 43566, 0), - (43569, 43570, 0), - (43573, 43574, 0), - (43587, 43587, 0), - (43596, 43596, 0), - (43644, 43644, 0), - (43696, 43696, 0), - (43698, 43700, 0), - (43703, 43704, 0), - (43710, 43711, 0), - (43713, 43713, 0), - (43756, 43757, 0), - (43766, 43766, 0), - (44005, 44005, 0), - (44008, 44008, 0), - (44013, 44013, 0), (44032, 55203, 2), (63744, 64255, 2), - (64286, 64286, 0), - (65024, 65039, 0), (65040, 65049, 2), - (65056, 65071, 0), (65072, 65106, 2), (65108, 65126, 2), (65128, 65131, 2), (65281, 65376, 2), (65504, 65510, 2), - (66045, 66045, 0), - (66272, 66272, 0), - (66422, 66426, 0), - (68097, 68099, 0), - (68101, 68102, 0), - (68108, 68111, 0), - (68152, 68154, 0), - (68159, 68159, 0), - (68325, 68326, 0), - (68900, 68903, 0), - (69291, 69292, 0), - (69373, 69375, 0), - (69446, 69456, 0), - (69506, 69509, 0), - (69633, 69633, 0), - (69688, 69702, 0), - (69744, 69744, 0), - (69747, 69748, 0), - (69759, 69761, 0), - (69811, 69814, 0), - (69817, 69818, 0), - (69826, 69826, 0), - (69888, 69890, 0), - (69927, 69931, 0), - (69933, 69940, 0), - (70003, 70003, 0), - (70016, 70017, 0), - (70070, 70078, 0), - (70089, 70092, 0), - (70095, 70095, 0), - (70191, 70193, 0), - (70196, 70196, 0), - (70198, 70199, 0), - (70206, 70206, 0), - (70209, 70209, 0), - (70367, 70367, 0), - (70371, 70378, 0), - (70400, 70401, 0), - (70459, 70460, 0), - (70464, 70464, 0), - (70502, 70508, 0), - (70512, 70516, 0), - (70712, 70719, 0), - (70722, 70724, 0), - (70726, 70726, 0), - (70750, 70750, 0), - (70835, 70840, 0), - (70842, 70842, 0), - (70847, 70848, 0), - (70850, 70851, 0), - (71090, 71093, 0), - (71100, 71101, 0), - (71103, 71104, 0), - (71132, 71133, 0), - (71219, 71226, 0), - (71229, 71229, 0), - (71231, 71232, 0), - (71339, 71339, 0), - (71341, 71341, 0), - (71344, 71349, 0), - (71351, 71351, 0), - (71453, 71455, 0), - (71458, 71461, 0), - (71463, 71467, 0), - (71727, 71735, 0), - (71737, 71738, 0), - (71995, 71996, 0), - (71998, 71998, 0), - (72003, 72003, 0), - (72148, 72151, 0), - (72154, 72155, 0), - (72160, 72160, 0), - (72193, 72202, 0), - (72243, 72248, 0), - (72251, 72254, 0), - (72263, 72263, 0), - (72273, 72278, 0), - (72281, 72283, 0), - (72330, 72342, 0), - (72344, 72345, 0), - (72752, 72758, 0), - (72760, 72765, 0), - (72767, 72767, 0), - (72850, 72871, 0), - (72874, 72880, 0), - (72882, 72883, 0), - (72885, 72886, 0), - (73009, 73014, 0), - (73018, 73018, 0), - (73020, 73021, 0), - (73023, 73029, 0), - (73031, 73031, 0), - (73104, 73105, 0), - (73109, 73109, 0), - (73111, 73111, 0), - (73459, 73460, 0), - (73472, 73473, 0), - (73526, 73530, 0), - (73536, 73536, 0), - (73538, 73538, 0), - (78912, 78912, 0), - (78919, 78933, 0), - (92912, 92916, 0), - (92976, 92982, 0), - (94031, 94031, 0), - (94095, 94098, 0), (94176, 94179, 2), - (94180, 94180, 0), - (94192, 94193, 2), - (94208, 100343, 2), - (100352, 101589, 2), - (101632, 101640, 2), + (94194, 94198, 2), + (94208, 101589, 2), + (101631, 101662, 2), + (101760, 101874, 2), (110576, 110579, 2), (110581, 110587, 2), (110589, 110590, 2), @@ -400,32 +78,8 @@ (110933, 110933, 2), (110948, 110951, 2), (110960, 111355, 2), - (113821, 113822, 0), - (118528, 118573, 0), - (118576, 118598, 0), - (119143, 119145, 0), - (119163, 119170, 0), - (119173, 119179, 0), - (119210, 119213, 0), - (119362, 119364, 0), - (121344, 121398, 0), - (121403, 121452, 0), - (121461, 121461, 0), - (121476, 121476, 0), - (121499, 121503, 0), - (121505, 121519, 0), - (122880, 122886, 0), - (122888, 122904, 0), - (122907, 122913, 0), - (122915, 122916, 0), - (122918, 122922, 0), - (123023, 123023, 0), - (123184, 123190, 0), - (123566, 123566, 0), - (123628, 123631, 0), - (124140, 124143, 0), - (125136, 125142, 0), - (125252, 125258, 0), + (119552, 119638, 2), + (119648, 119670, 2), (126980, 126980, 2), (127183, 127183, 2), (127374, 127374, 2), @@ -443,7 +97,8 @@ (127951, 127955, 2), (127968, 127984, 2), (127988, 127988, 2), - (127992, 128062, 2), + (127992, 127994, 2), + (128000, 128062, 2), (128064, 128064, 2), (128066, 128252, 2), (128255, 128317, 2), @@ -456,7 +111,7 @@ (128640, 128709, 2), (128716, 128716, 2), (128720, 128722, 2), - (128725, 128727, 2), + (128725, 128728, 2), (128732, 128735, 2), (128747, 128748, 2), (128756, 128764, 2), @@ -466,13 +121,12 @@ (129340, 129349, 2), (129351, 129535, 2), (129648, 129660, 2), - (129664, 129672, 2), - (129680, 129725, 2), - (129727, 129733, 2), - (129742, 129755, 2), - (129760, 129768, 2), - (129776, 129784, 2), + (129664, 129674, 2), + (129678, 129734, 2), + (129736, 129736, 2), + (129741, 129756, 2), + (129759, 129770, 2), + (129775, 129784, 2), (131072, 196605, 2), (196608, 262141, 2), - (917760, 917999, 0), ) diff --git a/src/pyink/comments.py b/src/pyink/comments.py index 4154ee8298a..3d142eb2d14 100644 --- a/src/pyink/comments.py +++ b/src/pyink/comments.py @@ -4,8 +4,7 @@ from functools import lru_cache from typing import Final, Union -from pyink import ink_comments -from pyink.mode import Mode, Preview +from pyink.mode import Mode from pyink.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -21,6 +20,8 @@ from blib2to3.pgen2 import token from blib2to3.pytree import Leaf, Node +from pyink import ink_comments + # types LN = Union[Leaf, Node] @@ -177,8 +178,7 @@ def make_comment(content: str, mode: Mode) -> str: ): content = " " + content[1:] # Replace NBSP by a simple space if ( - Preview.standardize_type_comments in mode - and content + content and "\N{NO-BREAK SPACE}" not in content and is_type_comment_string("#" + content, mode=mode) ): @@ -645,23 +645,12 @@ def _generate_ignored_nodes_from_fmt_skip( if not comments or comment.value != comments[0].value: return - if Preview.fix_fmt_skip_in_one_liners in mode and not prev_sibling and parent: + if not prev_sibling and parent: prev_sibling = parent.prev_sibling if prev_sibling is not None: leaf.prefix = leaf.prefix[comment.consumed :] - if Preview.fix_fmt_skip_in_one_liners not in mode: - siblings = [prev_sibling] - while ( - "\n" not in prev_sibling.prefix - and prev_sibling.prev_sibling is not None - ): - prev_sibling = prev_sibling.prev_sibling - siblings.insert(0, prev_sibling) - yield from siblings - return - # Generates the nodes to be ignored by `fmt: skip`. # Nodes to ignore are the ones on the same line as the @@ -740,7 +729,7 @@ def _generate_ignored_nodes_from_fmt_skip( current_node = current_node.parent # Special handling for compound statements with semicolon-separated bodies - if Preview.fix_fmt_skip_in_one_liners in mode and isinstance(parent, Node): + if isinstance(parent, Node): body_node = _find_compound_statement_context(parent) if body_node is not None: header_nodes = _get_compound_statement_header(body_node, parent) diff --git a/src/pyink/concurrency.py b/src/pyink/concurrency.py index 79c99ef4783..794430ebd54 100644 --- a/src/pyink/concurrency.py +++ b/src/pyink/concurrency.py @@ -27,18 +27,24 @@ from pyink.report import Changed, Report -def maybe_install_uvloop() -> None: - """If our environment has uvloop installed we use it. +def maybe_use_uvloop() -> asyncio.AbstractEventLoop: + """If our environment has uvloop or winloop installed we use it otherwise + a normal asyncio eventloop is called as fallback. This is called only from command-line entry points to avoid interfering with the parent process if Black is used as a library. """ try: - import uvloop + if sys.platform != "win32": + import uvloop - uvloop.install() + return uvloop.new_event_loop() + else: + import winloop + + return winloop.new_event_loop() except ImportError: - pass + return asyncio.new_event_loop() def cancel(tasks: Iterable[asyncio.Future[Any]]) -> None: @@ -81,7 +87,6 @@ def reformat_many( no_cache: bool = False, ) -> None: """Reformat multiple files using a ProcessPoolExecutor.""" - maybe_install_uvloop() if workers is None: workers = int(os.environ.get("PYINK_NUM_WORKERS", 0)) @@ -89,6 +94,12 @@ def reformat_many( if sys.platform == "win32": # Work around https://bugs.python.org/issue26903 workers = min(workers, 60) + if getattr(sys, "frozen", False): + # In frozen builds (e.g. PyInstaller), avoid spawning worker processes (i.e. + # avoid using ProcessPoolExecutor) to prevent shutdown errors when workers + # try to import modules after cleanup begins. + # See https://github.com/psf/black/issues/4823 + workers = 1 executor: Executor | None = None if workers > 1: @@ -104,7 +115,7 @@ def reformat_many( if executor is None: executor = ThreadPoolExecutor(max_workers=1) - loop = asyncio.new_event_loop() + loop = maybe_use_uvloop() asyncio.set_event_loop(loop) try: loop.run_until_complete( @@ -159,46 +170,52 @@ async def schedule_formatting( cancelled = [] sources_to_cache = [] lock = None + manager = None if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): # For diff output, we need locks to ensure we don't interleave output # from different processes. manager = Manager() lock = manager.Lock() - tasks = { - asyncio.ensure_future( - loop.run_in_executor( - executor, format_file_in_place, src, fast, mode, write_back, lock - ) - ): src - for src in sorted(sources) - } - pending = tasks.keys() + try: - loop.add_signal_handler(signal.SIGINT, cancel, pending) - loop.add_signal_handler(signal.SIGTERM, cancel, pending) - except NotImplementedError: - # There are no good alternatives for these on Windows. - pass - while pending: - done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) - for task in done: - src = tasks.pop(task) - if task.cancelled(): - cancelled.append(task) - elif exc := task.exception(): - if report.verbose: - traceback.print_exception(type(exc), exc, exc.__traceback__) - report.failed(src, str(exc)) - else: - changed = Changed.YES if task.result() else Changed.NO - # If the file was written back or was successfully checked as - # well-formatted, store this information in the cache. - if write_back is WriteBack.YES or ( - write_back is WriteBack.CHECK and changed is Changed.NO - ): - sources_to_cache.append(src) - report.done(src, changed) - if cancelled: - await asyncio.gather(*cancelled, return_exceptions=True) - if sources_to_cache and not no_cache and cache is not None: - cache.write(sources_to_cache) + tasks = { + asyncio.ensure_future( + loop.run_in_executor( + executor, format_file_in_place, src, fast, mode, write_back, lock + ) + ): src + for src in sorted(sources) + } + pending = tasks.keys() + try: + loop.add_signal_handler(signal.SIGINT, cancel, pending) + loop.add_signal_handler(signal.SIGTERM, cancel, pending) + except NotImplementedError: + # There are no good alternatives for these on Windows. + pass + while pending: + done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + for task in done: + src = tasks.pop(task) + if task.cancelled(): + cancelled.append(task) + elif exc := task.exception(): + if report.verbose: + traceback.print_exception(type(exc), exc, exc.__traceback__) + report.failed(src, str(exc)) + else: + changed = Changed.YES if task.result() else Changed.NO + # If the file was written back or was successfully checked as + # well-formatted, store this information in the cache. + if write_back is WriteBack.YES or ( + write_back is WriteBack.CHECK and changed is Changed.NO + ): + sources_to_cache.append(src) + report.done(src, changed) + if cancelled: + await asyncio.gather(*cancelled, return_exceptions=True) + if sources_to_cache and not no_cache and cache is not None: + cache.write(sources_to_cache) + finally: + if manager is not None: + manager.shutdown() diff --git a/src/pyink/files.py b/src/pyink/files.py index 4cb56e16c27..46e7cc4ea9c 100644 --- a/src/pyink/files.py +++ b/src/pyink/files.py @@ -10,8 +10,8 @@ from mypy_extensions import mypyc_attr from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet from packaging.version import InvalidVersion, Version -from pathspec import PathSpec -from pathspec.patterns.gitwildmatch import GitWildMatchPatternError +from pathspec import GitIgnoreSpec +from pathspec.patterns.gitignore import GitIgnorePatternError if sys.version_info >= (3, 11): try: @@ -238,16 +238,16 @@ def find_user_pyproject_toml() -> Path: @lru_cache -def get_gitignore(root: Path) -> PathSpec: - """Return a PathSpec matching gitignore content if present.""" +def get_gitignore(root: Path) -> GitIgnoreSpec: + """Return a GitIgnoreSpec matching gitignore content if present.""" gitignore = root / ".gitignore" lines: list[str] = [] if gitignore.is_file(): with gitignore.open(encoding="utf-8") as gf: lines = gf.readlines() try: - return PathSpec.from_lines("gitwildmatch", lines) - except GitWildMatchPatternError as e: + return GitIgnoreSpec.from_lines(lines) + except GitIgnorePatternError as e: err(f"Could not parse {gitignore}: {e}") raise @@ -292,7 +292,7 @@ def best_effort_relative_path(path: Path, root: Path) -> Path: def _path_is_ignored( root_relative_path: str, root: Path, - gitignore_dict: dict[Path, PathSpec], + gitignore_dict: dict[Path, GitIgnoreSpec], ) -> bool: path = root / root_relative_path # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must @@ -325,7 +325,7 @@ def gen_python_files( extend_exclude: Pattern[str] | None, force_exclude: Pattern[str] | None, report: Report, - gitignore_dict: dict[Path, PathSpec] | None, + gitignore_dict: dict[Path, GitIgnoreSpec] | None, *, verbose: bool, quiet: bool, diff --git a/src/pyink/handle_ipynb_magics.py b/src/pyink/handle_ipynb_magics.py index 01cfb554ec7..b8a4b882760 100644 --- a/src/pyink/handle_ipynb_magics.py +++ b/src/pyink/handle_ipynb_magics.py @@ -5,6 +5,8 @@ import dataclasses import re import secrets +import string +from collections.abc import Collection from functools import lru_cache from importlib.util import find_spec from typing import TypeGuard @@ -188,6 +190,13 @@ def mask_cell(src: str) -> tuple[str, list[Replacement]]: def create_token(n_chars: int) -> str: """Create a randomly generated token that is n_chars characters long.""" assert n_chars > 0 + if n_chars == 1: + return secrets.choice(string.ascii_letters) + if n_chars < 4: + return "_" + "".join( + secrets.choice(string.ascii_letters + string.digits + "_") + for _ in range(n_chars - 1) + ) n_bytes = max(n_chars // 2 - 1, 1) token = secrets.token_hex(n_bytes) if len(token) + 3 > n_chars: @@ -197,7 +206,7 @@ def create_token(n_chars: int) -> str: return f'b"{token}"' -def get_token(src: str, magic: str) -> str: +def get_token(src: str, magic: str, existing_tokens: Collection[str] = ()) -> str: """Return randomly generated token to mask IPython magic with. For example, if 'magic' was `%matplotlib inline`, then a possible @@ -209,7 +218,7 @@ def get_token(src: str, magic: str) -> str: n_chars = len(magic) token = create_token(n_chars) counter = 0 - while token in src: + while token in src or token in existing_tokens: token = create_token(n_chars) counter += 1 if counter > 100: @@ -271,6 +280,7 @@ def replace_magics(src: str) -> tuple[str, list[Replacement]]: The replacement, along with the transformed code, are returned. """ replacements = [] + existing_tokens: set[str] = set() magic_finder = MagicFinder() magic_finder.visit(ast.parse(src)) new_srcs = [] @@ -286,8 +296,9 @@ def replace_magics(src: str) -> tuple[str, list[Replacement]]: offsets_and_magics[0].col_offset, offsets_and_magics[0].magic, ) - mask = get_token(src, magic) + mask = get_token(src, magic, existing_tokens) replacements.append(Replacement(mask=mask, src=magic)) + existing_tokens.add(mask) line = line[:col_offset] + mask new_srcs.append(line) return "\n".join(new_srcs), replacements @@ -307,7 +318,9 @@ def unmask_cell(src: str, replacements: list[Replacement]) -> str: foo = bar """ for replacement in replacements: - src = src.replace(replacement.mask, replacement.src) + if src.count(replacement.mask) != 1: + raise NothingChanged + src = src.replace(replacement.mask, replacement.src, 1) # Strings in src might have been reformatted with single quotes. src = src.replace(f"b'{replacement.mask[2:-1]}'", replacement.src) return src diff --git a/src/pyink/linegen.py b/src/pyink/linegen.py index fa07b442bd4..8e52d4066cf 100644 --- a/src/pyink/linegen.py +++ b/src/pyink/linegen.py @@ -723,6 +723,7 @@ def __post_init__(self) -> None: self.visit_guard = partial(v, keywords=Ø, parens={"if"}) +# Remove when `simplify_power_operator_hugging` becomes stable. def _hugging_power_ops_line_to_string( line: Line, features: Collection[Feature], @@ -749,11 +750,15 @@ def transform_line( line_str = line_to_string(line) - # We need the line string when power operators are hugging to determine if we should - # split the line. Default to line_str, if no power operator are present on the line. - line_str_hugging_power_ops = ( - _hugging_power_ops_line_to_string(line, features, mode) or line_str - ) + if Preview.simplify_power_operator_hugging in mode: + line_str_hugging_power_ops = line_str + else: + # We need the line string when power operators are hugging to determine if we + # should split the line. Default to line_str, if no power operator are present + # on the line. + line_str_hugging_power_ops = ( + _hugging_power_ops_line_to_string(line, features, mode) or line_str + ) ll = mode.line_length sn = mode.string_normalization @@ -847,9 +852,11 @@ def _rhs( transformers = [delimiter_split, standalone_comment_split, rhs] else: transformers = [rhs] - # It's always safe to attempt hugging of power operations and pretty much every line - # could match. - transformers.append(hug_power_op) + + if Preview.simplify_power_operator_hugging not in mode: + # It's always safe to attempt hugging of power operations and pretty much every + # line could match. + transformers.append(hug_power_op) for transform in transformers: # We are accumulating lines in `result` because we might want to abort @@ -946,8 +953,7 @@ def left_hand_split( current_leaves.append(leaf) if current_leaves is head_leaves: if leaf.type == leaf_type and ( - Preview.fix_type_expansion_split not in mode - or not (leaf_type == token.LPAR and depth > 0) + not (leaf_type == token.LPAR and depth > 0) ): matching_bracket = leaf current_leaves = body_leaves @@ -1571,10 +1577,8 @@ def normalize_invisible_parens( ): check_lpar = True - # Check for assignment LHS with preview feature enabled if ( - Preview.remove_parens_from_assignment_lhs in mode - and index == 0 + index == 0 and isinstance(child, Node) and child.type == syms.atom and node.type == syms.expr_stmt @@ -1646,7 +1650,18 @@ def normalize_invisible_parens( break elif not is_multiline_string(child): - wrap_in_parentheses(node, child, visible=False) + if ( + Preview.fix_if_guard_explosion_in_case_statement in mode + and node.type == syms.guard + ): + mock_line = Line(mode=mode) + for leaf in child.leaves(): + mock_line.append(leaf) + # If it's a guard AND it's short, we DON'T wrap + if not is_line_short_enough(mock_line, mode=mode): + wrap_in_parentheses(node, child, visible=False) + else: + wrap_in_parentheses(node, child, visible=False) comma_check = child.type == token.COMMA @@ -1854,12 +1869,10 @@ def maybe_make_parens_invisible_in_atom( # and option to skip this check for `for` and `with` statements. not remove_brackets_around_comma and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY - # Skip this check in Preview mode in order to # Remove parentheses around multiple exception types in except and # except* without as. See PEP 758 for details. and not ( - Preview.remove_parens_around_except_types in mode - and Feature.UNPARENTHESIZED_EXCEPT_TYPES in features + Feature.UNPARENTHESIZED_EXCEPT_TYPES in features # is a tuple and is_tuple(node) # has a parent node diff --git a/src/pyink/lines.py b/src/pyink/lines.py index cddbcdc6f05..fce3d17fd6c 100644 --- a/src/pyink/lines.py +++ b/src/pyink/lines.py @@ -6,7 +6,7 @@ from typing import Optional, TypeVar, Union, cast from pyink.brackets import COMMA_PRIORITY, DOT_PRIORITY, BracketTracker -from pyink.mode import Mode, Preview +from pyink.mode import Mode from pyink.nodes import ( BRACKETS, CLOSING_BRACKETS, @@ -580,20 +580,10 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: before, after = self._maybe_empty_lines(current_line) previous_after = self.previous_block.after if self.previous_block else 0 before = max(0, before - previous_after) - if Preview.fix_module_docstring_detection in self.mode: - # Always have one empty line after a module docstring - if self._line_is_module_docstring(current_line): - before = 1 - else: - if ( - # Always have one empty line after a module docstring - self.previous_block - and self.previous_block.previous_block is None - and len(self.previous_block.original_line.leaves) == 1 - and self.previous_block.original_line.is_docstring - and not (current_line.is_class or current_line.is_def) - ): - before = 1 + + # Always have one empty line after a module docstring + if self._line_is_module_docstring(current_line): + before = 1 block = LinesBlock( mode=self.mode, @@ -718,10 +708,10 @@ def _maybe_empty_lines(self, current_line: Line) -> tuple[int, int]: and not self.previous_line.depth and not current_line.depth and not current_line.is_import + and not current_line.is_comment and not current_line.is_fmt_pass_converted(first_leaf_matches=is_import) - and Preview.always_one_newline_after_import in self.mode ): - return 1, 0 + return (before or 1), 0 if ( ( @@ -875,13 +865,6 @@ def is_line_short_enough(line: Line, *, mode: Mode, line_str: str = "") -> bool: if not line_str: line_str = line_to_string(line) - if Preview.multiline_string_handling not in mode: - return ( - str_width(line_str) <= mode.line_length - and "\n" not in line_str # multiline strings - and not line.contains_standalone_comments() - ) - if line.contains_standalone_comments(): return False if "\n" not in line_str: @@ -1009,6 +992,44 @@ def can_omit_invisible_parens( """ line = rhs.body + # We can't omit parens if doing so would result in a type: ignore comment + # sharing a line with other comments, as that breaks type: ignore parsing. + # Check if the opening bracket (last leaf of head) has comments that would merge + # with comments from the first line of the body. + if rhs.head.leaves: + opening_bracket = rhs.head.leaves[-1] + head_comments = rhs.head.comments.get(id(opening_bracket), []) + + # If there are comments on the opening bracket line, check if any would + # conflict with type: ignore comments in the body + if head_comments: + has_type_ignore_in_head = any( + is_pragma_comment(comment, rhs.head.mode) + for comment in head_comments + ) + has_other_comment_in_head = any( + not is_pragma_comment(comment, rhs.head.mode) + for comment in head_comments + ) + + # Check for comments in the body that would potentially end up on the + # same line as the head comments when parens are removed + has_type_ignore_in_body = False + has_other_comment_in_body = False + for leaf in rhs.body.leaves: + for comment in rhs.body.comments.get(id(leaf), []): + if is_pragma_comment(comment, rhs.body.mode): + has_type_ignore_in_body = True + else: + has_other_comment_in_body = True + + # Preserve parens if we have both type: ignore and other comments that + # could end up on the same line + if (has_type_ignore_in_head and has_other_comment_in_body) or ( + has_other_comment_in_head and has_type_ignore_in_body + ): + return False + # We need optional parens in order to split standalone comments to their own lines # if there are no nested parens around the standalone comments closing_bracket: Leaf | None = None diff --git a/src/pyink/mode.py b/src/pyink/mode.py index 91b5d203f9d..f7700a95caf 100644 --- a/src/pyink/mode.py +++ b/src/pyink/mode.py @@ -214,7 +214,7 @@ class Feature(Enum): def supports_feature(target_versions: set[TargetVersion], feature: Feature) -> bool: if not target_versions: - raise ValueError("target_versions must not be empty") + raise ValueError("At least one target Python version must be specified.") return all(feature in VERSION_TO_FEATURES[version] for version in target_versions) @@ -226,33 +226,20 @@ class Preview(Enum): # for https://github.com/psf/black/issues/3117 to be fixed. string_processing = auto() hug_parens_with_braces_and_square_brackets = auto() - wrap_long_dict_values_in_parens = auto() - multiline_string_handling = auto() - always_one_newline_after_import = auto() - fix_fmt_skip_in_one_liners = auto() - standardize_type_comments = auto() wrap_comprehension_in = auto() - # Remove parentheses around multiple exception types in except and - # except* without as. See PEP 758 for details. - remove_parens_around_except_types = auto() - normalize_cr_newlines = auto() - fix_module_docstring_detection = auto() - fix_type_expansion_split = auto() - remove_parens_from_assignment_lhs = auto() + simplify_power_operator_hugging = auto() + wrap_long_dict_values_in_parens = auto() + fix_if_guard_explosion_in_case_statement = auto() UNSTABLE_FEATURES: set[Preview] = { - # Many issues, see summary in https://github.com/psf/black/issues/4042 + # Many issues, see summary in https://github.com/psf/black/issues/4208 Preview.string_processing, # See issue #4036 (crash), #4098, #4099 (proposed tweaks) Preview.hug_parens_with_braces_and_square_brackets, } -class Deprecated(UserWarning): - """Visible deprecation warning.""" - - class Quote(Enum): SINGLE = "'" DOUBLE = '"' @@ -311,11 +298,6 @@ def __contains__(self, feature: Preview) -> bool: except those in UNSTABLE_FEATURES are enabled. Any features in `self.enabled_features` are also enabled. """ - # The following feature is temporarily disabled in Pyink because it is - # not compatible with range formatting. It's because format skipping in - # Black is broken. - if feature is Preview.always_one_newline_after_import and self.is_pyink: - return False if self.unstable: return True if feature in self.enabled_features: @@ -339,10 +321,9 @@ def get_cache_key(self) -> str: + "@" + ",".join(sorted(self.python_cell_magics)) ) - if len(features_and_magics) > _MAX_CACHE_KEY_PART_LENGTH: - features_and_magics = sha256(features_and_magics.encode()).hexdigest()[ - :_MAX_CACHE_KEY_PART_LENGTH - ] + features_and_magics = sha256(features_and_magics.encode()).hexdigest()[ + :_MAX_CACHE_KEY_PART_LENGTH + ] parts = [ version_str, str(self.line_length), diff --git a/src/pyink/nodes.py b/src/pyink/nodes.py index a7ca659e2e4..016ae0e4faa 100644 --- a/src/pyink/nodes.py +++ b/src/pyink/nodes.py @@ -417,6 +417,15 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: if t == token.STAR: return NO + if Preview.simplify_power_operator_hugging in mode: + # Power operator hugging + if t == token.DOUBLESTAR and is_simple_exponentiation(p): + return NO + prevp = preceding_leaf(leaf) + if prevp and prevp.type == token.DOUBLESTAR: + if prevp.parent and is_simple_exponentiation(prevp.parent): + return NO + return SPACE @@ -544,6 +553,25 @@ def is_arith_like(node: LN) -> bool: } +def is_simple_exponentiation(node: LN) -> bool: + """Whether whitespace around `**` should be removed.""" + + def is_simple(node: LN) -> bool: + if isinstance(node, Leaf): + return node.type in (token.NAME, token.NUMBER, token.DOT, token.DOUBLESTAR) + elif node.type == syms.factor: # unary operators + return is_simple(node.children[1]) + else: + return all(is_simple(child) for child in node.children) + + return ( + node.type == syms.power + and len(node.children) >= 3 + and node.children[-2].type == token.DOUBLESTAR + and is_simple(node) + ) + + def is_docstring(node: NL) -> bool: if isinstance(node, Leaf): if node.type != token.STRING: @@ -662,7 +690,7 @@ def is_one_sequence_between( break else: - raise LookupError("Opening paren not found in `leaves`") + return False commas = 0 _opening_index += 1 @@ -815,10 +843,6 @@ def is_parent_function_or_class(node: Node) -> bool: return node.parent.type in {syms.funcdef, syms.classdef} -def is_function_or_class(node: Node) -> bool: - return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} - - def is_stub_suite(node: Node, mode: Mode) -> bool: """Return True if `node` is a suite with a stub body.""" if ( @@ -937,7 +961,7 @@ def is_async_stmt_or_funcdef(leaf: Leaf) -> bool: def is_type_comment(leaf: Leaf, mode: Mode) -> bool: """Return True if the given leaf is a type comment. This function should only be used for general type comments (excluding ignore annotations, which should - use `is_type_ignore_comment`). Note that general type comments are no longer + use `is_pragma_comment`). Note that general type comments are no longer used in modern version of Python, this function may be deprecated in the future.""" t = leaf.type v = leaf.value @@ -945,15 +969,11 @@ def is_type_comment(leaf: Leaf, mode: Mode) -> bool: def is_type_comment_string(value: str, mode: Mode) -> bool: - if Preview.standardize_type_comments in mode: - is_valid = value.startswith("#") and value[1:].lstrip().startswith("type:") - else: - is_valid = value.startswith("# type:") - return is_valid + return value.startswith("#") and value[1:].lstrip().startswith("type:") def is_pragma_comment(leaf: Leaf, mode: Mode) -> bool: - """Return True if the given leaf is a type comment with ignore annotation.""" + """Return True if the given leaf is an annotation pragma (e.g., type: ignore, pylint, noqa).""" t = leaf.type v = leaf.value return t in {token.COMMENT, STANDALONE_COMMENT} and ( @@ -964,14 +984,9 @@ def is_pragma_comment(leaf: Leaf, mode: Mode) -> bool: def is_type_ignore_comment_string(value: str, mode: Mode) -> bool: """Return True if the given string match with type comment with ignore annotation.""" - if Preview.standardize_type_comments in mode: - is_valid = is_type_comment_string(value, mode) and value.split(":", 1)[ - 1 - ].lstrip().startswith("ignore") - else: - is_valid = value.startswith("# type: ignore") - - return is_valid + return is_type_comment_string(value, mode) and value.split(":", 1)[ + 1 + ].lstrip().startswith("ignore") def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: diff --git a/src/pyink/parsing.py b/src/pyink/parsing.py index 1a18203523b..e1dd10a990c 100644 --- a/src/pyink/parsing.py +++ b/src/pyink/parsing.py @@ -102,22 +102,6 @@ def lib2to3_parse( return result -def matches_grammar(src_txt: str, grammar: Grammar) -> bool: - drv = driver.Driver(grammar) - try: - drv.parse_string(src_txt, False) - except (ParseError, TokenError, IndentationError): - return False - else: - return True - - -def lib2to3_unparse(node: Node) -> str: - """Given a lib2to3 node, return its string representation.""" - code = str(node) - return code - - class ASTSafetyError(Exception): """Raised when Black's generated code is not equivalent to the old AST.""" diff --git a/src/pyink/resources/pyink.schema.json b/src/pyink/resources/pyink.schema.json index 93eecd617a8..0dfbd456d6e 100644 --- a/src/pyink/resources/pyink.schema.json +++ b/src/pyink/resources/pyink.schema.json @@ -82,17 +82,10 @@ "enum": [ "string_processing", "hug_parens_with_braces_and_square_brackets", - "wrap_long_dict_values_in_parens", - "multiline_string_handling", - "always_one_newline_after_import", - "fix_fmt_skip_in_one_liners", - "standardize_type_comments", "wrap_comprehension_in", - "remove_parens_around_except_types", - "normalize_cr_newlines", - "fix_module_docstring_detection", - "fix_type_expansion_split", - "remove_parens_from_assignment_lhs" + "simplify_power_operator_hugging", + "wrap_long_dict_values_in_parens", + "fix_if_guard_explosion_in_case_statement" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/src/pyink/strings.py b/src/pyink/strings.py index 5cc90b31533..c9f0cfe630d 100644 --- a/src/pyink/strings.py +++ b/src/pyink/strings.py @@ -291,9 +291,11 @@ def normalize_fstring_quotes( # edge case: new_segments[-1] = new_segments[-1][:-1] + '\\"' + orig_escape_count = 0 + new_escape_count = 0 for middle, new_segment in zip(middles, new_segments, strict=True): - orig_escape_count = middle.value.count("\\") - new_escape_count = new_segment.count("\\") + orig_escape_count += middle.value.count("\\") + new_escape_count += new_segment.count("\\") if new_escape_count > orig_escape_count: return middles, quote # Do not introduce more escaping diff --git a/src/pyink/trans.py b/src/pyink/trans.py index 7164542a92b..ac06cf95e38 100644 --- a/src/pyink/trans.py +++ b/src/pyink/trans.py @@ -66,6 +66,7 @@ def TErr(err_msg: str) -> Err[CannotTransform]: return Err(cant_transform) +# Remove when `simplify_power_operator_hugging` becomes stable. def hug_power_op( line: Line, features: Collection[Feature], mode: Mode ) -> Iterator[Line]: @@ -133,6 +134,7 @@ def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: yield new_line +# Remove when `simplify_power_operator_hugging` becomes stable. def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) -> bool: """ Handling the determination of is_simple_lookup for the lines prior to the doublestar @@ -155,6 +157,7 @@ def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) return True +# Remove when `simplify_power_operator_hugging` becomes stable. def handle_is_simple_lookup_forward( line: Line, index: int, disallowed: set[int] ) -> bool: @@ -181,6 +184,7 @@ def handle_is_simple_lookup_forward( return True +# Remove when `simplify_power_operator_hugging` becomes stable. def is_expression_chained(chained_leaves: list[Leaf]) -> bool: """ Function to determine if the variable is a chained call. @@ -1444,6 +1448,20 @@ def do_splitter_match(self, line: Line) -> TMatchResult: if self._prefer_paren_wrap_match(LL) is not None: return TErr("Line needs to be wrapped in parens first.") + # If the line is just STRING + COMMA (a one-item tuple) and not inside + # brackets, we need to defer to StringParenWrapper to wrap it first. + # Otherwise, splitting the string would create multiple expressions where + # only the last has the comma, breaking AST equivalence. See issue #4912. + if ( + not line.inside_brackets + and len(LL) == 2 + and LL[0].type == token.STRING + and LL[1].type == token.COMMA + ): + return TErr( + "Line with trailing comma tuple needs to be wrapped in parens first." + ) + is_valid_index = is_valid_index_factory(LL) idx = 0 @@ -1979,9 +1997,14 @@ def do_splitter_match(self, line: Line) -> TMatchResult: or self._assert_match(LL) or self._assign_match(LL) or self._dict_or_lambda_match(LL) - or self._prefer_paren_wrap_match(LL) ) + if string_idx is None: + string_idx = self._trailing_comma_tuple_match(line) + + if string_idx is None: + string_idx = self._prefer_paren_wrap_match(LL) + if string_idx is not None: string_value = line.leaves[string_idx].value # If the string has neither spaces nor East Asian stops... @@ -2177,6 +2200,32 @@ def _dict_or_lambda_match(LL: list[Leaf]) -> int | None: return None + @staticmethod + def _trailing_comma_tuple_match(line: Line) -> int | None: + """ + Returns: + string_idx such that @line.leaves[string_idx] is equal to our target + (i.e. matched) string, if the line is a bare trailing comma tuple + (STRING + COMMA) not inside brackets. + OR + None, otherwise. + + This handles the case from issue #4912 where a long string with a + trailing comma (making it a one-item tuple) needs to be wrapped in + parentheses before splitting to preserve AST equivalence. + """ + LL = line.leaves + # Match: STRING followed by COMMA, not inside brackets + if ( + not line.inside_brackets + and len(LL) == 2 + and LL[0].type == token.STRING + and LL[1].type == token.COMMA + ): + return 0 + + return None + def do_transform( self, line: Line, string_indices: list[int] ) -> Iterator[TResult[Line]]: diff --git a/test_requirements.txt b/test_requirements.txt deleted file mode 100644 index b13f25dce8c..00000000000 --- a/test_requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -click == 8.1.8 -coverage >= 5.3 -pre-commit -pytest >= 7 -pytest-xdist >= 3.0.2 -pytest-cov >= 4.1.0 -tox diff --git a/tests/data/cases/comments5.py b/tests/data/cases/comments5.py index 4270d3a09a2..f86e54732e8 100644 --- a/tests/data/cases/comments5.py +++ b/tests/data/cases/comments5.py @@ -56,13 +56,10 @@ def decorated1(): ... def decorated1(): ... -# Note: this is fixed in -# Preview.empty_lines_before_class_or_def_with_leading_comments. -# In the current style, the user will have to split those lines by hand. some_instruction -# This comment should be split from `some_instruction` by two lines but isn't. +# This comment should be split from `some_instruction` by two lines. def g(): ... diff --git a/tests/data/cases/comments_in_lambda_default.py b/tests/data/cases/comments_in_lambda_default.py new file mode 100644 index 00000000000..9aee769c8f3 --- /dev/null +++ b/tests/data/cases/comments_in_lambda_default.py @@ -0,0 +1,27 @@ +help(lambda x=( + # comment + "bar", +): False) + +result = (lambda x=( + # a standalone comment + 1, + 2, + 3, +): x) + +# output + +help( + lambda x=( + # comment + "bar", + ): False, +) + +result = lambda x=( + # a standalone comment + 1, + 2, + 3, +): x diff --git a/tests/data/cases/composition.py b/tests/data/cases/composition.py index 0f88567e687..092bceb7e3c 100644 --- a/tests/data/cases/composition.py +++ b/tests/data/cases/composition.py @@ -161,9 +161,7 @@ def tricky_asserts(self) -> None: 8 STORE_ATTR 0 (x) 10 LOAD_CONST 0 (None) 12 RETURN_VALUE - """ % ( - _C.__init__.__code__.co_firstlineno + 1, - ) + """ % (_C.__init__.__code__.co_firstlineno + 1,) assert ( expectedexpectedexpectedexpectedexpectedexpectedexpectedexpectedexpect diff --git a/tests/data/cases/composition_no_trailing_comma.py b/tests/data/cases/composition_no_trailing_comma.py index 5b13b2b45a8..99b2e86f2ce 100644 --- a/tests/data/cases/composition_no_trailing_comma.py +++ b/tests/data/cases/composition_no_trailing_comma.py @@ -347,9 +347,7 @@ def tricky_asserts(self) -> None: 8 STORE_ATTR 0 (x) 10 LOAD_CONST 0 (None) 12 RETURN_VALUE - """ % ( - _C.__init__.__code__.co_firstlineno + 1, - ) + """ % (_C.__init__.__code__.co_firstlineno + 1,) assert ( expectedexpectedexpectedexpectedexpectedexpectedexpectedexpectedexpect diff --git a/tests/data/cases/docstring_preview.py b/tests/data/cases/docstring2.py similarity index 100% rename from tests/data/cases/docstring_preview.py rename to tests/data/cases/docstring2.py diff --git a/tests/data/cases/fmtskip10.py b/tests/data/cases/fmtskip10.py index f4f4981eb26..640ac4ddec9 100644 --- a/tests/data/cases/fmtskip10.py +++ b/tests/data/cases/fmtskip10.py @@ -1,4 +1,3 @@ -# flags: --preview def foo(): return "mock" # fmt: skip if True: print("yay") # fmt: skip for i in range(10): print(i) # fmt: skip diff --git a/tests/data/cases/fmtskip12.py b/tests/data/cases/fmtskip12.py index 3af6b4443a1..217fc2b5bb2 100644 --- a/tests/data/cases/fmtskip12.py +++ b/tests/data/cases/fmtskip12.py @@ -1,5 +1,3 @@ -# flags: --preview - with open("file.txt") as f: content = f.read() # fmt: skip # Ideally, only the last line would be ignored diff --git a/tests/data/cases/fmtskip13.py b/tests/data/cases/fmtskip13.py index f3abcba9219..1f2d53437cb 100644 --- a/tests/data/cases/fmtskip13.py +++ b/tests/data/cases/fmtskip13.py @@ -1,5 +1,3 @@ -# flags: --preview - t = ( {"foo": "very long string", "bar": "another very long string", "baz": "we should run out of space by now"}, # fmt: skip {"foo": "bar"}, diff --git a/tests/data/cases/if_guard_inside_case.py b/tests/data/cases/if_guard_inside_case.py new file mode 100644 index 00000000000..374469977e4 --- /dev/null +++ b/tests/data/cases/if_guard_inside_case.py @@ -0,0 +1,49 @@ +# flags: --preview + +def f(x): + match x: + # good refactor + case [ + y + ] if y == 123: + pass + + case [ + y + ] if True: + pass + + case [ + y, + ] if True: + pass + + # bad refactor + case [ + y, + ] if y == 123: + pass + + +# output + + +def f(x): + match x: + # good refactor + case [y] if y == 123: + pass + + case [y] if True: + pass + + case [ + y, + ] if True: + pass + + # bad refactor + case [ + y, + ] if y == 123: + pass diff --git a/tests/data/cases/preview_import_line_collapse.py b/tests/data/cases/import_line_collapse.py similarity index 99% rename from tests/data/cases/preview_import_line_collapse.py rename to tests/data/cases/import_line_collapse.py index 74ae349a2ca..2c6c915f539 100644 --- a/tests/data/cases/preview_import_line_collapse.py +++ b/tests/data/cases/import_line_collapse.py @@ -1,4 +1,3 @@ -# flags: --preview from middleman.authentication import validate_oauth_token diff --git a/tests/data/cases/module_docstring_after_comment.py b/tests/data/cases/module_docstring_after_comment.py index 6a755b36dd2..5709b92b197 100644 --- a/tests/data/cases/module_docstring_after_comment.py +++ b/tests/data/cases/module_docstring_after_comment.py @@ -1,4 +1,3 @@ -# flags: --preview #!/python # regression test for #4762 diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/multiline_strings.py similarity index 97% rename from tests/data/cases/preview_multiline_strings.py rename to tests/data/cases/multiline_strings.py index 9596357b287..10662d8dc38 100644 --- a/tests/data/cases/preview_multiline_strings.py +++ b/tests/data/cases/multiline_strings.py @@ -1,4 +1,3 @@ -# flags: --preview """cow say""", call(3, "dogsay", textwrap.dedent("""dove @@ -449,10 +448,8 @@ def foo(): """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" ), - "xxxxxxxx": ( - """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx - xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" - ), + "xxxxxxxx": """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx + xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""", }, } @@ -471,14 +468,12 @@ def foo(): a a""" ), - "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( - """ + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": """ a a a a -a""" - ), +a""", } a = ( diff --git a/tests/data/cases/preview_fmtpass_imports.py b/tests/data/cases/preview_fmtpass_imports.py deleted file mode 100644 index d7ea3883aa4..00000000000 --- a/tests/data/cases/preview_fmtpass_imports.py +++ /dev/null @@ -1,21 +0,0 @@ -# flags: --preview - -# Regression test for https://github.com/psf/black/issues/3438 - -import ast -import collections # fmt: skip -import dataclasses -# fmt: off -import os -# fmt: on -import pathlib - -import re # fmt: skip -import secrets - -# fmt: off -import sys -# fmt: on - -import tempfile -import zoneinfo diff --git a/tests/data/cases/preview_long_strings.py b/tests/data/cases/preview_long_strings.py index a267a509057..177f866be0a 100644 --- a/tests/data/cases/preview_long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -39,6 +39,9 @@ T2 = ("This is a really long string that can't be expected to fit in one line and is the only child of a tuple literal.",) +# Test case for https://github.com/psf/black/issues/4912 - unassigned long string with trailing comma +"A long string literal that is not assigned to a variable, exceeds line length when string-processing is enabled, and has a trailing comma (to make it a one-item tuple)", + func_with_keywords(my_arg, my_kwarg="Long keyword strings also need to be wrapped, but they will probably need to be handled a little bit differently.") bad_split1 = ( @@ -506,6 +509,13 @@ def foo(): ), ) +# Test case for https://github.com/psf/black/issues/4912 - unassigned long string with trailing comma +( + "A long string literal that is not assigned to a variable, exceeds line length when" + " string-processing is enabled, and has a trailing comma (to make it a one-item" + " tuple)" +), + func_with_keywords( my_arg, my_kwarg=( diff --git a/tests/data/cases/preview_long_strings__east_asian_width.py b/tests/data/cases/preview_long_strings__east_asian_width.py index 022b0452522..01165f9c032 100644 --- a/tests/data/cases/preview_long_strings__east_asian_width.py +++ b/tests/data/cases/preview_long_strings__east_asian_width.py @@ -5,6 +5,20 @@ hangul = '코드포인트 수는 적으나 실제 터미널이나 에디터에서 렌더링될 땐 너무 길어서 줄바꿈이 필요한 문자열' hanzi = '中文測試：代碼點數量少，但在真正的終端模擬器或編輯器中呈現時太長，因此需要換行的字符串。' japanese = 'コードポイントの数は少ないが、実際の端末エミュレータやエディタでレンダリングされる時は長すぎる為、改行が要る文字列' +khmer = 'សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ' +# Should stay the same +khmer_same = [ + "text, expected_language", + [ + ( + ( + "សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ" + " ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ " + ), + "km", + ), # Khmer + ], +] # output @@ -24,3 +38,20 @@ "実際の端末エミュレータやエディタでレンダリングされる時は長すぎる為、" "改行が要る文字列" ) +khmer = ( + "សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ" + " ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ" +) +# Should stay the same +khmer_same = [ + "text, expected_language", + [ + ( + ( + "សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ" + " ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ " + ), + "km", + ), # Khmer + ], +] diff --git a/tests/data/cases/preview_simplify_power_operator_hugging.py b/tests/data/cases/preview_simplify_power_operator_hugging.py new file mode 100644 index 00000000000..1977396685c --- /dev/null +++ b/tests/data/cases/preview_simplify_power_operator_hugging.py @@ -0,0 +1,153 @@ +# flags: --preview +# This is a copy of `power_op_spacing.py`. Remove when `simplify_power_operator_hugging` becomes stable. + +def function(**kwargs): + t = a**2 + b**3 + return t ** 2 + + +def function_replace_spaces(**kwargs): + t = a **2 + b** 3 + c ** 4 + + +def function_dont_replace_spaces(): + {**a, **b, **c} + + +a = 5**~4 +b = 5 ** f() +c = -(5**2) +d = 5 ** f["hi"] +e = lazy(lambda **kwargs: 5) +f = f() ** 5 +g = a.b**c.d +h = 5 ** funcs.f() +i = funcs.f() ** 5 +j = super().name ** 5 +k = [(2**idx, value) for idx, value in pairs] +l = mod.weights_[0] == pytest.approx(0.95**100, abs=0.001) +m = [([2**63], [1, 2**63])] +n = count <= 10**5 +o = settings(max_examples=10**6) +p = {(k, k**2): v**2 for k, v in pairs} +q = [10**i for i in range(6)] +r = x**y +s = 1 ** 1 +t = ( + 1 + ** 1 + **1 + ** 1 +) + +a = 5.0**~4.0 +b = 5.0 ** f() +c = -(5.0**2.0) +d = 5.0 ** f["hi"] +e = lazy(lambda **kwargs: 5) +f = f() ** 5.0 +g = a.b**c.d +h = 5.0 ** funcs.f() +i = funcs.f() ** 5.0 +j = super().name ** 5.0 +k = [(2.0**idx, value) for idx, value in pairs] +l = mod.weights_[0] == pytest.approx(0.95**100, abs=0.001) +m = [([2.0**63.0], [1.0, 2**63.0])] +n = count <= 10**5.0 +o = settings(max_examples=10**6.0) +p = {(k, k**2): v**2.0 for k, v in pairs} +q = [10.5**i for i in range(6)] +s = 1.0 ** 1.0 +t = ( + 1.0 + ** 1.0 + **1.0 + ** 1.0 +) + + +# WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873) +if hasattr(view, "sum_of_weights"): + return np.divide( # type: ignore[no-any-return] + view.variance, # type: ignore[union-attr] + view.sum_of_weights, # type: ignore[union-attr] + out=np.full(view.sum_of_weights.shape, np.nan), # type: ignore[union-attr] + where=view.sum_of_weights**2 > view.sum_of_weights_squared, # type: ignore[union-attr] + ) + +return np.divide( + where=view.sum_of_weights_of_weight_long**2 > view.sum_of_weights_squared, # type: ignore +) + + +# output +# This is a copy of `power_op_spacing.py`. Remove when `simplify_power_operator_hugging` becomes stable. + + +def function(**kwargs): + t = a**2 + b**3 + return t**2 + + +def function_replace_spaces(**kwargs): + t = a**2 + b**3 + c**4 + + +def function_dont_replace_spaces(): + {**a, **b, **c} + + +a = 5**~4 +b = 5 ** f() +c = -(5**2) +d = 5 ** f["hi"] +e = lazy(lambda **kwargs: 5) +f = f() ** 5 +g = a.b**c.d +h = 5 ** funcs.f() +i = funcs.f() ** 5 +j = super().name ** 5 +k = [(2**idx, value) for idx, value in pairs] +l = mod.weights_[0] == pytest.approx(0.95**100, abs=0.001) +m = [([2**63], [1, 2**63])] +n = count <= 10**5 +o = settings(max_examples=10**6) +p = {(k, k**2): v**2 for k, v in pairs} +q = [10**i for i in range(6)] +r = x**y +s = 1**1 +t = 1**1**1**1 + +a = 5.0**~4.0 +b = 5.0 ** f() +c = -(5.0**2.0) +d = 5.0 ** f["hi"] +e = lazy(lambda **kwargs: 5) +f = f() ** 5.0 +g = a.b**c.d +h = 5.0 ** funcs.f() +i = funcs.f() ** 5.0 +j = super().name ** 5.0 +k = [(2.0**idx, value) for idx, value in pairs] +l = mod.weights_[0] == pytest.approx(0.95**100, abs=0.001) +m = [([2.0**63.0], [1.0, 2**63.0])] +n = count <= 10**5.0 +o = settings(max_examples=10**6.0) +p = {(k, k**2): v**2.0 for k, v in pairs} +q = [10.5**i for i in range(6)] +s = 1.0**1.0 +t = 1.0**1.0**1.0**1.0 + + +# WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873) +if hasattr(view, "sum_of_weights"): + return np.divide( # type: ignore[no-any-return] + view.variance, # type: ignore[union-attr] + view.sum_of_weights, # type: ignore[union-attr] + out=np.full(view.sum_of_weights.shape, np.nan), # type: ignore[union-attr] + where=view.sum_of_weights**2 > view.sum_of_weights_squared, # type: ignore[union-attr] + ) + +return np.divide( + where=view.sum_of_weights_of_weight_long**2 > view.sum_of_weights_squared, # type: ignore +) diff --git a/tests/data/cases/preview_simplify_power_operator_hugging_long.py b/tests/data/cases/preview_simplify_power_operator_hugging_long.py new file mode 100644 index 00000000000..1ec7acdd7d6 --- /dev/null +++ b/tests/data/cases/preview_simplify_power_operator_hugging_long.py @@ -0,0 +1,99 @@ +# flags: --preview +# This is a copy of `power_op_spacing_long.py` with output adjusted for `simplify_power_operator_hugging`. +a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +b = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +c = 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 +d = 1**1 ** 1**1 ** 1**1 ** 1**1 ** 1**1**1 ** 1 ** 1**1 ** 1**1**1**1**1 ** 1 ** 1**1**1 **1**1** 1 ** 1 ** 1 +e = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 +f = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 + +a = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +b = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +c = 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 +d = 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0**1.0 ** 1.0 ** 1.0**1.0 ** 1.0**1.0**1.0 + +# output +# This is a copy of `power_op_spacing_long.py` with output adjusted for `simplify_power_operator_hugging`. +a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +b = ( + 1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 + **1 +) +c = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +d = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +e = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 +f = ( + 𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 + **𨉟 +) + +a = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +b = ( + 1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 + **1.0 +) +c = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +d = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 diff --git a/tests/data/cases/remove_except_types_parens.py b/tests/data/cases/remove_except_types_parens.py index 71f2d229d3a..463be7e52d4 100644 --- a/tests/data/cases/remove_except_types_parens.py +++ b/tests/data/cases/remove_except_types_parens.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.14 +# flags: --minimum-version=3.14 # SEE PEP 758 FOR MORE DETAILS # remains unchanged try: diff --git a/tests/data/cases/remove_except_types_parens_pre_py314.py b/tests/data/cases/remove_except_types_parens_pre_py314.py index 9f3a3b25652..acd2458c8a9 100644 --- a/tests/data/cases/remove_except_types_parens_pre_py314.py +++ b/tests/data/cases/remove_except_types_parens_pre_py314.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.11 +# flags: --minimum-version=3.11 # SEE PEP 758 FOR MORE DETAILS # remains unchanged try: diff --git a/tests/data/cases/remove_parens_from_lhs.py b/tests/data/cases/remove_parens_from_lhs.py index 92d0731b87b..b172aa748f7 100644 --- a/tests/data/cases/remove_parens_from_lhs.py +++ b/tests/data/cases/remove_parens_from_lhs.py @@ -1,4 +1,3 @@ -# flags: --preview # Remove unnecessary parentheses from LHS of assignments diff --git a/tests/data/cases/skip_magic_trailing_comma.py b/tests/data/cases/skip_magic_trailing_comma.py index 4dda5df40f0..1ef34af4f11 100644 --- a/tests/data/cases/skip_magic_trailing_comma.py +++ b/tests/data/cases/skip_magic_trailing_comma.py @@ -70,6 +70,6 @@ func1(arg1).func2(arg2).func3(arg3).func4(arg4).func5(arg5) -(a, b, c, d) = func1(arg1) and func2(arg2) +a, b, c, d = func1(arg1) and func2(arg2) func(argument1, (one, two), argument4, argument5, argument6) diff --git a/tests/data/cases/preview_standardize_type_comments.py b/tests/data/cases/standardize_type_comments.py similarity index 95% rename from tests/data/cases/preview_standardize_type_comments.py rename to tests/data/cases/standardize_type_comments.py index 2ab45533cf9..0b3e63c92f7 100644 --- a/tests/data/cases/preview_standardize_type_comments.py +++ b/tests/data/cases/standardize_type_comments.py @@ -1,4 +1,3 @@ -# flags: --preview def foo( a, #type:int b, #type: str diff --git a/tests/data/cases/type_expansion.py b/tests/data/cases/type_expansion.py index 6cd7a0b7736..d5cbc6e7274 100644 --- a/tests/data/cases/type_expansion.py +++ b/tests/data/cases/type_expansion.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.12 +# flags: --minimum-version=3.12 def f1[T: (int, str)](a,): pass diff --git a/tests/data/cases/type_ignore_with_other_comment.py b/tests/data/cases/type_ignore_with_other_comment.py new file mode 100644 index 00000000000..bfa330d4b02 --- /dev/null +++ b/tests/data/cases/type_ignore_with_other_comment.py @@ -0,0 +1,27 @@ +import pandas as pd + +interval_td = pd.Interval( + pd.Timedelta("1 days"), pd.Timedelta("2 days"), closed="neither" +) + +_td = ( # pyright: ignore[reportOperatorIssue,reportUnknownVariableType] + interval_td + - pd.Interval( # type: ignore[operator] + pd.Timedelta(1, "ns"), pd.Timedelta(2, "ns") + ) +) + +# output + +import pandas as pd + +interval_td = pd.Interval( + pd.Timedelta("1 days"), pd.Timedelta("2 days"), closed="neither" +) + +_td = ( # pyright: ignore[reportOperatorIssue,reportUnknownVariableType] + interval_td + - pd.Interval( # type: ignore[operator] + pd.Timedelta(1, "ns"), pd.Timedelta(2, "ns") + ) +) diff --git a/tests/data/cases/walrus_in_dict.py b/tests/data/cases/walrus_in_dict.py index 68ec5d5df2f..33af6ab907f 100644 --- a/tests/data/cases/walrus_in_dict.py +++ b/tests/data/cases/walrus_in_dict.py @@ -1,9 +1,6 @@ -# flags: --preview -# This is testing an issue that is specific to the preview style (wrap_long_dict_values_in_parens) { "is_update": (up := commit.hash in update_hashes) } # output -# This is testing an issue that is specific to the preview style (wrap_long_dict_values_in_parens) {"is_update": (up := commit.hash in update_hashes)} diff --git a/tests/data/include_exclude_tests/b/exclude/still_exclude/a.pie b/tests/data/include_exclude_tests/b/exclude/still_exclude/a.pie new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/include_exclude_tests/b/exclude/still_exclude/a.py b/tests/data/include_exclude_tests/b/exclude/still_exclude/a.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/include_exclude_tests/b/exclude/still_exclude/a.pyi b/tests/data/include_exclude_tests/b/exclude/still_exclude/a.pyi new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/line_ranges_formatted/basic.py b/tests/data/line_ranges_formatted/basic.py index b419b1f16ae..9d5567d36b6 100644 --- a/tests/data/line_ranges_formatted/basic.py +++ b/tests/data/line_ranges_formatted/basic.py @@ -5,7 +5,6 @@ Literal, ) - # fmt: off class Unformatted: def should_also_work(self): diff --git a/tests/test_black.py b/tests/test_black.py index 643f9cfc311..404e900e195 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -19,7 +19,7 @@ from io import BytesIO from pathlib import Path, WindowsPath from platform import system -from tempfile import TemporaryDirectory +from tempfile import NamedTemporaryFile, TemporaryDirectory from typing import Any, TypeVar from unittest.mock import MagicMock, patch @@ -28,7 +28,7 @@ from click import unstyle from click.testing import CliRunner from packaging.version import Version -from pathspec import PathSpec +from pathspec import GitIgnoreSpec import pyink import pyink.files @@ -2059,17 +2059,17 @@ def test_carriage_return_edge_cases(self) -> None: "try:\\\r# type: ignore\n pass\nfinally:\n pass\n", mode=pyink.FileMode(), ) - == "try: # type: ignore\n pass\nfinally:\n pass\n" + == "try: # type: ignore\r pass\rfinally:\r pass\r" ) - assert pyink.format_str("{\r}", mode=pyink.FileMode()) == "{}\n" - assert pyink.format_str("pass #\r#\n", mode=pyink.FileMode()) == "pass #\n#\n" + assert pyink.format_str("{\r}", mode=pyink.FileMode()) == "{}\r" + assert pyink.format_str("pass #\r#\n", mode=pyink.FileMode()) == "pass #\r#\r" - assert pyink.format_str("x=\\\r\n1", mode=pyink.FileMode()) == "x = 1\n" + assert pyink.format_str("x=\\\r\n1", mode=pyink.FileMode()) == "x = 1\r\n" assert pyink.format_str("x=\\\n1", mode=pyink.FileMode()) == "x = 1\n" - assert pyink.format_str("x=\\\r1", mode=pyink.FileMode()) == "x = 1\n" + assert pyink.format_str("x=\\\r1", mode=pyink.FileMode()) == "x = 1\r" assert ( pyink.format_str("class A\\\r\n:...", mode=pyink.FileMode()) - == "class A: ...\n" + == "class A: ...\r\n" ) assert ( pyink.format_str("class A\\\n:...", mode=pyink.FileMode()) @@ -2077,15 +2077,42 @@ def test_carriage_return_edge_cases(self) -> None: ) assert ( pyink.format_str("class A\\\r:...", mode=pyink.FileMode()) - == "class A: ...\n" + == "class A: ...\r" ) - def test_preview_newline_type_detection(self) -> None: - mode = Mode(enabled_features={Preview.normalize_cr_newlines}) + def test_newline_type_detection(self) -> None: + mode = Mode() newline_types = ["A\n", "A\r\n", "A\r"] for test_case in itertools.permutations(newline_types): assert pyink.format_str("".join(test_case), mode=mode) == test_case[0] * 3 + def test_decode_with_encoding(self) -> None: + # This uses temporary files since some editors (including GitHub) + # struggle with displaying and/or editing non utf-8 data + # \xfc is iso-8859-1 for ü + with NamedTemporaryFile(delete=False) as first_line: + first_line.write( + b"# -*- coding: iso-8859-1 -*-\n" + b"# 2002-11-22 J\xfcrgen Hermann \n" + ) + first_line.close() + self.assertFalse( + ff(Path(first_line.name)), + "Failed to properly detect encoding", + ) + + with NamedTemporaryFile(delete=False) as second_line: + second_line.write( + b"#! /usr/bin/env python3\n" + b"# -*- coding: iso-8859-1 -*-\n" + b"# 2002-11-22 J\xfcrgen Hermann \n" + ) + second_line.close() + self.assertFalse( + ff(Path(second_line.name)), + "Failed to properly detect encoding on second line", + ) + class TestCaching: def test_get_cache_dir( @@ -2139,6 +2166,15 @@ def test_cache_file_length(self) -> None: # doesn't get too crazy. assert len(cache_file.name) <= 96 + def test_cache_file_path_ignores_python_cell_magic_separators(self) -> None: + mode = replace(DEFAULT_MODE, python_cell_magics={"../../../tmp/pwned"}) + with cache_dir() as workspace: + cache_file = get_cache_file(mode) + assert cache_file.parent == workspace + assert "/" not in cache_file.name + assert ".." not in cache_file.name + assert "../../../tmp/pwned" not in mode.get_cache_key() + def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: @@ -2416,7 +2452,7 @@ def test_cache_key(self) -> None: # If you are looking to remove one of these features, just # replace it with any other feature. values = [ - {Preview.multiline_string_handling}, + {Preview.wrap_comprehension_in}, {Preview.string_processing}, ] elif field.type is Quote: @@ -2527,8 +2563,8 @@ def test_gitignore_exclude(self) -> None: include = re.compile(r"\.pyi?$") exclude = re.compile(r"") report = pyink.Report() - gitignore = PathSpec.from_lines( - "gitwildmatch", ["exclude/", ".definitely_exclude"] + gitignore = GitIgnoreSpec.from_lines( + ["exclude/", ".definitely_exclude", "!exclude/still_exclude/"] ) sources: list[Path] = [] expected = [ @@ -2552,6 +2588,70 @@ def test_gitignore_exclude(self) -> None: ) assert sorted(expected) == sorted(sources) + def test_gitignore_reinclude(self) -> None: + path = THIS_DIR / "data" / "include_exclude_tests" + include = re.compile(r"\.pyi?$") + exclude = re.compile(r"") + report = pyink.Report() + gitignore = GitIgnoreSpec.from_lines( + ["*/exclude/*", ".definitely_exclude", "!*/exclude/still_exclude/"] + ) + sources: list[Path] = [] + expected = [ + Path(path / "b/dont_exclude/a.py"), + Path(path / "b/dont_exclude/a.pyi"), + Path(path / "b/exclude/still_exclude/a.py"), + Path(path / "b/exclude/still_exclude/a.pyi"), + ] + this_abs = THIS_DIR.resolve() + sources.extend( + pyink.gen_python_files( + path.iterdir(), + this_abs, + include, + exclude, + None, + None, + report, + {path: gitignore}, + verbose=False, + quiet=False, + ) + ) + assert sorted(expected) == sorted(sources) + + def test_gitignore_reinclude_root(self) -> None: + path = THIS_DIR / "data" / "include_exclude_tests" / "b" + include = re.compile(r"\.pyi?$") + exclude = re.compile(r"") + report = pyink.Report() + gitignore = GitIgnoreSpec.from_lines( + ["exclude/*", ".definitely_exclude", "!exclude/still_exclude/"] + ) + sources: list[Path] = [] + expected = [ + Path(path / "dont_exclude/a.py"), + Path(path / "dont_exclude/a.pyi"), + Path(path / "exclude/still_exclude/a.py"), + Path(path / "exclude/still_exclude/a.pyi"), + ] + this_abs = THIS_DIR.resolve() + sources.extend( + pyink.gen_python_files( + path.iterdir(), + this_abs, + include, + exclude, + None, + None, + report, + {path: gitignore}, + verbose=False, + quiet=False, + ) + ) + assert sorted(expected) == sorted(sources) + def test_nested_gitignore(self) -> None: path = Path(THIS_DIR / "data" / "nested_gitignore_tests") include = re.compile(r"\.pyi?$") @@ -2653,6 +2753,9 @@ def test_empty_include(self) -> None: Path(path / "b/exclude/a.pie"), Path(path / "b/exclude/a.py"), Path(path / "b/exclude/a.pyi"), + Path(path / "b/exclude/still_exclude/a.pie"), + Path(path / "b/exclude/still_exclude/a.py"), + Path(path / "b/exclude/still_exclude/a.pyi"), Path(path / "b/dont_exclude/a.pie"), Path(path / "b/dont_exclude/a.py"), Path(path / "b/dont_exclude/a.pyi"), @@ -2680,6 +2783,7 @@ def test_exclude_absolute_path(self) -> None: src = [path] expected = [ Path(path / "b/dont_exclude/a.py"), + Path(path / "b/exclude/still_exclude/a.py"), Path(path / "b/.definitely_exclude/a.py"), ] assert_collected_sources( @@ -2691,6 +2795,7 @@ def test_extend_exclude(self) -> None: src = [path] expected = [ Path(path / "b/exclude/a.py"), + Path(path / "b/exclude/still_exclude/a.py"), Path(path / "b/dont_exclude/a.py"), ] assert_collected_sources( @@ -2703,7 +2808,7 @@ def test_symlinks(self) -> None: include = re.compile(pyink.DEFAULT_INCLUDES) exclude = re.compile(pyink.DEFAULT_EXCLUDES) report = pyink.Report() - gitignore = PathSpec.from_lines("gitwildmatch", []) + gitignore = GitIgnoreSpec.from_lines([]) regular = MagicMock() regular.relative_to.return_value = Path("regular.py") @@ -3167,6 +3272,53 @@ def test_equivalency_ast_parse_failure_includes_error(self) -> None: err.match("invalid character") err.match(r"\(, line 1\)") + def test_target_version_exceeds_runtime_warning(self) -> None: + max_target = max(TargetVersion, key=lambda tv: tv.value) + if sys.version_info[1] >= max_target.value: + pytest.skip("no target version higher than runtime available") + target_name = f"py3{sys.version_info[1] + 1}" + code = "x = 1\n" + args = ["--target-version", target_name, "--code", code] + result = CliRunner().invoke(pyink.main, args) + stderr = result.stderr_bytes.decode() if result.stderr_bytes else "" + assert "Warning:" in stderr + + def test_target_version_exceeds_runtime_no_warning_with_fast(self) -> None: + max_target = max(TargetVersion, key=lambda tv: tv.value) + if sys.version_info[1] >= max_target.value: + pytest.skip("no target version higher than runtime available") + target_name = f"py3{sys.version_info[1] + 1}" + code = "x = 1\n" + args = ["--fast", "--target-version", target_name, "--code", code] + result = CliRunner().invoke(pyink.main, args) + stderr = result.stderr_bytes.decode() if result.stderr_bytes else "" + assert "Warning:" not in stderr + + def test_target_version_at_runtime_no_warning(self) -> None: + current_minor = sys.version_info[1] + target_name = f"py3{current_minor}" + code = "x = 1\n" + args = ["--target-version", target_name, "--code", code] + result = CliRunner().invoke(pyink.main, args) + stderr = result.stderr_bytes.decode() if result.stderr_bytes else "" + assert "Warning:" not in stderr + + @pytest.mark.incompatible_with_mypyc + def test_target_version_exceeds_runtime_clear_error_message(self) -> None: + max_target = max(TargetVersion, key=lambda tv: tv.value) + if sys.version_info[1] >= max_target.value: + pytest.skip("no target version higher than runtime available") + future_target = TargetVersion[f"PY3{sys.version_info[1] + 1}"] + mode = Mode(target_versions={future_target}) + with patch.object( + pyink, + "assert_equivalent", + side_effect=ASTSafetyError("mocked parse failure"), + ): + with pytest.raises(ASTSafetyError) as exc_info: + pyink.check_stability_and_equivalence("x = 1\n", "x = 1\n", mode=mode) + assert "INTERNAL ERROR" not in str(exc_info.value) + try: with open(pyink.__file__, encoding="utf-8") as _bf: diff --git a/tests/test_concurrency_manager_shutdown.py b/tests/test_concurrency_manager_shutdown.py new file mode 100644 index 00000000000..b5e64afb42b --- /dev/null +++ b/tests/test_concurrency_manager_shutdown.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Optional + +import pyink.concurrency as concurrency +from pyink import Mode, WriteBack +from pyink.report import Report + + +class FakeManager: + shutdown_called: bool + + def __init__(self) -> None: + self.shutdown_called = False + + def Lock(self) -> object: + return object() + + def shutdown(self) -> None: + self.shutdown_called = True + + +def test_manager_shutdown_called_for_diff(monkeypatch: Any, tmp_path: Path) -> None: + """ + schedule_formatting() creates multiprocessing.Manager() for DIFF/COLOR_DIFF + and must shut it down deterministically. + """ + fake_manager = FakeManager() + + monkeypatch.setattr(concurrency, "Manager", lambda: fake_manager) + + def fake_format_file_in_place( + src: Path, + fast: bool, + mode: Mode, + write_back: WriteBack, + lock: Optional[object], + ) -> bool: + assert lock is not None + return False + + monkeypatch.setattr(concurrency, "format_file_in_place", fake_format_file_in_place) + + src = tmp_path / "a.py" + src.write_text("x=1\n", encoding="utf8") + + async def run() -> None: + loop = asyncio.get_running_loop() + with ThreadPoolExecutor(max_workers=1) as executor: + await concurrency.schedule_formatting( + sources={src}, + fast=False, + write_back=WriteBack.DIFF, + mode=Mode(), + report=Report(), + loop=loop, + executor=executor, + no_cache=True, + ) + + asyncio.run(run()) + + assert fake_manager.shutdown_called is True diff --git a/tests/test_format.py b/tests/test_format.py index 072574d2338..0d16d9be192 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -49,7 +49,15 @@ def check_file(subdir: str, filename: str, *, data: bool = True) -> None: @pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning") -@pytest.mark.parametrize("filename", all_data_cases("cases")) +@pytest.mark.parametrize( + "filename", + [ + pytest.param(name, marks=pytest.mark.skip(reason="Skipping to suppress black incompatibility.")) + if name in ["preview_comments7", "import_line_collapse"] + else name + for name in all_data_cases("cases") + ], +) def test_simple_format(filename: str) -> None: check_file("cases", filename) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 6b1ef879c3c..ae8a6f28aff 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -6,8 +6,8 @@ from dataclasses import replace import pytest -from _pytest.monkeypatch import MonkeyPatch from click.testing import CliRunner +from pytest import MonkeyPatch from pyink import ( Mode, @@ -18,7 +18,12 @@ format_file_in_place, main, ) -from pyink.handle_ipynb_magics import jupyter_dependencies_are_installed +from pyink.handle_ipynb_magics import ( + Replacement, + create_token, + jupyter_dependencies_are_installed, + unmask_cell, +) from tests.util import DATA_DIR, get_case_path, read_jupyter_notebook with contextlib.suppress(ModuleNotFoundError): @@ -47,6 +52,17 @@ def test_noop() -> None: format_cell(src, fast=True, mode=JUPYTER_MODE) +@pytest.mark.parametrize("n_chars", [1, 2, 3, 4, 5, 17]) +def test_create_token_uses_requested_length(n_chars: int) -> None: + assert len(create_token(n_chars)) == n_chars + + +def test_unmask_cell_raises_when_token_is_not_unique() -> None: + replacement = Replacement(mask='b"dead"', src="%time") + with pytest.raises(NothingChanged): + unmask_cell(f"{replacement.mask}\nvalue = {replacement.mask}", [replacement]) + + @pytest.mark.parametrize("fast", [True, False]) def test_trailing_semicolon(fast: bool) -> None: src = 'foo = "a" ;' diff --git a/tox.ini b/tox.ini index ed483b62d80..bf4cfd399be 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,7 @@ skip_install = True # the `no_jupyter` tests would run with the jupyter extra dependencies installed. # See https://github.com/psf/black/issues/2367. recreate = True -deps = - -r{toxinidir}/test_requirements.txt +dependency_groups = cov-tests commands = pip install -e .[d] coverage erase @@ -27,51 +26,23 @@ commands = coverage report [testenv:{,ci-}pypy3] -setenv = PYTHONPATH = {toxinidir}/src +setenv = + PYTHONPATH = {toxinidir}/src skip_install = True recreate = True -deps = - -r{toxinidir}/test_requirements.txt +dependency_groups = tests commands = pip install -e .[d] - pytest tests \ - --run-optional no_jupyter \ + pytest tests --run-optional no_jupyter \ --numprocesses auto pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ --numprocesses auto -[testenv:{,ci-}311] -setenv = - PYTHONPATH = {toxinidir}/src - AIOHTTP_NO_EXTENSIONS = 1 -skip_install = True -recreate = True -deps = -; We currently need > aiohttp 3.8.1 that is on PyPI for 3.11 - git+https://github.com/aio-libs/aiohttp - -r{toxinidir}/test_requirements.txt -commands = - pip install -e .[d] - coverage erase - pytest tests \ - --run-optional no_jupyter \ - --numprocesses auto \ - --cov {posargs} - pip install -e .[jupyter] - pytest tests --run-optional jupyter \ - -m jupyter \ - --numprocesses auto \ - --cov --cov-append {posargs} - coverage report - [testenv:fuzz] skip_install = True -deps = - -r{toxinidir}/test_requirements.txt - hypothesmith - lark-parser +dependency_groups = fuzz commands = pip install -e .[d] coverage erase @@ -79,8 +50,17 @@ commands = coverage report [testenv:run_self] -setenv = PYTHONPATH = {toxinidir}/src +setenv = + PYTHONPATH = {toxinidir}/src +skip_install = True +commands = + pip install -e . + black --check {toxinidir} + +[testenv:generate_schema] +setenv = + PYTHONWARNDEFAULTENCODING = skip_install = True commands = pip install -e . - pyink --check {toxinidir}/src {toxinidir}/tests {toxinidir}/docs {toxinidir}/scripts + python {toxinidir}/scripts/generate_schema.py --outfile {toxinidir}/src/black/resources/black.schema.json