From bd42cba1ef502ad63694b956956c81b402038007 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Thu, 13 Jul 2023 19:50:54 +0100 Subject: [PATCH 1/4] Add some ruff autofixes to CI --- .pre-commit-config.yaml | 8 ++--- CONTRIBUTING.md | 8 ++--- pyproject.toml | 50 +++++++++++++++++++++++++++----- requirements-tests.txt | 2 +- scripts/create_baseline_stubs.py | 10 +++---- scripts/generate_proto_stubs.sh | 4 +-- scripts/runtests.py | 4 +-- tests/check_consistent.py | 5 +++- 8 files changed, 65 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c03eec00f65a..5ccec269febd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,11 +21,11 @@ repos: hooks: - id: black language_version: python3.10 - - repo: https://github.com/pycqa/isort - rev: 5.12.0 # must match requirements-tests.txt + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.278 # must match requirements-tests.txt hooks: - - id: isort - name: isort (python) + - id: ruff + args: [--exit-non-zero-on-fix] - repo: https://github.com/pycqa/flake8 rev: 6.0.0 # must match requirements-tests.txt hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a185c385fd0d..4508159f67fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ it takes a bit longer. For more details, read below. Typeshed runs continuous integration (CI) on all pull requests. This means that if you file a pull request (PR), our full test suite -- including our linter, `flake8` -- is run on your PR. It also means that bots will automatically apply -changes to your PR (using `pycln`, `black` and `isort`) to fix any formatting issues. +changes to your PR (using `pycln`, `black` and `ruff`) to fix any formatting issues. This frees you up to ignore all local setup on your side, focus on the code and rely on the CI to fix everything, or point you to the places that need fixing. @@ -84,7 +84,7 @@ terminal to install all non-pytype requirements: ## Code formatting -The code is formatted using `black` and `isort`. Unused imports are also +The code is formatted using `black` and `ruff`. Unused imports are also auto-removed using `pycln`. The repository is equipped with a [`pre-commit.ci`](https://pre-commit.ci/) @@ -93,11 +93,11 @@ run the code formatters. When you push a commit, a bot will run those for you right away and add a commit to your PR. That being said, if you *want* to run the checks locally when you commit, -you're free to do so. Either run `pycln`, `black` and `isort` manually... +you're free to do so. Either run `pycln`, `black` and `ruff` manually... ```bash $ pycln --config=pyproject.toml . -$ isort . +$ ruff . $ black . ``` diff --git a/pyproject.toml b/pyproject.toml index 663e9a65bb8f..b8c830e5abfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,12 +7,48 @@ skip_magic_trailing_comma = true # for just these files, but doesn't seem possible yet. force-exclude = ".*_pb2.pyi" -[tool.isort] -profile = "black" -combine_as_imports = true -line_length = 130 -skip = [".git", ".github", ".venv"] -extra_standard_library = [ +[tool.ruff] +line-length = 130 +target-version = "py37" +fix = true +exclude = [ + # We're only interested in autofixes for our stubs + "*.py", + # Ignore generated protobuf stubs + "*_pb2.pyi", + # virtual environment, cache directories, etc.: + "env", + ".env", + ".venv", + ".git", + ".mypy_cache", + ".pytype", +] + +# Only enable rules that have safe autofixes; +# only enable rules that are relevant to stubs +select = [ + "I001", # isort + "UP004", # Remove explicit `object` inheritance + "UP006", # PEP-585 autofixes + "UP007", # PEP-604 autofixes + "UP013", # Class-based syntax for TypedDicts + "UP014", # Class-based syntax for NamedTuples + "UP019", # Use str over typing.Text + "UP035", # import from typing, not typing_extensions, wherever possible + "UP039", # don't use parens after a class definition with no bases + "PYI009", # use `...`, not `pass`, in empty class bodies + "PYI010", # function bodies must be empty + "PYI012", # class bodies must not contain `pass` + "PYI013", # non-empty class bodies must not contain `...` + "PYI020", # quoted annotations are always unnecessary in stubs + "PYI025", # always alias `collections.abc.Set` as `AbstractSet` when importing it + "PYI032", # use `object`, not `Any`, as the second parameter to `__eq__` +] + +[tool.ruff.isort] +combine-as-imports = true +extra-standard-library = [ "typing_extensions", "_typeshed", # Extra modules not recognized by isort @@ -55,7 +91,7 @@ extra_standard_library = [ "opcode", "pyexpat", ] -known_first_party = ["utils", "parse_metadata"] +known-first-party = ["utils", "parse_metadata"] [tool.pycln] all = true diff --git a/requirements-tests.txt b/requirements-tests.txt index 5a53fbb55883..db26f3fd4b79 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,11 +6,11 @@ flake8==6.0.0; python_version >= "3.8" # must match .pre-commit-confi flake8-bugbear==23.6.5; python_version >= "3.8" # must match .pre-commit-config.yaml flake8-noqa==1.3.2; python_version >= "3.8" # must match .pre-commit-config.yaml flake8-pyi==23.6.0; python_version >= "3.8" # must match .pre-commit-config.yaml -isort==5.12.0; python_version >= "3.8" # must match .pre-commit-config.yaml mypy==1.4.1 pre-commit-hooks==4.4.0 # must match .pre-commit-config.yaml pycln==2.1.5 # must match .pre-commit-config.yaml pytype==2023.6.16; platform_system != "Windows" and python_version < "3.11" +ruff==0.0.278 # must match .pre-commit-config.yaml # Libraries used by our various scripts. aiohttp==3.8.4; python_version < "3.12" # aiohttp can't be installed on 3.12 yet diff --git a/scripts/create_baseline_stubs.py b/scripts/create_baseline_stubs.py index 4675ee58bce0..6279a7645936 100755 --- a/scripts/create_baseline_stubs.py +++ b/scripts/create_baseline_stubs.py @@ -60,9 +60,9 @@ def run_black(stub_dir: str) -> None: subprocess.run(["black", stub_dir]) -def run_isort(stub_dir: str) -> None: - print(f"Running isort: isort {stub_dir}") - subprocess.run([sys.executable, "-m", "isort", stub_dir]) +def run_ruff(stub_dir: str) -> None: + print(f"Running ruff: ruff {stub_dir}") + subprocess.run([sys.executable, "-m", "ruff", stub_dir]) def create_metadata(stub_dir: str, version: str) -> None: @@ -110,7 +110,7 @@ def add_pyright_exclusion(stub_dir: str) -> None: def main() -> None: parser = argparse.ArgumentParser( description="""Generate baseline stubs automatically for an installed pip package - using stubgen. Also run black and isort. If the name of + using stubgen. Also run black and ruff. If the name of the project is different from the runtime Python package name, you may need to use --package (example: --package yaml PyYAML).""" ) @@ -159,7 +159,7 @@ def main() -> None: run_stubgen(package, stub_dir) run_stubdefaulter(stub_dir) - run_isort(stub_dir) + run_ruff(stub_dir) run_black(stub_dir) create_metadata(stub_dir, version) diff --git a/scripts/generate_proto_stubs.sh b/scripts/generate_proto_stubs.sh index f1315b8a466a..249ca81f12ef 100755 --- a/scripts/generate_proto_stubs.sh +++ b/scripts/generate_proto_stubs.sh @@ -45,7 +45,7 @@ PYTHON_PROTOBUF_DIR="protobuf-$PYTHON_PROTOBUF_VERSION" VENV=venv python3 -m venv "$VENV" source "$VENV/bin/activate" -pip install -r "$REPO_ROOT/requirements-tests.txt" # for black and isort +pip install -r "$REPO_ROOT/requirements-tests.txt" # for black and ruff # Install mypy-protobuf pip install "git+https://github.com/dropbox/mypy-protobuf@$MYPY_PROTOBUF_VERSION" @@ -73,7 +73,7 @@ PROTO_FILES=$(grep "GenProto.*google" $PYTHON_PROTOBUF_DIR/python/setup.py | \ # shellcheck disable=SC2086 protoc_install/bin/protoc --proto_path="$PYTHON_PROTOBUF_DIR/src" --mypy_out="relax_strict_optional_primitives:$REPO_ROOT/stubs/protobuf" $PROTO_FILES -isort "$REPO_ROOT/stubs/protobuf" +ruff "$REPO_ROOT/stubs/protobuf" black "$REPO_ROOT/stubs/protobuf" sed -i "" "s/mypy-protobuf [^\"]*/mypy-protobuf ${MYPY_PROTOBUF_VERSION}/" "$REPO_ROOT/stubs/protobuf/METADATA.toml" diff --git a/scripts/runtests.py b/scripts/runtests.py index 8c6efd71cb0f..1b0a55e7a1c8 100644 --- a/scripts/runtests.py +++ b/scripts/runtests.py @@ -77,8 +77,8 @@ def main() -> None: # Run formatters first. Order matters. print("\nRunning pycln...") subprocess.run([sys.executable, "-m", "pycln", path, "--config=pyproject.toml"]) - print("\nRunning isort...") - subprocess.run([sys.executable, "-m", "isort", path]) + print("\nRunning ruff...") + subprocess.run([sys.executable, "-m", "ruff", path]) print("\nRunning Black...") black_result = subprocess.run([sys.executable, "-m", "black", path]) if black_result.returncode == 123: diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 6b62d2869150..aa8eb4e9dc4c 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -23,7 +23,7 @@ # These type checkers and linters must have exact versions in the requirements file to ensure # consistent CI runs. -linters = {"black", "flake8", "flake8-bugbear", "flake8-noqa", "flake8-pyi", "isort", "mypy", "pycln", "pytype"} +linters = {"black", "flake8", "flake8-bugbear", "flake8-noqa", "flake8-pyi", "ruff", "mypy", "pycln", "pytype"} def assert_consistent_filetypes( @@ -191,6 +191,9 @@ def check_precommit_requirements() -> None: precommit_requirements = get_precommit_requirements() no_txt_entry_msg = "All pre-commit requirements must also be listed in `requirements-tests.txt` (missing {requirement!r})" for requirement, specifier in precommit_requirements.items(): + # annoying: the ruff repo for pre-commit is different to the name in requirements-tests.txt + if requirement == "ruff-pre-commit": + requirement = "ruff" assert requirement in requirements_txt_requirements, no_txt_entry_msg.format(requirement=requirement) specifier_mismatch = ( f'Specifier "{specifier}" for {requirement!r} in `.pre-commit-config.yaml` ' From 7b5497520cc469245d9020022833bf878c5c3403 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 19 Jul 2023 13:16:39 +0100 Subject: [PATCH 2/4] Add isort back --- .pre-commit-config.yaml | 5 ++ CONTRIBUTING.md | 9 ++-- pyproject.toml | 88 ++++++++++++++++---------------- requirements-tests.txt | 1 + scripts/create_baseline_stubs.py | 8 ++- scripts/generate_proto_stubs.sh | 4 +- scripts/runtests.py | 2 + tests/check_consistent.py | 2 +- 8 files changed, 68 insertions(+), 51 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ccec269febd..9c451ffe34af 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,6 +21,11 @@ repos: hooks: - id: black language_version: python3.10 + - repo: https://github.com/pycqa/isort + rev: 5.12.0 # must match requirements-tests.txt + hooks: + - id: isort + name: isort (python) - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.0.278 # must match requirements-tests.txt hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4508159f67fe..1b3e15cf4905 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ it takes a bit longer. For more details, read below. Typeshed runs continuous integration (CI) on all pull requests. This means that if you file a pull request (PR), our full test suite -- including our linter, `flake8` -- is run on your PR. It also means that bots will automatically apply -changes to your PR (using `pycln`, `black` and `ruff`) to fix any formatting issues. +changes to your PR (using `pycln`, `black`, `isort` and `ruff`) to fix any formatting issues. This frees you up to ignore all local setup on your side, focus on the code and rely on the CI to fix everything, or point you to the places that need fixing. @@ -84,8 +84,8 @@ terminal to install all non-pytype requirements: ## Code formatting -The code is formatted using `black` and `ruff`. Unused imports are also -auto-removed using `pycln`. +The code is formatted using `black` and `isort`. Unused imports are also +auto-removed using `pycln`, and various other autofixes are performed by `ruff`. The repository is equipped with a [`pre-commit.ci`](https://pre-commit.ci/) configuration file. This means that you don't *need* to do anything yourself to @@ -93,10 +93,11 @@ run the code formatters. When you push a commit, a bot will run those for you right away and add a commit to your PR. That being said, if you *want* to run the checks locally when you commit, -you're free to do so. Either run `pycln`, `black` and `ruff` manually... +you're free to do so. Either run `pycln`, `isort`, `black` and `ruff` manually... ```bash $ pycln --config=pyproject.toml . +$ isort . $ ruff . $ black . ``` diff --git a/pyproject.toml b/pyproject.toml index b8c830e5abfc..316c04116ea5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,48 +7,12 @@ skip_magic_trailing_comma = true # for just these files, but doesn't seem possible yet. force-exclude = ".*_pb2.pyi" -[tool.ruff] -line-length = 130 -target-version = "py37" -fix = true -exclude = [ - # We're only interested in autofixes for our stubs - "*.py", - # Ignore generated protobuf stubs - "*_pb2.pyi", - # virtual environment, cache directories, etc.: - "env", - ".env", - ".venv", - ".git", - ".mypy_cache", - ".pytype", -] - -# Only enable rules that have safe autofixes; -# only enable rules that are relevant to stubs -select = [ - "I001", # isort - "UP004", # Remove explicit `object` inheritance - "UP006", # PEP-585 autofixes - "UP007", # PEP-604 autofixes - "UP013", # Class-based syntax for TypedDicts - "UP014", # Class-based syntax for NamedTuples - "UP019", # Use str over typing.Text - "UP035", # import from typing, not typing_extensions, wherever possible - "UP039", # don't use parens after a class definition with no bases - "PYI009", # use `...`, not `pass`, in empty class bodies - "PYI010", # function bodies must be empty - "PYI012", # class bodies must not contain `pass` - "PYI013", # non-empty class bodies must not contain `...` - "PYI020", # quoted annotations are always unnecessary in stubs - "PYI025", # always alias `collections.abc.Set` as `AbstractSet` when importing it - "PYI032", # use `object`, not `Any`, as the second parameter to `__eq__` -] - -[tool.ruff.isort] -combine-as-imports = true -extra-standard-library = [ +[tool.isort] +profile = "black" +combine_as_imports = true +line_length = 130 +skip = [".git", ".github", ".venv"] +extra_standard_library = [ "typing_extensions", "_typeshed", # Extra modules not recognized by isort @@ -91,7 +55,45 @@ extra-standard-library = [ "opcode", "pyexpat", ] -known-first-party = ["utils", "parse_metadata"] +known_first_party = ["utils", "parse_metadata"] + +[tool.ruff] +line-length = 130 +target-version = "py37" +fix = true +exclude = [ + # We're only interested in autofixes for our stubs + "*.py", + # Ignore generated protobuf stubs + "*_pb2.pyi", + # virtual environment, cache directories, etc.: + "env", + ".env", + ".venv", + ".git", + ".mypy_cache", + ".pytype", +] + +# Only enable rules that have safe autofixes; +# only enable rules that are relevant to stubs +select = [ + "UP004", # Remove explicit `object` inheritance + "UP006", # PEP-585 autofixes + "UP007", # PEP-604 autofixes + "UP013", # Class-based syntax for TypedDicts + "UP014", # Class-based syntax for NamedTuples + "UP019", # Use str over typing.Text + "UP035", # import from typing, not typing_extensions, wherever possible + "UP039", # don't use parens after a class definition with no bases + "PYI009", # use `...`, not `pass`, in empty class bodies + "PYI010", # function bodies must be empty + "PYI012", # class bodies must not contain `pass` + "PYI013", # non-empty class bodies must not contain `...` + "PYI020", # quoted annotations are always unnecessary in stubs + "PYI025", # always alias `collections.abc.Set` as `AbstractSet` when importing it + "PYI032", # use `object`, not `Any`, as the second parameter to `__eq__` +] [tool.pycln] all = true diff --git a/requirements-tests.txt b/requirements-tests.txt index db26f3fd4b79..13f6313d3f86 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,6 +6,7 @@ flake8==6.0.0; python_version >= "3.8" # must match .pre-commit-confi flake8-bugbear==23.6.5; python_version >= "3.8" # must match .pre-commit-config.yaml flake8-noqa==1.3.2; python_version >= "3.8" # must match .pre-commit-config.yaml flake8-pyi==23.6.0; python_version >= "3.8" # must match .pre-commit-config.yaml +isort==5.12.0; python_version >= "3.8" # must match .pre-commit-config.yaml mypy==1.4.1 pre-commit-hooks==4.4.0 # must match .pre-commit-config.yaml pycln==2.1.5 # must match .pre-commit-config.yaml diff --git a/scripts/create_baseline_stubs.py b/scripts/create_baseline_stubs.py index 6279a7645936..b2569be95353 100755 --- a/scripts/create_baseline_stubs.py +++ b/scripts/create_baseline_stubs.py @@ -60,6 +60,11 @@ def run_black(stub_dir: str) -> None: subprocess.run(["black", stub_dir]) +def run_isort(stub_dir: str) -> None: + print(f"Running isort: isort {stub_dir}") + subprocess.run([sys.executable, "-m", "isort", stub_dir]) + + def run_ruff(stub_dir: str) -> None: print(f"Running ruff: ruff {stub_dir}") subprocess.run([sys.executable, "-m", "ruff", stub_dir]) @@ -110,7 +115,7 @@ def add_pyright_exclusion(stub_dir: str) -> None: def main() -> None: parser = argparse.ArgumentParser( description="""Generate baseline stubs automatically for an installed pip package - using stubgen. Also run black and ruff. If the name of + using stubgen. Also run black, isort and ruff. If the name of the project is different from the runtime Python package name, you may need to use --package (example: --package yaml PyYAML).""" ) @@ -159,6 +164,7 @@ def main() -> None: run_stubgen(package, stub_dir) run_stubdefaulter(stub_dir) + run_isort(stub_dir) run_ruff(stub_dir) run_black(stub_dir) diff --git a/scripts/generate_proto_stubs.sh b/scripts/generate_proto_stubs.sh index 249ca81f12ef..f1315b8a466a 100755 --- a/scripts/generate_proto_stubs.sh +++ b/scripts/generate_proto_stubs.sh @@ -45,7 +45,7 @@ PYTHON_PROTOBUF_DIR="protobuf-$PYTHON_PROTOBUF_VERSION" VENV=venv python3 -m venv "$VENV" source "$VENV/bin/activate" -pip install -r "$REPO_ROOT/requirements-tests.txt" # for black and ruff +pip install -r "$REPO_ROOT/requirements-tests.txt" # for black and isort # Install mypy-protobuf pip install "git+https://github.com/dropbox/mypy-protobuf@$MYPY_PROTOBUF_VERSION" @@ -73,7 +73,7 @@ PROTO_FILES=$(grep "GenProto.*google" $PYTHON_PROTOBUF_DIR/python/setup.py | \ # shellcheck disable=SC2086 protoc_install/bin/protoc --proto_path="$PYTHON_PROTOBUF_DIR/src" --mypy_out="relax_strict_optional_primitives:$REPO_ROOT/stubs/protobuf" $PROTO_FILES -ruff "$REPO_ROOT/stubs/protobuf" +isort "$REPO_ROOT/stubs/protobuf" black "$REPO_ROOT/stubs/protobuf" sed -i "" "s/mypy-protobuf [^\"]*/mypy-protobuf ${MYPY_PROTOBUF_VERSION}/" "$REPO_ROOT/stubs/protobuf/METADATA.toml" diff --git a/scripts/runtests.py b/scripts/runtests.py index 1b0a55e7a1c8..001e4eff7e5c 100644 --- a/scripts/runtests.py +++ b/scripts/runtests.py @@ -79,6 +79,8 @@ def main() -> None: subprocess.run([sys.executable, "-m", "pycln", path, "--config=pyproject.toml"]) print("\nRunning ruff...") subprocess.run([sys.executable, "-m", "ruff", path]) + print("\nRunning isort...") + subprocess.run([sys.executable, "-m", "isort", path]) print("\nRunning Black...") black_result = subprocess.run([sys.executable, "-m", "black", path]) if black_result.returncode == 123: diff --git a/tests/check_consistent.py b/tests/check_consistent.py index aa8eb4e9dc4c..9eb8ffb26f12 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -23,7 +23,7 @@ # These type checkers and linters must have exact versions in the requirements file to ensure # consistent CI runs. -linters = {"black", "flake8", "flake8-bugbear", "flake8-noqa", "flake8-pyi", "ruff", "mypy", "pycln", "pytype"} +linters = {"black", "flake8", "flake8-bugbear", "flake8-noqa", "flake8-pyi", "isort", "ruff", "mypy", "pycln", "pytype"} def assert_consistent_filetypes( From 8085518bbf992bc0b5adeee76a9a382276f52b75 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 19 Jul 2023 12:17:54 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks --- stdlib/typing_extensions.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/typing_extensions.pyi b/stdlib/typing_extensions.pyi index 93087a45a108..aa55d29b1b4d 100644 --- a/stdlib/typing_extensions.pyi +++ b/stdlib/typing_extensions.pyi @@ -113,7 +113,7 @@ __all__ = [ _T = typing.TypeVar("_T") _F = typing.TypeVar("_F", bound=Callable[..., Any]) -_TC = typing.TypeVar("_TC", bound=Type[object]) +_TC = typing.TypeVar("_TC", bound=type[object]) # unfortunately we have to duplicate this class definition from typing.pyi or we break pytype class _SpecialForm: From 2a30ffbbb17a457771f0bda4f2f17fa0ab688a51 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 19 Jul 2023 18:00:15 +0100 Subject: [PATCH 4/4] Run ruff first in `create_baseline_stubs.py` Co-authored-by: Nikita Sobolev --- scripts/create_baseline_stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_baseline_stubs.py b/scripts/create_baseline_stubs.py index b2569be95353..7d41fad37b60 100755 --- a/scripts/create_baseline_stubs.py +++ b/scripts/create_baseline_stubs.py @@ -164,8 +164,8 @@ def main() -> None: run_stubgen(package, stub_dir) run_stubdefaulter(stub_dir) - run_isort(stub_dir) run_ruff(stub_dir) + run_isort(stub_dir) run_black(stub_dir) create_metadata(stub_dir, version)