From 74b587ec725be94ab41917417a5cc774e3e8e01b Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 09:58:46 +0800 Subject: [PATCH 01/17] drop python 3.8 --- packages/http-client-python/emitter/src/run-python3.ts | 2 +- packages/http-client-python/eng/scripts/ci/mypy.ini | 2 +- .../http-client-python/eng/scripts/ci/pyrightconfig.json | 2 +- packages/http-client-python/eng/scripts/ci/run_pylint.py | 2 +- .../eng/scripts/setup/build_pygen_wheel.py | 6 +++--- packages/http-client-python/eng/scripts/setup/install.py | 4 ++-- packages/http-client-python/eng/scripts/setup/prepare.py | 4 ++-- .../codegen/templates/packaging_templates/README.md.jinja2 | 4 ++-- .../codegen/templates/packaging_templates/setup.py.jinja2 | 3 +-- packages/http-client-python/generator/setup.py | 1 - 10 files changed, 14 insertions(+), 16 deletions(-) diff --git a/packages/http-client-python/emitter/src/run-python3.ts b/packages/http-client-python/emitter/src/run-python3.ts index e4df6e8a714..9336f672672 100644 --- a/packages/http-client-python/emitter/src/run-python3.ts +++ b/packages/http-client-python/emitter/src/run-python3.ts @@ -11,7 +11,7 @@ import { patchPythonPath } from "./system-requirements.js"; export async function runPython3(...args: string[]) { const command = await patchPythonPath(["python", ...args], { - version: ">=3.8", + version: ">=3.9", environmentVariable: "AUTOREST_PYTHON_EXE", }); cp.execSync(command.join(" "), { diff --git a/packages/http-client-python/eng/scripts/ci/mypy.ini b/packages/http-client-python/eng/scripts/ci/mypy.ini index 7eed55db035..7ed9b727577 100644 --- a/packages/http-client-python/eng/scripts/ci/mypy.ini +++ b/packages/http-client-python/eng/scripts/ci/mypy.ini @@ -1,6 +1,6 @@ # global configurations [mypy] -python_version = 3.8 +python_version = 3.9 # module level configurations diff --git a/packages/http-client-python/eng/scripts/ci/pyrightconfig.json b/packages/http-client-python/eng/scripts/ci/pyrightconfig.json index 8992909c9c4..86b3a380220 100644 --- a/packages/http-client-python/eng/scripts/ci/pyrightconfig.json +++ b/packages/http-client-python/eng/scripts/ci/pyrightconfig.json @@ -2,5 +2,5 @@ "reportUnnecessaryCast": "warning", "reportTypeCommentUsage": true, "reportMissingImports": false, - "pythonVersion": "3.8" + "pythonVersion": "3.9" } diff --git a/packages/http-client-python/eng/scripts/ci/run_pylint.py b/packages/http-client-python/eng/scripts/ci/run_pylint.py index 2a9d362f0c8..370cf605129 100644 --- a/packages/http-client-python/eng/scripts/ci/run_pylint.py +++ b/packages/http-client-python/eng/scripts/ci/run_pylint.py @@ -38,7 +38,7 @@ def _single_dir_pylint(mod): "--load-plugins=pylint_guidelines_checker", "--output-format=parseable", "--recursive=y", - "--py-version=3.8", + "--py-version=3.9", str(inner_class.absolute()), ] ) diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index f0c60d41087..2e3118d46c8 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -7,8 +7,8 @@ # -------------------------------------------------------------------------- import sys -if not sys.version_info >= (3, 8, 0): - raise Exception("Autorest for Python extension requires Python 3.8 at least") +if not sys.version_info >= (3, 9, 0): + raise Exception("Autorest for Python extension requires Python 3.9 at least") try: import pip @@ -16,7 +16,7 @@ raise Exception("Your Python installation doesn't have pip available") -# Now we have pip and Py >= 3.8, go to work +# Now we have pip and Py >= 3.9, go to work from pathlib import Path diff --git a/packages/http-client-python/eng/scripts/setup/install.py b/packages/http-client-python/eng/scripts/setup/install.py index eeebc1a8a77..e58d309a7c8 100644 --- a/packages/http-client-python/eng/scripts/setup/install.py +++ b/packages/http-client-python/eng/scripts/setup/install.py @@ -7,9 +7,9 @@ # -------------------------------------------------------------------------- import sys -if not sys.version_info >= (3, 8, 0): +if not sys.version_info >= (3, 9, 0): raise Warning( - "Autorest for Python extension requires Python 3.8 at least. We will run your code with Pyodide since your Python version isn't adequate." + "Autorest for Python extension requires Python 3.9 at least. We will run your code with Pyodide since your Python version isn't adequate." ) try: diff --git a/packages/http-client-python/eng/scripts/setup/prepare.py b/packages/http-client-python/eng/scripts/setup/prepare.py index 4e5ffd432fc..946c54a7d93 100644 --- a/packages/http-client-python/eng/scripts/setup/prepare.py +++ b/packages/http-client-python/eng/scripts/setup/prepare.py @@ -9,9 +9,9 @@ import os import argparse -if not sys.version_info >= (3, 8, 0): +if not sys.version_info >= (3, 9, 0): raise Warning( - "Autorest for Python extension requires Python 3.8 at least. We will run your code with Pyodide since your Python version isn't adequate." + "Autorest for Python extension requires Python 3.9 at least. We will run your code with Pyodide since your Python version isn't adequate." ) from pathlib import Path diff --git a/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 index f7dc4f8de52..c79dd57c470 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 @@ -3,7 +3,7 @@ # Microsoft Azure SDK for Python This is the Microsoft {{package_pprint_name}} Client Library. -This package has been tested with Python 3.8+. +This package has been tested with Python 3.9+. For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all). # Usage @@ -36,7 +36,7 @@ python -m pip install {{ package_name }} #### Prequisites -- Python 3.8 or later is required to use this package. +- Python 3.9 or later is required to use this package. - You need an [Azure subscription][azure_sub] to use this package. - An existing {{ package_pprint_name }} instance. diff --git a/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 index 2879b4a6716..ffd3d023e79 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 @@ -67,7 +67,6 @@ setup( "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -110,7 +109,7 @@ setup( "typing-extensions>=4.6.0", ], {% if package_mode %} - python_requires=">=3.8", + python_requires=">=3.9", {% else %} long_description="""\ {{ code_model.description }} diff --git a/packages/http-client-python/generator/setup.py b/packages/http-client-python/generator/setup.py index e6572d6cd17..251a42b82a7 100644 --- a/packages/http-client-python/generator/setup.py +++ b/packages/http-client-python/generator/setup.py @@ -35,7 +35,6 @@ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 0540245d6c0299162ee06070f8845a6097f29094 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 11:05:31 +0800 Subject: [PATCH 02/17] fix ci --- .../pipelines/templates/jobs/build-job.yml | 5 +++++ eng/emitters/pipelines/templates/jobs/test-job.yml | 5 +++++ .../pipelines/templates/stages/emitter-stages.yml | 14 ++++++++++++++ .../pipelines/templates/steps/build-step.yml | 9 +++++++++ .../pipelines/templates/steps/test-step.yml | 9 +++++++++ .../http-client-python/eng/pipeline/publish.yml | 1 + 6 files changed, 43 insertions(+) diff --git a/eng/emitters/pipelines/templates/jobs/build-job.yml b/eng/emitters/pipelines/templates/jobs/build-job.yml index 60144be2698..8380c734c35 100644 --- a/eng/emitters/pipelines/templates/jobs/build-job.yml +++ b/eng/emitters/pipelines/templates/jobs/build-job.yml @@ -50,6 +50,10 @@ parameters: type: string default: "" + - name: PythonVersion + type: string + default: "3.12" + jobs: - job: Build_${{ parameters.Os }}_${{ split(parameters.NodeVersion, '.')[0] }} ${{ if eq(parameters.Os, 'linux') }}: @@ -73,6 +77,7 @@ jobs: LanguageShortName: ${{ parameters.LanguageShortName }} PackagePath: ${{ parameters.PackagePath }} NodeVersion: ${{ parameters.NodeVersion }} + PythonVersion: ${{ parameters.PythonVersion }} ${{ if parameters.EmitArtifacts }}: templateContext: outputs: diff --git a/eng/emitters/pipelines/templates/jobs/test-job.yml b/eng/emitters/pipelines/templates/jobs/test-job.yml index 4fd7f8edc75..e2fa773bb62 100644 --- a/eng/emitters/pipelines/templates/jobs/test-job.yml +++ b/eng/emitters/pipelines/templates/jobs/test-job.yml @@ -48,6 +48,10 @@ parameters: type: boolean default: true + - name: PythonVersion + type: string + default: "3.12" + jobs: - job: Test_${{ parameters.Os }}_${{ split(parameters.NodeVersion, '.')[0] }} ${{ if eq(parameters.Os, 'linux') }}: @@ -78,6 +82,7 @@ jobs: EmitArtifacts: ${{ parameters.EmitArtifacts }} CadlRanchName: ${{ parameters.CadlRanchName }} EnableCadlRanchReport: ${{ parameters.EnableCadlRanchReport }} + PythonVersion: ${{ parameters.PythonVersion }} ${{ if parameters.EmitArtifacts }}: templateContext: outputs: diff --git a/eng/emitters/pipelines/templates/stages/emitter-stages.yml b/eng/emitters/pipelines/templates/stages/emitter-stages.yml index 7b2bdc27efe..32973e6bfc6 100644 --- a/eng/emitters/pipelines/templates/stages/emitter-stages.yml +++ b/eng/emitters/pipelines/templates/stages/emitter-stages.yml @@ -76,6 +76,10 @@ parameters: type: boolean default: true + - name: PythonVersion + type: string + default: "3.12" + stages: # Build stage # Responsible for building the autorest generator and typespec emitter packages @@ -104,6 +108,7 @@ stages: NodeVersion: 20.x Os: linux EmitArtifacts: true # Emit artifacts only for the first job + PythonVersion: ${{ parameters.PythonVersion }} - template: /eng/emitters/pipelines/templates/jobs/build-job.yml parameters: BuildPrereleaseVersion: ${{ parameters.BuildPrereleaseVersion }} @@ -115,6 +120,7 @@ stages: LanguageShortName: ${{ parameters.LanguageShortName }} NodeVersion: 22.x Os: linux + PythonVersion: ${{ parameters.PythonVersion }} - template: /eng/emitters/pipelines/templates/jobs/build-job.yml parameters: BuildPrereleaseVersion: ${{ parameters.BuildPrereleaseVersion }} @@ -126,6 +132,7 @@ stages: LanguageShortName: ${{ parameters.LanguageShortName }} NodeVersion: 20.x Os: windows + PythonVersion: ${{ parameters.PythonVersion }} - template: /eng/emitters/pipelines/templates/jobs/build-job.yml parameters: BuildPrereleaseVersion: ${{ parameters.BuildPrereleaseVersion }} @@ -137,6 +144,7 @@ stages: LanguageShortName: ${{ parameters.LanguageShortName }} NodeVersion: 22.x Os: windows + PythonVersion: ${{ parameters.PythonVersion }} - ${{ if eq(parameters.Publish, 'none') }}: - template: /eng/emitters/pipelines/templates/jobs/detect-api-changes.yml parameters: @@ -175,6 +183,7 @@ stages: EmitArtifacts: true # Emit artifacts only for the first job CadlRanchName: ${{ parameters.CadlRanchName }} # only needed for first job EnableCadlRanchReport: ${{ parameters.EnableCadlRanchReport }} + PythonVersion: ${{ parameters.PythonVersion }} - template: /eng/emitters/pipelines/templates/jobs/test-job.yml parameters: AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }} @@ -184,6 +193,7 @@ stages: BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }} NodeVersion: 22.x Os: linux + PythonVersion: ${{ parameters.PythonVersion }} - template: /eng/emitters/pipelines/templates/jobs/test-job.yml parameters: AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }} @@ -193,6 +203,7 @@ stages: BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }} NodeVersion: 20.x Os: windows + PythonVersion: ${{ parameters.PythonVersion }} - template: /eng/emitters/pipelines/templates/jobs/test-job.yml parameters: AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }} @@ -202,6 +213,7 @@ stages: BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }} NodeVersion: 22.x Os: windows + PythonVersion: ${{ parameters.PythonVersion }} # Regen Test stage # Responsible for running any regen tests needed to validate no manual changes were done. @@ -225,6 +237,7 @@ stages: BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }} Os: windows TestMatrix: ${{ parameters.TestMatrix }} + PythonVersion: ${{ parameters.PythonVersion }} - template: /eng/emitters/pipelines/templates/jobs/test-job.yml parameters: AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }} @@ -233,6 +246,7 @@ stages: BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }} Os: linux TestMatrix: ${{ parameters.TestMatrix }} + PythonVersion: ${{ parameters.PythonVersion }} # Publish stage # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` diff --git a/eng/emitters/pipelines/templates/steps/build-step.yml b/eng/emitters/pipelines/templates/steps/build-step.yml index 278a2a22e06..f0cfdf5820f 100644 --- a/eng/emitters/pipelines/templates/steps/build-step.yml +++ b/eng/emitters/pipelines/templates/steps/build-step.yml @@ -38,6 +38,10 @@ parameters: - name: LanguageShortName type: string + - name: PythonVersion + type: string + default: "3.12" + steps: - ${{ if notIn(parameters.Publish, 'none', 'internal', 'public') }}: - script: | @@ -61,6 +65,11 @@ steps: inputs: versionSpec: ${{ parameters.NodeVersion }} + - task: UsePythonVersion@0 + displayName: "Use Python ${{ parameters.PythonVersion }}" + inputs: + versionSpec: ${{ parameters.PythonVersion }} + - ${{ parameters.AdditionalInitializeSteps }} - task: PowerShell@2 diff --git a/eng/emitters/pipelines/templates/steps/test-step.yml b/eng/emitters/pipelines/templates/steps/test-step.yml index 020db7d7156..92163853d33 100644 --- a/eng/emitters/pipelines/templates/steps/test-step.yml +++ b/eng/emitters/pipelines/templates/steps/test-step.yml @@ -40,6 +40,10 @@ parameters: type: boolean default: true + - name: PythonVersion + type: string + default: "3.12" + steps: - checkout: self @@ -53,6 +57,11 @@ steps: inputs: versionSpec: ${{ parameters.NodeVersion }} + - task: UsePythonVersion@0 + displayName: "Use Python ${{ parameters.PythonVersion }}" + inputs: + versionSpec: ${{ parameters.PythonVersion }} + - ${{ parameters.AdditionalInitializeSteps }} - task: PowerShell@2 diff --git a/packages/http-client-python/eng/pipeline/publish.yml b/packages/http-client-python/eng/pipeline/publish.yml index da43948f893..e0f1bbb6870 100644 --- a/packages/http-client-python/eng/pipeline/publish.yml +++ b/packages/http-client-python/eng/pipeline/publish.yml @@ -30,3 +30,4 @@ extends: LanguageShortName: "python" CadlRanchName: "@typespec/http-client-python" EnableCadlRanchReport: false + PythonVersion: "3.9" From 6a57e159bfb76792e61c1c40a284295191de3e91 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 11:09:57 +0800 Subject: [PATCH 03/17] Format --- eng/emitters/pipelines/templates/steps/build-step.yml | 2 +- eng/emitters/pipelines/templates/steps/test-step.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/emitters/pipelines/templates/steps/build-step.yml b/eng/emitters/pipelines/templates/steps/build-step.yml index f0cfdf5820f..597d2e09618 100644 --- a/eng/emitters/pipelines/templates/steps/build-step.yml +++ b/eng/emitters/pipelines/templates/steps/build-step.yml @@ -68,7 +68,7 @@ steps: - task: UsePythonVersion@0 displayName: "Use Python ${{ parameters.PythonVersion }}" inputs: - versionSpec: ${{ parameters.PythonVersion }} + versionSpec: ${{ parameters.PythonVersion }} - ${{ parameters.AdditionalInitializeSteps }} diff --git a/eng/emitters/pipelines/templates/steps/test-step.yml b/eng/emitters/pipelines/templates/steps/test-step.yml index 92163853d33..c5b21d216c3 100644 --- a/eng/emitters/pipelines/templates/steps/test-step.yml +++ b/eng/emitters/pipelines/templates/steps/test-step.yml @@ -60,7 +60,7 @@ steps: - task: UsePythonVersion@0 displayName: "Use Python ${{ parameters.PythonVersion }}" inputs: - versionSpec: ${{ parameters.PythonVersion }} + versionSpec: ${{ parameters.PythonVersion }} - ${{ parameters.AdditionalInitializeSteps }} From 6314cc9fbd223d68d19331901106450a5bab62fd Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 11:44:06 +0800 Subject: [PATCH 04/17] fix ci --- packages/http-client-python/eng/pipeline/templates/ci-stages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/http-client-python/eng/pipeline/templates/ci-stages.yml b/packages/http-client-python/eng/pipeline/templates/ci-stages.yml index ba22fd71e1a..bb17aec9cd6 100644 --- a/packages/http-client-python/eng/pipeline/templates/ci-stages.yml +++ b/packages/http-client-python/eng/pipeline/templates/ci-stages.yml @@ -26,3 +26,4 @@ stages: Condition: ${{ parameters.Condition }} DependsOn: ${{ parameters.DependsOn }} LanguageShortName: "python" + PythonVersion: "3.9" From ade031773e024875c4adf0f67e479f8a57fbb802 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 12:55:26 +0800 Subject: [PATCH 05/17] update model_base --- .../pygen/codegen/templates/model_base.py.jinja2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 index 39cf191652b..6c0df38f98c 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 @@ -404,13 +404,13 @@ class _MyMutableMapping(MutableMapping[str, typing.Any]): # pylint: disable=uns return default @typing.overload - def pop(self, key: str) -> typing.Any: ... + def pop(self, key: str) -> typing.Any: ... # pylint: disable=arguments-differ @typing.overload - def pop(self, key: str, default: _T) -> _T: ... + def pop(self, key: str, default: _T) -> _T: ... # pylint: disable=signature-differs @typing.overload - def pop(self, key: str, default: typing.Any) -> typing.Any: ... + def pop(self, key: str, default: typing.Any) -> typing.Any: ... # pylint: disable=signature-differs def pop(self, key: str, default: typing.Any = _UNSET) -> typing.Any: """ @@ -440,7 +440,7 @@ class _MyMutableMapping(MutableMapping[str, typing.Any]): # pylint: disable=uns """ self._data.clear() - def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: + def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: # pylint: disable=arguments-differ """ Updates D from mapping/iterable E and F. :param any args: Either a mapping object or an iterable of key-value pairs. @@ -451,7 +451,7 @@ class _MyMutableMapping(MutableMapping[str, typing.Any]): # pylint: disable=uns def setdefault(self, key: str, default: None = None) -> None: ... @typing.overload - def setdefault(self, key: str, default: typing.Any) -> typing.Any: ... + def setdefault(self, key: str, default: typing.Any) -> typing.Any: ... # pylint: disable=signature-differs def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: """ @@ -677,7 +677,7 @@ class Model(_MyMutableMapping): discriminator_value = data.find(xml_name).text # pyright: ignore else: discriminator_value = data.get(discriminator._rest_name) - mapped_cls = cls.__mapping__.get(discriminator_value, cls) # pyright: ignore + mapped_cls = cls.__mapping__.get(discriminator_value, cls) # pyright: ignore # pylint: disable=no-member return mapped_cls._deserialize(data, exist_discriminators) def as_dict(self, *, exclude_readonly: bool = False) -> typing.Dict[str, typing.Any]: From 42adc6f98127d7c1ccdc1606bc7dcf7062b1c03f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 13:21:54 +0800 Subject: [PATCH 06/17] update for pylint --- .../generator/pygen/codegen/templates/model_base.py.jinja2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 index 6c0df38f98c..c649b565490 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 @@ -344,7 +344,7 @@ def _get_model(module_name: str, model_name: str): _UNSET = object() -class _MyMutableMapping(MutableMapping[str, typing.Any]): # pylint: disable=unsubscriptable-object +class _MyMutableMapping(MutableMapping[str, typing.Any]): def __init__(self, data: typing.Dict[str, typing.Any]) -> None: self._data = data @@ -641,7 +641,7 @@ class Model(_MyMutableMapping): cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items()) cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}") - return super().__new__(cls) # pylint: disable=no-value-for-parameter + return super().__new__(cls) def __init_subclass__(cls, discriminator: typing.Optional[str] = None) -> None: for base in cls.__bases__: From ef2fd1ce2b6bcd8e666fc2a41d2da9eb287e7252 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 14:02:11 +0800 Subject: [PATCH 07/17] update for pylint --- .../generator/pygen/codegen/__init__.py | 1 - .../generator/pygen/codegen/_utils.py | 2 +- .../generator/pygen/codegen/models/imports.py | 14 ++------------ .../pygen/codegen/templates/model_base.py.jinja2 | 6 +----- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/__init__.py b/packages/http-client-python/generator/pygen/codegen/__init__.py index 7443ed7325e..a410b988277 100644 --- a/packages/http-client-python/generator/pygen/codegen/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/__init__.py @@ -150,7 +150,6 @@ def packaging_files_config(self) -> Optional[Dict[str, Any]]: def package_version(self) -> Optional[str]: return str(self.options.get("package-version", "")) - @property def header_text(self) -> Optional[str]: return self.options.get("header-text") diff --git a/packages/http-client-python/generator/pygen/codegen/_utils.py b/packages/http-client-python/generator/pygen/codegen/_utils.py index 77ad6cf8335..21fbcaf9a63 100644 --- a/packages/http-client-python/generator/pygen/codegen/_utils.py +++ b/packages/http-client-python/generator/pygen/codegen/_utils.py @@ -17,7 +17,7 @@ "MIT License\n" "\n" "Permission is hereby granted, free of charge, to any person obtaining a copy\n" - "of this software and associated documentation files (the \"Software\"), to deal\n" + 'of this software and associated documentation files (the "Software"), to deal\n' "in the Software without restriction, including without limitation the rights\n" "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n" "copies of the Software, and to permit persons to whom the Software is\n" diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index ed98052a7bb..5992cd9a949 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -169,22 +169,12 @@ def merge(self, file_import: "FileImport") -> None: def add_mutable_mapping_import(self) -> None: self.add_import("sys", ImportType.STDLIB) - self.add_submodule_import( - "typing", - "MutableMapping", - ImportType.BY_VERSION, - TypingSection.REGULAR, - None, - (((3, 9), "collections.abc", None),), - ) + self.add_submodule_import("collections.abc", "MutableMapping", ImportType.STDLIB) def define_mutable_mapping_type(self) -> None: """Helper function for defining the mutable mapping type""" self.add_mutable_mapping_import() - self.define_mypy_type( - "JSON", - "MutableMapping[str, Any] # pylint: disable=unsubscriptable-object", - ) + self.define_mypy_type("JSON", "MutableMapping[str, Any]") self.add_submodule_import("typing", "Any", ImportType.STDLIB) def to_dict( diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 index c649b565490..928b7c0f95d 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 @@ -24,11 +24,7 @@ from {{ code_model.core_library }}.exceptions import DeserializationError from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta from {{ code_model.core_library }}.{{ "" if code_model.is_azure_flavor else "runtime." }}pipeline import PipelineResponse from {{ code_model.core_library }}.serialization import _Null - -if sys.version_info >= (3, 9): - from collections.abc import MutableMapping -else: - from typing import MutableMapping +from collections.abc import MutableMapping _LOGGER = logging.getLogger(__name__) From fc52c0ec93e034f88f76f1a5d675bd646d8ff0b6 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 14:15:57 +0800 Subject: [PATCH 08/17] fix for eng --- packages/http-client-python/eng/scripts/Build-Packages.ps1 | 5 ++--- packages/http-client-python/eng/scripts/ci/format.ts | 2 +- packages/http-client-python/eng/scripts/ci/utils.ts | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/eng/scripts/Build-Packages.ps1 b/packages/http-client-python/eng/scripts/Build-Packages.ps1 index 1ef566832c3..2044c5a6df8 100644 --- a/packages/http-client-python/eng/scripts/Build-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Build-Packages.ps1 @@ -57,9 +57,8 @@ try { Invoke-LoggedCommand "npm run build" -GroupOutput - # after python 3.13 fix pylint issue, will reopen the check - # Write-Host "run lint check for pygen" - # Invoke-LoggedCommand "npm run lint:py" -GroupOutput + Write-Host "run lint check for pygen" + Invoke-LoggedCommand "npm run lint:py" -GroupOutput # pack the emitter Invoke-LoggedCommand "npm pack" diff --git a/packages/http-client-python/eng/scripts/ci/format.ts b/packages/http-client-python/eng/scripts/ci/format.ts index a87d9d31e1b..39663edcae3 100644 --- a/packages/http-client-python/eng/scripts/ci/format.ts +++ b/packages/http-client-python/eng/scripts/ci/format.ts @@ -1,3 +1,3 @@ import { runCommand } from "./utils.js"; -runCommand("black . --config ./eng/scripts/ci/pyproject.toml", "black"); +runCommand("black", ["generator/", "eng/", "--config", "./eng/scripts/ci/pyproject.toml"]); diff --git a/packages/http-client-python/eng/scripts/ci/utils.ts b/packages/http-client-python/eng/scripts/ci/utils.ts index 4d81b205b33..498947c1c0f 100644 --- a/packages/http-client-python/eng/scripts/ci/utils.ts +++ b/packages/http-client-python/eng/scripts/ci/utils.ts @@ -20,7 +20,7 @@ const argv = parseArgs({ export async function executeCommand(command: string, args: string[]) { const execFileAsync = promisify(execFile); try { - await execFileAsync(command, args); + await execFileAsync(command, args, { shell: true }); console.log(chalk.green(`${command} passed`)); } catch (err) { console.error(chalk.red(`Error executing ${command}: ${err}`)); From 8bccd79fee0602a80d21e461fd751668f60553db Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 14:20:30 +0800 Subject: [PATCH 09/17] add format check --- packages/http-client-python/eng/scripts/Build-Packages.ps1 | 7 +++++-- .../asynctests/test_authentication_async.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/eng/scripts/Build-Packages.ps1 b/packages/http-client-python/eng/scripts/Build-Packages.ps1 index 2044c5a6df8..0d5f539533c 100644 --- a/packages/http-client-python/eng/scripts/Build-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Build-Packages.ps1 @@ -47,14 +47,17 @@ New-Item -ItemType Directory -Force -Path "$outputPath/packages" | Out-Null Write-Host "Getting existing version" $emitterVersion = node -p -e "require('$packageRoot/package.json').version" -# build the generator jar +# build the generator Push-Location "$packageRoot/generator" -# build and pack the emitter with the generator jar +# build and pack the emitter with the generator Push-Location "$packageRoot" try { Write-Host "Working in $PWD" + Write-Host "run format" + Invoke-LoggedCommand "npm run format" -GroupOutput + Invoke-LoggedCommand "npm run build" -GroupOutput Write-Host "run lint check for pygen" diff --git a/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py b/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py index bb92d42bc4a..9442bf24aa3 100644 --- a/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py +++ b/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py @@ -121,5 +121,5 @@ async def test_http_custom_invalid(http_custom_client, core_library): client = http_custom_client(key="invalid-key") with pytest.raises(core_library.exceptions.HttpResponseError) as ex: await client.invalid() - assert ex.value.status_code == 403 + assert ex.value.status_code == 403 assert ex.value.reason == "Forbidden" From d46e660ce37cd467cfba6a43b5689faa148aa766 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 15:08:37 +0800 Subject: [PATCH 10/17] debug --- packages/http-client-python/emitter/src/emitter.ts | 2 +- .../http-client-python/eng/scripts/Build-Packages.ps1 | 10 +++++++++- packages/http-client-python/package.json | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 4ba2d9b71f7..b117d906f19 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -137,7 +137,7 @@ async function onEmitMain(context: EmitContext) { const sdkContext = await createPythonSdkContext(context); const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const outputDir = context.emitterOutputDir; - addDefaultOptions(sdkContext); + addDefaultOptions(sdkContext); const yamlMap = emitCodeModel(sdkContext); if (yamlMap.clients.length === 0) { reportDiagnostic(program, { diff --git a/packages/http-client-python/eng/scripts/Build-Packages.ps1 b/packages/http-client-python/eng/scripts/Build-Packages.ps1 index 0d5f539533c..4eec6c2c8b5 100644 --- a/packages/http-client-python/eng/scripts/Build-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Build-Packages.ps1 @@ -56,7 +56,15 @@ try { Write-Host "Working in $PWD" Write-Host "run format" - Invoke-LoggedCommand "npm run format" -GroupOutput + Invoke-LoggedCommand "npm run format:py" -GroupOutput + + try { + Write-Host "check git diff after run format" + & "eng/scripts/Check-GitChanges.ps1" + } + catch { + Write-Error 'There is diff after running format. Please run: npm run format' + } Invoke-LoggedCommand "npm run build" -GroupOutput diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 53404e45a7a..240af48ba29 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -35,6 +35,7 @@ "lint": "eslint emitter/ --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator/pygen", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", + "format:py": "tsx ./eng/scripts/ci/format.ts", "install": "tsx ./eng/scripts/setup/install.ts", "prepare": "tsx ./eng/scripts/setup/prepare.ts", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", From 354c3494ef5117ab7be722142bc5663d4331d7c3 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 15:27:44 +0800 Subject: [PATCH 11/17] update --- packages/http-client-python/emitter/src/emitter.ts | 2 +- .../http-client-python/eng/scripts/Build-Packages.ps1 | 11 ----------- .../asynctests/test_authentication_async.py | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index b117d906f19..4ba2d9b71f7 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -137,7 +137,7 @@ async function onEmitMain(context: EmitContext) { const sdkContext = await createPythonSdkContext(context); const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const outputDir = context.emitterOutputDir; - addDefaultOptions(sdkContext); + addDefaultOptions(sdkContext); const yamlMap = emitCodeModel(sdkContext); if (yamlMap.clients.length === 0) { reportDiagnostic(program, { diff --git a/packages/http-client-python/eng/scripts/Build-Packages.ps1 b/packages/http-client-python/eng/scripts/Build-Packages.ps1 index 4eec6c2c8b5..b35ee08ed54 100644 --- a/packages/http-client-python/eng/scripts/Build-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Build-Packages.ps1 @@ -55,17 +55,6 @@ Push-Location "$packageRoot" try { Write-Host "Working in $PWD" - Write-Host "run format" - Invoke-LoggedCommand "npm run format:py" -GroupOutput - - try { - Write-Host "check git diff after run format" - & "eng/scripts/Check-GitChanges.ps1" - } - catch { - Write-Error 'There is diff after running format. Please run: npm run format' - } - Invoke-LoggedCommand "npm run build" -GroupOutput Write-Host "run lint check for pygen" diff --git a/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py b/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py index 9442bf24aa3..bb92d42bc4a 100644 --- a/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py +++ b/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_authentication_async.py @@ -121,5 +121,5 @@ async def test_http_custom_invalid(http_custom_client, core_library): client = http_custom_client(key="invalid-key") with pytest.raises(core_library.exceptions.HttpResponseError) as ex: await client.invalid() - assert ex.value.status_code == 403 + assert ex.value.status_code == 403 assert ex.value.reason == "Forbidden" From ddf30da49d6f3947ead3042b9dc84bd00fd1a907 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 2 Apr 2025 15:29:03 +0800 Subject: [PATCH 12/17] udpate --- packages/http-client-python/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 240af48ba29..53404e45a7a 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -35,7 +35,6 @@ "lint": "eslint emitter/ --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator/pygen", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", - "format:py": "tsx ./eng/scripts/ci/format.ts", "install": "tsx ./eng/scripts/setup/install.ts", "prepare": "tsx ./eng/scripts/setup/prepare.ts", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", From 8c3c633ca212ea227d8cbdba3fb94917106541d0 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 7 Apr 2025 15:23:20 +0800 Subject: [PATCH 13/17] enable pylint in windows --- packages/http-client-python/eng/scripts/ci/run_pylint.py | 4 ---- .../generator/pygen/codegen/serializers/builder_serializer.py | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/run_pylint.py b/packages/http-client-python/eng/scripts/ci/run_pylint.py index 370cf605129..b95299aacff 100644 --- a/packages/http-client-python/eng/scripts/ci/run_pylint.py +++ b/packages/http-client-python/eng/scripts/ci/run_pylint.py @@ -49,8 +49,4 @@ def _single_dir_pylint(mod): if __name__ == "__main__": - if os.name == "nt": - # Before https://github.com/microsoft/typespec/issues/4759 fixed, skip running Pylint for now on Windows - logging.info("Skip running Pylint on Windows for now") - sys.exit(0) run_check("pylint", _single_dir_pylint, "Pylint") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 75cdee833fd..07ba2cc581d 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -1373,9 +1373,7 @@ def _extract_data_callback(self, builder: PagingOperationType) -> List[str]: # return retval def _get_next_callback(self, builder: PagingOperationType) -> List[str]: - retval = [ - f"{'async ' if self.async_mode else ''}def get_next({builder.next_variable_name}=None):" # pylint: disable=line-too-long - ] + retval = [f"{'async ' if self.async_mode else ''}def get_next({builder.next_variable_name}=None):"] retval.append(f" _request = prepare_request({builder.next_variable_name})") retval.append("") retval.extend([f" {l}" for l in self.make_pipeline_call(builder)]) From ff2834b28661cbc1408d971a15d17b2aa21999b1 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 7 Apr 2025 16:24:35 +0800 Subject: [PATCH 14/17] revert --- packages/http-client-python/eng/scripts/ci/run_pylint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/http-client-python/eng/scripts/ci/run_pylint.py b/packages/http-client-python/eng/scripts/ci/run_pylint.py index b95299aacff..370cf605129 100644 --- a/packages/http-client-python/eng/scripts/ci/run_pylint.py +++ b/packages/http-client-python/eng/scripts/ci/run_pylint.py @@ -49,4 +49,8 @@ def _single_dir_pylint(mod): if __name__ == "__main__": + if os.name == "nt": + # Before https://github.com/microsoft/typespec/issues/4759 fixed, skip running Pylint for now on Windows + logging.info("Skip running Pylint on Windows for now") + sys.exit(0) run_check("pylint", _single_dir_pylint, "Pylint") From 3aa513b5f11ec469a2ce8951613b3a9b832cbb79 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 7 Apr 2025 16:25:24 +0800 Subject: [PATCH 15/17] add log when error occurs --- packages/http-client-python/eng/scripts/ci/utils.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/utils.ts b/packages/http-client-python/eng/scripts/ci/utils.ts index 498947c1c0f..118b09c60da 100644 --- a/packages/http-client-python/eng/scripts/ci/utils.ts +++ b/packages/http-client-python/eng/scripts/ci/utils.ts @@ -22,8 +22,10 @@ export async function executeCommand(command: string, args: string[]) { try { await execFileAsync(command, args, { shell: true }); console.log(chalk.green(`${command} passed`)); - } catch (err) { - console.error(chalk.red(`Error executing ${command}: ${err}`)); + } catch (err: any) { + console.error(chalk.red(`Error executing ${command}`)); + if (err.stdout) console.error(chalk.yellow("STDOUT:"), err.stdout); + if (err.stderr) console.error(chalk.yellow("STDERR:"), err.stderr); process.exit(1); } } From a810ecca279ea8488e8b0d411e5ab3976b10843e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 7 Apr 2025 16:45:39 +0800 Subject: [PATCH 16/17] fix CI --- packages/http-client-python/eng/scripts/Build-Packages.ps1 | 7 +++++-- packages/http-client-python/eng/scripts/ci/pylintrc | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/eng/scripts/Build-Packages.ps1 b/packages/http-client-python/eng/scripts/Build-Packages.ps1 index b35ee08ed54..45036845b6f 100644 --- a/packages/http-client-python/eng/scripts/Build-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Build-Packages.ps1 @@ -57,8 +57,11 @@ try { Invoke-LoggedCommand "npm run build" -GroupOutput - Write-Host "run lint check for pygen" - Invoke-LoggedCommand "npm run lint:py" -GroupOutput + # Only run lint:py on Linux OS + if ($IsLinux) { + Write-Host "run lint check for pygen" + Invoke-LoggedCommand "npm run lint:py" -GroupOutput + } # pack the emitter Invoke-LoggedCommand "npm pack" diff --git a/packages/http-client-python/eng/scripts/ci/pylintrc b/packages/http-client-python/eng/scripts/ci/pylintrc index 2ae56b9e246..ad23a561c6c 100644 --- a/packages/http-client-python/eng/scripts/ci/pylintrc +++ b/packages/http-client-python/eng/scripts/ci/pylintrc @@ -1,5 +1,5 @@ [MASTER] -py-version=3.8 +py-version=3.9 ignore-patterns=test_*,conftest,setup reports=no From 0b7c690374896bd643da089a17a2b38dcd195910 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 8 Apr 2025 10:35:23 +0800 Subject: [PATCH 17/17] fix pylint --- .../generator/pygen/codegen/models/imports.py | 1 - .../generator/pygen/codegen/templates/model_base.py.jinja2 | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index 5992cd9a949..ed434df81de 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -168,7 +168,6 @@ def merge(self, file_import: "FileImport") -> None: self.type_definitions.update(file_import.type_definitions) def add_mutable_mapping_import(self) -> None: - self.add_import("sys", ImportType.STDLIB) self.add_submodule_import("collections.abc", "MutableMapping", ImportType.STDLIB) def define_mutable_mapping_type(self) -> None: diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 index 928b7c0f95d..1d34a2df875 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 @@ -18,13 +18,13 @@ import email.utils from datetime import datetime, date, time, timedelta, timezone from json import JSONEncoder import xml.etree.ElementTree as ET +from collections.abc import MutableMapping from typing_extensions import Self import isodate from {{ code_model.core_library }}.exceptions import DeserializationError from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta from {{ code_model.core_library }}.{{ "" if code_model.is_azure_flavor else "runtime." }}pipeline import PipelineResponse from {{ code_model.core_library }}.serialization import _Null -from collections.abc import MutableMapping _LOGGER = logging.getLogger(__name__)