From f0be092860463395e29cbe0ebaf963305c8e18f9 Mon Sep 17 00:00:00 2001 From: pierrejeambrun Date: Tue, 27 Aug 2024 12:04:18 +0200 Subject: [PATCH 1/5] update requirements --- hatch_build.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index fd4dba6cbc3b3..08f47731f2c47 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -116,9 +116,6 @@ "bcrypt>=2.0.0", "flask-bcrypt>=0.7.1", ], - "pydantic": [ - "pydantic>=2.3.0", - ], "rabbitmq": [ "amqp", ], @@ -431,6 +428,7 @@ "cryptography>=41.0.0", "deprecated>=1.2.13", "dill>=0.2.2", + "fastapi[standard]>=0.112.2", "flask-caching>=2.0.0", # Flask-Session 0.6 add new arguments into the SqlAlchemySessionInterface constructor as well as # all parameters now are mandatory which make AirflowDatabaseSessionInterface incopatible with this version. @@ -468,6 +466,7 @@ 'pendulum>=3.0.0,<4.0;python_version>="3.12"', "pluggy>=1.5.0", "psutil>=5.8.0", + "pydantic>=2.3.0", "pygments>=2.0.1", "pyjwt>=2.0.0", "python-daemon>=3.0.0", From e8e3963f0c56456e29e5199d81110f43dde8d4d9 Mon Sep 17 00:00:00 2001 From: pierrejeambrun Date: Thu, 29 Aug 2024 12:03:24 +0200 Subject: [PATCH 2/5] Pydantic dependancy is not optional anymore --- .github/workflows/run-unit-tests.yml | 6 - .github/workflows/special-tests.yml | 44 ------- Dockerfile.ci | 41 ------- airflow/operators/python.py | 5 +- airflow/serialization/pydantic/dag.py | 10 +- airflow/serialization/pydantic/dag_run.py | 6 +- airflow/serialization/pydantic/dataset.py | 2 +- airflow/serialization/pydantic/job.py | 3 +- .../serialization/pydantic/taskinstance.py | 18 ++- airflow/serialization/pydantic/tasklog.py | 2 +- airflow/serialization/pydantic/trigger.py | 3 +- airflow/serialization/serialized_objects.py | 7 +- airflow/utils/pydantic.py | 61 --------- dev/breeze/doc/images/output_shell.svg | 48 ++++---- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_testing_db-tests.svg | 116 +++++++++--------- .../doc/images/output_testing_db-tests.txt | 2 +- .../images/output_testing_non-db-tests.svg | 116 +++++++++--------- .../images/output_testing_non-db-tests.txt | 2 +- .../doc/images/output_testing_tests.svg | 116 +++++++++--------- .../doc/images/output_testing_tests.txt | 2 +- .../airflow_breeze/commands/common_options.py | 9 -- .../commands/developer_commands.py | 4 - .../commands/developer_commands_config.py | 1 - .../commands/testing_commands.py | 11 -- .../commands/testing_commands_config.py | 3 - .../src/airflow_breeze/global_constants.py | 1 - .../src/airflow_breeze/params/shell_params.py | 3 - scripts/ci/docker-compose/devcontainer.env | 1 - scripts/docker/entrypoint_ci.sh | 42 ------- tests/always/test_example_dags.py | 6 - tests/operators/test_python.py | 18 +-- tests/serialization/test_serde.py | 2 +- .../serialization/test_serialized_objects.py | 2 +- 34 files changed, 227 insertions(+), 488 deletions(-) delete mode 100644 airflow/utils/pydantic.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 7828e50ed7e95..88ac30c2098f3 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -89,11 +89,6 @@ on: # yamllint disable-line rule:truthy required: false default: "false" type: string - pydantic: - description: "The version of pydantic to use" - required: false - default: "v2" - type: string downgrade-pendulum: description: "Whether to downgrade pendulum or not (true/false)" required: false @@ -148,7 +143,6 @@ jobs: JOB_ID: "${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}" MOUNT_SOURCES: "skip" PARALLEL_TEST_TYPES: "${{ inputs.parallel-test-types-list-as-string }}" - PYDANTIC: "${{ inputs.pydantic }}" PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" UPGRADE_BOTO: "${{ inputs.upgrade-boto }}" AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}" diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 000b5aa3d958b..c6b4275de4311 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -105,50 +105,6 @@ jobs: run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} - tests-pydantic-v1: - name: "Pydantic v1 test" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} - pydantic: "v1" - test-name: "Pydantic-V1-Postgres" - test-scope: "All" - backend: "postgres" - image-tag: ${{ inputs.image-tag }} - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - - tests-pydantic-none: - name: "Pydantic removed test" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} - pydantic: "none" - test-name: "Pydantic-Removed-Postgres" - test-scope: "All" - backend: "postgres" - image-tag: ${{ inputs.image-tag }} - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - tests-pendulum-2: name: "Pendulum2 test" uses: ./.github/workflows/run-unit-tests.yml diff --git a/Dockerfile.ci b/Dockerfile.ci index 25151df8e983e..67b6bf861e11f 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1032,46 +1032,6 @@ function check_boto_upgrade() { pip check } -function check_pydantic() { - if [[ ${PYDANTIC=} == "none" ]]; then - echo - echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . - echo - echo "${COLOR_YELLOW}Remove pydantic and 3rd party libraries that depend on it${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pydantic aws-sam-translator openai \ - pyiceberg qdrant-client cfn-lint weaviate-client google-cloud-aiplatform - pip check - elif [[ ${PYDANTIC=} == "v1" ]]; then - echo - echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . - echo - echo "${COLOR_YELLOW}Uninstalling dependencies which are not compatible with Pydantic 1${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pyiceberg weaviate-client - echo - echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}" - echo - # Pydantic 1.10.17/1.10.15 conflicts with aws-sam-translator so we need to exclude it - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0,!=1.10.17,!=1.10.15" - pip check - else - echo - echo "${COLOR_BLUE}Leaving default pydantic v2${COLOR_RESET}" - echo - fi -} - - function check_downgrade_sqlalchemy() { if [[ ${DOWNGRADE_SQLALCHEMY=} != "true" ]]; then return @@ -1181,7 +1141,6 @@ function check_force_lowest_dependencies() { determine_airflow_to_use environment_initialization check_boto_upgrade -check_pydantic check_downgrade_sqlalchemy check_downgrade_pendulum check_force_lowest_dependencies diff --git a/airflow/operators/python.py b/airflow/operators/python.py index 47e0cf3193352..25c0b8ca68632 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -56,7 +56,6 @@ from airflow.utils.file import get_unique_dag_module_name from airflow.utils.operator_helpers import ExecutionCallableRunner, KeywordParameters from airflow.utils.process_utils import execute_in_subprocess -from airflow.utils.pydantic import is_pydantic_2_installed from airflow.utils.python_virtualenv import prepare_virtualenv, write_python_script from airflow.utils.session import create_session @@ -549,8 +548,8 @@ def _execute_python_callable_in_subprocess(self, python_path: Path): self._write_args(input_path) self._write_string_args(string_args_path) - if self.use_airflow_context and (not is_pydantic_2_installed() or not _ENABLE_AIP_44): - error_msg = "`get_current_context()` needs to be used with Pydantic 2 and AIP-44 enabled." + if self.use_airflow_context and not _ENABLE_AIP_44: + error_msg = "`get_current_context()` needs to be used with AIP-44 enabled." raise AirflowException(error_msg) jinja_context = { diff --git a/airflow/serialization/pydantic/dag.py b/airflow/serialization/pydantic/dag.py index b916a0e580efd..f7cb90797d5c8 100644 --- a/airflow/serialization/pydantic/dag.py +++ b/airflow/serialization/pydantic/dag.py @@ -20,17 +20,17 @@ from datetime import datetime from typing import Any, List, Optional -from typing_extensions import Annotated - -from airflow import DAG, settings -from airflow.configuration import conf as airflow_conf -from airflow.utils.pydantic import ( +from pydantic import ( BaseModel as BaseModelPydantic, ConfigDict, PlainSerializer, PlainValidator, ValidationInfo, ) +from typing_extensions import Annotated + +from airflow import DAG, settings +from airflow.configuration import conf as airflow_conf def serialize_operator(x: DAG) -> dict: diff --git a/airflow/serialization/pydantic/dag_run.py b/airflow/serialization/pydantic/dag_run.py index 63b20ff487b6b..31b7171a3ace7 100644 --- a/airflow/serialization/pydantic/dag_run.py +++ b/airflow/serialization/pydantic/dag_run.py @@ -19,10 +19,11 @@ from datetime import datetime from typing import TYPE_CHECKING, Iterable, List, Optional +from pydantic import BaseModel as BaseModelPydantic, ConfigDict + from airflow.models.dagrun import DagRun from airflow.serialization.pydantic.dag import PydanticDag from airflow.serialization.pydantic.dataset import DatasetEventPydantic -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict, is_pydantic_2_installed if TYPE_CHECKING: from sqlalchemy.orm import Session @@ -111,5 +112,4 @@ def get_log_template(self, session: Session): return DagRun._get_log_template(log_template_id=self.log_template_id) -if is_pydantic_2_installed(): - DagRunPydantic.model_rebuild() +DagRunPydantic.model_rebuild() diff --git a/airflow/serialization/pydantic/dataset.py b/airflow/serialization/pydantic/dataset.py index 44822c546d957..0c233a3fd67c6 100644 --- a/airflow/serialization/pydantic/dataset.py +++ b/airflow/serialization/pydantic/dataset.py @@ -17,7 +17,7 @@ from datetime import datetime from typing import List, Optional -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict +from pydantic import BaseModel as BaseModelPydantic, ConfigDict class DagScheduleDatasetReferencePydantic(BaseModelPydantic): diff --git a/airflow/serialization/pydantic/job.py b/airflow/serialization/pydantic/job.py index bea8e9a3af73f..ab2dafa1787b6 100644 --- a/airflow/serialization/pydantic/job.py +++ b/airflow/serialization/pydantic/job.py @@ -18,9 +18,10 @@ from functools import cached_property from typing import TYPE_CHECKING, Optional +from pydantic import BaseModel as BaseModelPydantic, ConfigDict + from airflow.executors.executor_loader import ExecutorLoader from airflow.jobs.base_job_runner import BaseJobRunner -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict def check_runner_initialized(job_runner: Optional[BaseJobRunner], job_type: str) -> BaseJobRunner: diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py index e89d21cd98da6..0dcb7880eba7d 100644 --- a/airflow/serialization/pydantic/taskinstance.py +++ b/airflow/serialization/pydantic/taskinstance.py @@ -19,6 +19,12 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Iterable, Optional +from pydantic import ( + BaseModel as BaseModelPydantic, + ConfigDict, + PlainSerializer, + PlainValidator, +) from typing_extensions import Annotated from airflow.exceptions import AirflowRescheduleException, TaskDeferred @@ -36,22 +42,15 @@ from airflow.serialization.pydantic.dag_run import DagRunPydantic from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.net import get_hostname -from airflow.utils.pydantic import ( - BaseModel as BaseModelPydantic, - ConfigDict, - PlainSerializer, - PlainValidator, - is_pydantic_2_installed, -) from airflow.utils.xcom import XCOM_RETURN_KEY if TYPE_CHECKING: import pendulum + from pydantic import ValidationInfo from sqlalchemy.orm import Session from airflow.models.dagrun import DagRun from airflow.utils.context import Context - from airflow.utils.pydantic import ValidationInfo from airflow.utils.state import DagRunState @@ -552,5 +551,4 @@ def get_relevant_upstream_map_indexes( ) -if is_pydantic_2_installed(): - TaskInstancePydantic.model_rebuild() +TaskInstancePydantic.model_rebuild() diff --git a/airflow/serialization/pydantic/tasklog.py b/airflow/serialization/pydantic/tasklog.py index 3fbbf872b8f8a..a23204400c1f9 100644 --- a/airflow/serialization/pydantic/tasklog.py +++ b/airflow/serialization/pydantic/tasklog.py @@ -16,7 +16,7 @@ # under the License. from datetime import datetime -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict +from pydantic import BaseModel as BaseModelPydantic, ConfigDict class LogTemplatePydantic(BaseModelPydantic): diff --git a/airflow/serialization/pydantic/trigger.py b/airflow/serialization/pydantic/trigger.py index bf2ce2340cfa8..19a8f6b70c007 100644 --- a/airflow/serialization/pydantic/trigger.py +++ b/airflow/serialization/pydantic/trigger.py @@ -18,8 +18,9 @@ import datetime from typing import Any, Dict, Optional +from pydantic import BaseModel as BaseModelPydantic, ConfigDict + from airflow.utils import timezone -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict class TriggerPydantic(BaseModelPydantic): diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 71bc64c61aea6..2bbeb8aae917c 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -96,6 +96,8 @@ if TYPE_CHECKING: from inspect import Parameter + from pydantic import BaseModel + from airflow.models.baseoperatorlink import BaseOperatorLink from airflow.models.expandinput import ExpandInput from airflow.models.operator import Operator @@ -103,7 +105,6 @@ from airflow.serialization.json_schema import Validator from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.timetables.base import Timetable - from airflow.utils.pydantic import BaseModel HAS_KUBERNETES: bool try: @@ -364,8 +365,8 @@ def encode_start_trigger_args(var: StartTriggerArgs) -> dict[str, Any]: :meta private: """ - serialize_kwargs = ( - lambda key: BaseSerialization.serialize(getattr(var, key)) if getattr(var, key) is not None else None + serialize_kwargs = lambda key: ( + BaseSerialization.serialize(getattr(var, key)) if getattr(var, key) is not None else None ) return { "__type": "START_TRIGGER_ARGS", diff --git a/airflow/utils/pydantic.py b/airflow/utils/pydantic.py deleted file mode 100644 index 85fde06195362..0000000000000 --- a/airflow/utils/pydantic.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This is an util module that makes Pydantic use optional. While we are using Pydantic in the airflow core -# codebase, we don't want to make it a hard dependency for all the users of the core codebase, because -# it is only used in the serialization and deserialization of the models for Internal API and for nothing -# else, and since Pydantic is a very popular library, we don't want to force the users of the core codebase -# to install specific Pydantic version - especially that a lot of libraries out there still depend on -# Pydantic 1 and our internal API uses Pydantic 2+ - -from __future__ import annotations - -from importlib import metadata - -from packaging import version - - -def is_pydantic_2_installed() -> bool: - try: - return version.parse(metadata.version("pydantic")).major == 2 - except ImportError: - return False - - -if is_pydantic_2_installed(): - from pydantic import BaseModel, ConfigDict, PlainSerializer, PlainValidator, ValidationInfo -else: - - class BaseModel: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class ConfigDict: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class PlainSerializer: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class PlainValidator: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class ValidationInfo: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 54814bb1078a5..53fb9b978f4d7 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -1,4 +1,4 @@ - +