diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 7828e50ed7e95..88ac30c2098f3 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -89,11 +89,6 @@ on: # yamllint disable-line rule:truthy required: false default: "false" type: string - pydantic: - description: "The version of pydantic to use" - required: false - default: "v2" - type: string downgrade-pendulum: description: "Whether to downgrade pendulum or not (true/false)" required: false @@ -148,7 +143,6 @@ jobs: JOB_ID: "${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}" MOUNT_SOURCES: "skip" PARALLEL_TEST_TYPES: "${{ inputs.parallel-test-types-list-as-string }}" - PYDANTIC: "${{ inputs.pydantic }}" PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" UPGRADE_BOTO: "${{ inputs.upgrade-boto }}" AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}" diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 000b5aa3d958b..c6b4275de4311 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -105,50 +105,6 @@ jobs: run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} - tests-pydantic-v1: - name: "Pydantic v1 test" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} - pydantic: "v1" - test-name: "Pydantic-V1-Postgres" - test-scope: "All" - backend: "postgres" - image-tag: ${{ inputs.image-tag }} - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - - tests-pydantic-none: - name: "Pydantic removed test" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} - pydantic: "none" - test-name: "Pydantic-Removed-Postgres" - test-scope: "All" - backend: "postgres" - image-tag: ${{ inputs.image-tag }} - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - tests-pendulum-2: name: "Pendulum2 test" uses: ./.github/workflows/run-unit-tests.yml diff --git a/Dockerfile.ci b/Dockerfile.ci index 25151df8e983e..67b6bf861e11f 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1032,46 +1032,6 @@ function check_boto_upgrade() { pip check } -function check_pydantic() { - if [[ ${PYDANTIC=} == "none" ]]; then - echo - echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . - echo - echo "${COLOR_YELLOW}Remove pydantic and 3rd party libraries that depend on it${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pydantic aws-sam-translator openai \ - pyiceberg qdrant-client cfn-lint weaviate-client google-cloud-aiplatform - pip check - elif [[ ${PYDANTIC=} == "v1" ]]; then - echo - echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . - echo - echo "${COLOR_YELLOW}Uninstalling dependencies which are not compatible with Pydantic 1${COLOR_RESET}" - echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pyiceberg weaviate-client - echo - echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}" - echo - # Pydantic 1.10.17/1.10.15 conflicts with aws-sam-translator so we need to exclude it - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0,!=1.10.17,!=1.10.15" - pip check - else - echo - echo "${COLOR_BLUE}Leaving default pydantic v2${COLOR_RESET}" - echo - fi -} - - function check_downgrade_sqlalchemy() { if [[ ${DOWNGRADE_SQLALCHEMY=} != "true" ]]; then return @@ -1181,7 +1141,6 @@ function check_force_lowest_dependencies() { determine_airflow_to_use environment_initialization check_boto_upgrade -check_pydantic check_downgrade_sqlalchemy check_downgrade_pendulum check_force_lowest_dependencies diff --git a/INSTALL b/INSTALL index a875fd372811a..72ebca79e7040 100644 --- a/INSTALL +++ b/INSTALL @@ -257,8 +257,8 @@ Those extras are available as regular core airflow extras - they install optiona # START CORE EXTRAS HERE aiobotocore, apache-atlas, apache-webhdfs, async, cgroups, cloudpickle, github-enterprise, google- -auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, pydantic, rabbitmq, s3fs, saml, -sentry, statsd, uv, virtualenv +auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, rabbitmq, s3fs, saml, sentry, +statsd, uv, virtualenv # END CORE EXTRAS HERE diff --git a/airflow/operators/python.py b/airflow/operators/python.py index 47e0cf3193352..25c0b8ca68632 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -56,7 +56,6 @@ from airflow.utils.file import get_unique_dag_module_name from airflow.utils.operator_helpers import ExecutionCallableRunner, KeywordParameters from airflow.utils.process_utils import execute_in_subprocess -from airflow.utils.pydantic import is_pydantic_2_installed from airflow.utils.python_virtualenv import prepare_virtualenv, write_python_script from airflow.utils.session import create_session @@ -549,8 +548,8 @@ def _execute_python_callable_in_subprocess(self, python_path: Path): self._write_args(input_path) self._write_string_args(string_args_path) - if self.use_airflow_context and (not is_pydantic_2_installed() or not _ENABLE_AIP_44): - error_msg = "`get_current_context()` needs to be used with Pydantic 2 and AIP-44 enabled." + if self.use_airflow_context and not _ENABLE_AIP_44: + error_msg = "`get_current_context()` needs to be used with AIP-44 enabled." raise AirflowException(error_msg) jinja_context = { diff --git a/airflow/serialization/pydantic/dag.py b/airflow/serialization/pydantic/dag.py index b916a0e580efd..f7cb90797d5c8 100644 --- a/airflow/serialization/pydantic/dag.py +++ b/airflow/serialization/pydantic/dag.py @@ -20,17 +20,17 @@ from datetime import datetime from typing import Any, List, Optional -from typing_extensions import Annotated - -from airflow import DAG, settings -from airflow.configuration import conf as airflow_conf -from airflow.utils.pydantic import ( +from pydantic import ( BaseModel as BaseModelPydantic, ConfigDict, PlainSerializer, PlainValidator, ValidationInfo, ) +from typing_extensions import Annotated + +from airflow import DAG, settings +from airflow.configuration import conf as airflow_conf def serialize_operator(x: DAG) -> dict: diff --git a/airflow/serialization/pydantic/dag_run.py b/airflow/serialization/pydantic/dag_run.py index 63b20ff487b6b..31b7171a3ace7 100644 --- a/airflow/serialization/pydantic/dag_run.py +++ b/airflow/serialization/pydantic/dag_run.py @@ -19,10 +19,11 @@ from datetime import datetime from typing import TYPE_CHECKING, Iterable, List, Optional +from pydantic import BaseModel as BaseModelPydantic, ConfigDict + from airflow.models.dagrun import DagRun from airflow.serialization.pydantic.dag import PydanticDag from airflow.serialization.pydantic.dataset import DatasetEventPydantic -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict, is_pydantic_2_installed if TYPE_CHECKING: from sqlalchemy.orm import Session @@ -111,5 +112,4 @@ def get_log_template(self, session: Session): return DagRun._get_log_template(log_template_id=self.log_template_id) -if is_pydantic_2_installed(): - DagRunPydantic.model_rebuild() +DagRunPydantic.model_rebuild() diff --git a/airflow/serialization/pydantic/dataset.py b/airflow/serialization/pydantic/dataset.py index 44822c546d957..0c233a3fd67c6 100644 --- a/airflow/serialization/pydantic/dataset.py +++ b/airflow/serialization/pydantic/dataset.py @@ -17,7 +17,7 @@ from datetime import datetime from typing import List, Optional -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict +from pydantic import BaseModel as BaseModelPydantic, ConfigDict class DagScheduleDatasetReferencePydantic(BaseModelPydantic): diff --git a/airflow/serialization/pydantic/job.py b/airflow/serialization/pydantic/job.py index bea8e9a3af73f..ab2dafa1787b6 100644 --- a/airflow/serialization/pydantic/job.py +++ b/airflow/serialization/pydantic/job.py @@ -18,9 +18,10 @@ from functools import cached_property from typing import TYPE_CHECKING, Optional +from pydantic import BaseModel as BaseModelPydantic, ConfigDict + from airflow.executors.executor_loader import ExecutorLoader from airflow.jobs.base_job_runner import BaseJobRunner -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict def check_runner_initialized(job_runner: Optional[BaseJobRunner], job_type: str) -> BaseJobRunner: diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py index e89d21cd98da6..0dcb7880eba7d 100644 --- a/airflow/serialization/pydantic/taskinstance.py +++ b/airflow/serialization/pydantic/taskinstance.py @@ -19,6 +19,12 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Iterable, Optional +from pydantic import ( + BaseModel as BaseModelPydantic, + ConfigDict, + PlainSerializer, + PlainValidator, +) from typing_extensions import Annotated from airflow.exceptions import AirflowRescheduleException, TaskDeferred @@ -36,22 +42,15 @@ from airflow.serialization.pydantic.dag_run import DagRunPydantic from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.net import get_hostname -from airflow.utils.pydantic import ( - BaseModel as BaseModelPydantic, - ConfigDict, - PlainSerializer, - PlainValidator, - is_pydantic_2_installed, -) from airflow.utils.xcom import XCOM_RETURN_KEY if TYPE_CHECKING: import pendulum + from pydantic import ValidationInfo from sqlalchemy.orm import Session from airflow.models.dagrun import DagRun from airflow.utils.context import Context - from airflow.utils.pydantic import ValidationInfo from airflow.utils.state import DagRunState @@ -552,5 +551,4 @@ def get_relevant_upstream_map_indexes( ) -if is_pydantic_2_installed(): - TaskInstancePydantic.model_rebuild() +TaskInstancePydantic.model_rebuild() diff --git a/airflow/serialization/pydantic/tasklog.py b/airflow/serialization/pydantic/tasklog.py index 3fbbf872b8f8a..a23204400c1f9 100644 --- a/airflow/serialization/pydantic/tasklog.py +++ b/airflow/serialization/pydantic/tasklog.py @@ -16,7 +16,7 @@ # under the License. from datetime import datetime -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict +from pydantic import BaseModel as BaseModelPydantic, ConfigDict class LogTemplatePydantic(BaseModelPydantic): diff --git a/airflow/serialization/pydantic/trigger.py b/airflow/serialization/pydantic/trigger.py index bf2ce2340cfa8..19a8f6b70c007 100644 --- a/airflow/serialization/pydantic/trigger.py +++ b/airflow/serialization/pydantic/trigger.py @@ -18,8 +18,9 @@ import datetime from typing import Any, Dict, Optional +from pydantic import BaseModel as BaseModelPydantic, ConfigDict + from airflow.utils import timezone -from airflow.utils.pydantic import BaseModel as BaseModelPydantic, ConfigDict class TriggerPydantic(BaseModelPydantic): diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 71bc64c61aea6..2bbeb8aae917c 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -96,6 +96,8 @@ if TYPE_CHECKING: from inspect import Parameter + from pydantic import BaseModel + from airflow.models.baseoperatorlink import BaseOperatorLink from airflow.models.expandinput import ExpandInput from airflow.models.operator import Operator @@ -103,7 +105,6 @@ from airflow.serialization.json_schema import Validator from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.timetables.base import Timetable - from airflow.utils.pydantic import BaseModel HAS_KUBERNETES: bool try: @@ -364,8 +365,8 @@ def encode_start_trigger_args(var: StartTriggerArgs) -> dict[str, Any]: :meta private: """ - serialize_kwargs = ( - lambda key: BaseSerialization.serialize(getattr(var, key)) if getattr(var, key) is not None else None + serialize_kwargs = lambda key: ( + BaseSerialization.serialize(getattr(var, key)) if getattr(var, key) is not None else None ) return { "__type": "START_TRIGGER_ARGS", diff --git a/airflow/utils/pydantic.py b/airflow/utils/pydantic.py deleted file mode 100644 index 85fde06195362..0000000000000 --- a/airflow/utils/pydantic.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This is an util module that makes Pydantic use optional. While we are using Pydantic in the airflow core -# codebase, we don't want to make it a hard dependency for all the users of the core codebase, because -# it is only used in the serialization and deserialization of the models for Internal API and for nothing -# else, and since Pydantic is a very popular library, we don't want to force the users of the core codebase -# to install specific Pydantic version - especially that a lot of libraries out there still depend on -# Pydantic 1 and our internal API uses Pydantic 2+ - -from __future__ import annotations - -from importlib import metadata - -from packaging import version - - -def is_pydantic_2_installed() -> bool: - try: - return version.parse(metadata.version("pydantic")).major == 2 - except ImportError: - return False - - -if is_pydantic_2_installed(): - from pydantic import BaseModel, ConfigDict, PlainSerializer, PlainValidator, ValidationInfo -else: - - class BaseModel: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class ConfigDict: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class PlainSerializer: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class PlainValidator: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass - - class ValidationInfo: # type: ignore[no-redef] # noqa: D101 - def __init__(self, *args, **kwargs): - pass diff --git a/contributing-docs/12_airflow_dependencies_and_extras.rst b/contributing-docs/12_airflow_dependencies_and_extras.rst index 2bd2c1f953037..baccdbf6e0e60 100644 --- a/contributing-docs/12_airflow_dependencies_and_extras.rst +++ b/contributing-docs/12_airflow_dependencies_and_extras.rst @@ -165,8 +165,8 @@ Those extras are available as regular core airflow extras - they install optiona .. START CORE EXTRAS HERE aiobotocore, apache-atlas, apache-webhdfs, async, cgroups, cloudpickle, github-enterprise, google- -auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, pydantic, rabbitmq, s3fs, saml, -sentry, statsd, uv, virtualenv +auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, rabbitmq, s3fs, saml, sentry, +statsd, uv, virtualenv .. END CORE EXTRAS HERE diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 54814bb1078a5..53fb9b978f4d7 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -1,4 +1,4 @@ - +