From 77fd8632604d5741b2920a573b525e04d6e66873 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Mon, 11 Nov 2024 22:31:29 +0530 Subject: [PATCH 1/8] Moved filesystem sensor to standard provider --- .../src/airflow/providers/standard}/sensors/filesystem.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {airflow => providers/src/airflow/providers/standard}/sensors/filesystem.py (100%) diff --git a/airflow/sensors/filesystem.py b/providers/src/airflow/providers/standard/sensors/filesystem.py similarity index 100% rename from airflow/sensors/filesystem.py rename to providers/src/airflow/providers/standard/sensors/filesystem.py From 31520329ed9e7860b282c3fd47993224a15c8458 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Mon, 11 Nov 2024 22:34:04 +0530 Subject: [PATCH 2/8] Update all references to filesystem sensor --- RELEASE_NOTES.rst | 2 +- airflow/example_dags/example_sensors.py | 2 +- airflow/reproducible_build.yaml | 4 ++-- docs/apache-airflow/howto/operator/file.rst | 2 +- docs/apache-airflow/operators-and-hooks-ref.rst | 2 +- docs/apache-airflow/tutorial/taskflow.rst | 2 +- tests/cli/commands/test_task_command.py | 2 +- tests/sensors/test_filesystem.py | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 920529d777b34..c55bcf360953e 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -6964,7 +6964,7 @@ The change is backwards compatible, setting ``provide_context`` will add the ``p PR: `#5990 `_ -``airflow.sensors.filesystem.FileSensor`` +``airflow.providers.standard.sensors.filesystem.FileSensor`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FileSensor is now takes a glob pattern, not just a filename. If the filename you are looking for has ``*``\ , ``?``\ , or ``[`` in it then you should replace these with ``[*]``\ , ``[?]``\ , and ``[[]``. diff --git a/airflow/example_dags/example_sensors.py b/airflow/example_dags/example_sensors.py index 52b1b84e223e8..39d7b8d29635f 100644 --- a/airflow/example_dags/example_sensors.py +++ b/airflow/example_dags/example_sensors.py @@ -24,11 +24,11 @@ from airflow.models.dag import DAG from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.standard.sensors.bash import BashSensor +from airflow.providers.standard.sensors.filesystem import FileSensor from airflow.providers.standard.sensors.python import PythonSensor from airflow.providers.standard.sensors.time import TimeSensor, TimeSensorAsync from airflow.providers.standard.sensors.time_delta import TimeDeltaSensor, TimeDeltaSensorAsync from airflow.providers.standard.sensors.weekday import DayOfWeekSensor -from airflow.sensors.filesystem import FileSensor from airflow.utils.trigger_rule import TriggerRule from airflow.utils.weekday import WeekDay diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index 79316e3ecdd6f..628110124d00c 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: 4893030c9c8a4d22d13291823b88ad4d -source-date-epoch: 1730818031 +release-notes-hash: c079f97067295b2b30677775961f93c3 +source-date-epoch: 1731344550 diff --git a/docs/apache-airflow/howto/operator/file.rst b/docs/apache-airflow/howto/operator/file.rst index 49ca1c75f6042..6241cc2e50533 100644 --- a/docs/apache-airflow/howto/operator/file.rst +++ b/docs/apache-airflow/howto/operator/file.rst @@ -22,7 +22,7 @@ FileSensor ========== -Use the :class:`~airflow.sensors.filesystem.FileSensor` to detect files appearing in your local +Use the :class:`~airflow.providers.standard.sensors.filesystem.FileSensor` to detect files appearing in your local filesystem. You need to have connection defined to use it (pass connection id via ``fs_conn_id``). Default connection is ``fs_default``. diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst index ae131fb8d268f..cc47dc2f22f0c 100644 --- a/docs/apache-airflow/operators-and-hooks-ref.rst +++ b/docs/apache-airflow/operators-and-hooks-ref.rst @@ -79,7 +79,7 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - :mod:`airflow.sensors.external_task` - :doc:`How to use ` - * - :mod:`airflow.sensors.filesystem` + * - :mod:`airflow.providers.standard.sensors.filesystem` - :ref:`How to use ` diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index 72b5ac82dd049..0b6bf77316fb2 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -438,7 +438,7 @@ Adding dependencies between decorated and traditional tasks ----------------------------------------------------------- The above tutorial shows how to create dependencies between TaskFlow functions. However, dependencies can also be set between traditional tasks (such as :class:`~airflow.providers.standard.operators.bash.BashOperator` -or :class:`~airflow.sensors.filesystem.FileSensor`) and TaskFlow functions. +or :class:`~airflow.providers.standard.sensors.filesystem.FileSensor`) and TaskFlow functions. Building this dependency is shown in the code below: diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index 50e3d393f8108..c6fec18541a7e 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -397,7 +397,7 @@ def test_cli_test_with_env_vars(self): @mock.patch("airflow.triggers.file.os.path.getmtime", return_value=0) @mock.patch("airflow.triggers.file.glob", return_value=["/tmp/test"]) @mock.patch("airflow.triggers.file.os.path.isfile", return_value=True) - @mock.patch("airflow.sensors.filesystem.FileSensor.poke", return_value=False) + @mock.patch("airflow.providers.standard.sensors.filesystem.FileSensor.poke", return_value=False) def test_cli_test_with_deferrable_operator( self, mock_pock, mock_is_file, mock_glob, mock_getmtime, caplog ): diff --git a/tests/sensors/test_filesystem.py b/tests/sensors/test_filesystem.py index 641f2f218f2db..0774b56c31cd8 100644 --- a/tests/sensors/test_filesystem.py +++ b/tests/sensors/test_filesystem.py @@ -26,7 +26,7 @@ from airflow.exceptions import AirflowSensorTimeout, TaskDeferred from airflow.models.dag import DAG -from airflow.sensors.filesystem import FileSensor +from airflow.providers.standard.sensors.filesystem import FileSensor from airflow.triggers.file import FileTrigger from airflow.utils.timezone import datetime From 6fd1bb6fe199b1cabd656608786e49abbc66f4ec Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Tue, 12 Nov 2024 11:46:14 +0530 Subject: [PATCH 3/8] Update provider.yaml with standard filesystem sensor name --- providers/src/airflow/providers/standard/provider.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/providers/src/airflow/providers/standard/provider.yaml b/providers/src/airflow/providers/standard/provider.yaml index 007cfd812b7af..4c4c2ae5d1c20 100644 --- a/providers/src/airflow/providers/standard/provider.yaml +++ b/providers/src/airflow/providers/standard/provider.yaml @@ -56,6 +56,7 @@ sensors: - airflow.providers.standard.sensors.weekday - airflow.providers.standard.sensors.bash - airflow.providers.standard.sensors.python + - airflow.providers.standard.sensors.filesystem hooks: - integration-name: Standard python-modules: From a189fcac8f37fdf76affd68b08d4096ae8742b6b Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Tue, 12 Nov 2024 12:57:52 +0530 Subject: [PATCH 4/8] Typo fix in contributing-docs --- contributing-docs/03_contributors_quick_start.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index e6279936e6854..121323c5c9bdb 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -478,7 +478,7 @@ or pipx install pre-commit -You can add ````uv`` support for ``pre-commit`` even you install it with ``pipx`` using the commands +You can add ``uv`` support for ``pre-commit`` even you install it with ``pipx`` using the commands (then pre-commit will use ``uv`` to create virtualenvs for the hooks): .. code-block:: bash From 7a6da7eb0f25d528b3e1176f65d697515eb433e9 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Tue, 12 Nov 2024 18:09:44 +0530 Subject: [PATCH 5/8] Changing title underline as per static test error msg --- RELEASE_NOTES.rst | 2 +- airflow/reproducible_build.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index c55bcf360953e..a8b5a265f3afa 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -6965,7 +6965,7 @@ The change is backwards compatible, setting ``provide_context`` will add the ``p PR: `#5990 `_ ``airflow.providers.standard.sensors.filesystem.FileSensor`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FileSensor is now takes a glob pattern, not just a filename. If the filename you are looking for has ``*``\ , ``?``\ , or ``[`` in it then you should replace these with ``[*]``\ , ``[?]``\ , and ``[[]``. diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index 628110124d00c..1c13b028bb4f0 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: c079f97067295b2b30677775961f93c3 -source-date-epoch: 1731344550 +release-notes-hash: c68f3fa23f84c7fc270d73baaa2cc18d +source-date-epoch: 1731415143 From 215e490c6122382618175b0ee6fc1f87286cbcdb Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 15 Nov 2024 19:01:42 +0530 Subject: [PATCH 6/8] DAT-1897 - Update docs and references as per review comments --- .../operators}/file.rst | 0 docs/apache-airflow/operators-and-hooks-ref.rst | 3 --- 2 files changed, 3 deletions(-) rename docs/{apache-airflow/howto/operator => apache-airflow-providers-standard/operators}/file.rst (100%) diff --git a/docs/apache-airflow/howto/operator/file.rst b/docs/apache-airflow-providers-standard/operators/file.rst similarity index 100% rename from docs/apache-airflow/howto/operator/file.rst rename to docs/apache-airflow-providers-standard/operators/file.rst diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst index cc47dc2f22f0c..f9efbfd2b52f6 100644 --- a/docs/apache-airflow/operators-and-hooks-ref.rst +++ b/docs/apache-airflow/operators-and-hooks-ref.rst @@ -79,9 +79,6 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - :mod:`airflow.sensors.external_task` - :doc:`How to use ` - * - :mod:`airflow.providers.standard.sensors.filesystem` - - :ref:`How to use ` - **Hooks:** From f36e1d74dbcea17998b4ee98f6cc9bf81b39c743 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 15 Nov 2024 20:09:12 +0530 Subject: [PATCH 7/8] Moved sensor documentation to standard provider --- .../{operators => sensors}/file.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/apache-airflow-providers-standard/{operators => sensors}/file.rst (100%) diff --git a/docs/apache-airflow-providers-standard/operators/file.rst b/docs/apache-airflow-providers-standard/sensors/file.rst similarity index 100% rename from docs/apache-airflow-providers-standard/operators/file.rst rename to docs/apache-airflow-providers-standard/sensors/file.rst From 8b6a62551f6d5ef0b149c016c8019001718a6982 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 15 Nov 2024 20:09:48 +0530 Subject: [PATCH 8/8] Removed reference to file sensor from index.rst --- docs/apache-airflow/howto/operator/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/apache-airflow/howto/operator/index.rst b/docs/apache-airflow/howto/operator/index.rst index c16ec419c14f2..5dcda247522ca 100644 --- a/docs/apache-airflow/howto/operator/index.rst +++ b/docs/apache-airflow/howto/operator/index.rst @@ -29,5 +29,4 @@ determine what actually executes when your DAG runs. .. toctree:: :maxdepth: 2 - file external_task_sensor