From 7cc1c6a46eaa62b7fb04dd4b841a73fee2c7ea38 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 13 Oct 2023 11:58:13 +0100 Subject: [PATCH 1/4] Add extra fields to plugins endpoint I added three extra fields, ti_deps, timetables, and listeners which I think are worth having since they will help in visualizing if those are included in a plugin. I also found out that the UI has admin_views and menu_links which seems to come from airflow 1 but for consistency, I merged the attributes for both UI & webserver to be the same. The REST API does not have these two attributes as I feel they will soon be removed. --- airflow/api_connexion/openapi/v1.yaml | 18 ++++++++++++ .../api_connexion/schemas/plugin_schema.py | 3 ++ airflow/plugins_manager.py | 4 ++- airflow/www/static/js/types/api-generated.ts | 6 ++++ airflow/www/views.py | 15 ++-------- .../endpoints/test_plugin_endpoint.py | 28 +++++++++++++++++++ .../schemas/test_plugin_schema.py | 9 ++++++ 7 files changed, 69 insertions(+), 14 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index ebd10e855a679..f6892cf6c5398 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -3833,6 +3833,24 @@ components: type: string description: The plugin source nullable: true + ti_deps: + type: array + items: + type: string + nullable: true + description: The plugin task instance dependencies + listeners: + type: array + items: + type: string + nullable: true + description: The plugin listeners + timetables: + type: array + items: + type: string + nullable: true + description: The plugin timetables PluginCollection: type: object diff --git a/airflow/api_connexion/schemas/plugin_schema.py b/airflow/api_connexion/schemas/plugin_schema.py index 4b62111482075..afdef350bc6fc 100644 --- a/airflow/api_connexion/schemas/plugin_schema.py +++ b/airflow/api_connexion/schemas/plugin_schema.py @@ -34,6 +34,9 @@ class PluginSchema(Schema): global_operator_extra_links = fields.List(fields.String()) operator_extra_links = fields.List(fields.String()) source = fields.String() + ti_deps = fields.List(fields.String()) + listeners = fields.List(fields.String()) + timetables = fields.List(fields.String()) class PluginCollection(NamedTuple): diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py index 7275588d52e07..143e3af5707bc 100644 --- a/airflow/plugins_manager.py +++ b/airflow/plugins_manager.py @@ -78,14 +78,16 @@ "hooks", "executors", "macros", + "admin_views", "flask_blueprints", + "menu_links", "appbuilder_views", "appbuilder_menu_items", "global_operator_extra_links", "operator_extra_links", + "source", "ti_deps", "timetables", - "source", "listeners", } diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 5c7a8ecb085f0..8865ea21ffed7 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1597,6 +1597,12 @@ export interface components { operator_extra_links?: (string | null)[]; /** @description The plugin source */ source?: string | null; + /** @description The plugin task instance dependencies */ + ti_deps?: (string | null)[]; + /** @description The plugin listeners */ + listeners?: (string | null)[]; + /** @description The plugin timetables */ + timetables?: (string | null)[]; }; /** * @description A collection of plugin. diff --git a/airflow/www/views.py b/airflow/www/views.py index 1669413815a1e..e6f56e42161c9 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -106,6 +106,7 @@ from airflow.models.operator import needs_expansion from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskinstance import TaskInstance, TaskInstanceNote +from airflow.plugins_manager import PLUGINS_ATTRIBUTES_TO_DUMP from airflow.providers_manager import ProvidersManager from airflow.security import permissions from airflow.ti_deps.dep_context import DepContext @@ -4521,19 +4522,7 @@ class PluginView(AirflowBaseView): permissions.ACTION_CAN_ACCESS_MENU, ] - plugins_attributes_to_dump = [ - "hooks", - "executors", - "macros", - "admin_views", - "flask_blueprints", - "menu_links", - "appbuilder_views", - "appbuilder_menu_items", - "global_operator_extra_links", - "operator_extra_links", - "source", - ] + plugins_attributes_to_dump = PLUGINS_ATTRIBUTES_TO_DUMP @expose("/plugin") @auth.has_access_website() diff --git a/tests/api_connexion/endpoints/test_plugin_endpoint.py b/tests/api_connexion/endpoints/test_plugin_endpoint.py index a6f67ab5a7017..42de5efc4aaaa 100644 --- a/tests/api_connexion/endpoints/test_plugin_endpoint.py +++ b/tests/api_connexion/endpoints/test_plugin_endpoint.py @@ -24,6 +24,8 @@ from airflow.models.baseoperator import BaseOperatorLink from airflow.plugins_manager import AirflowPlugin from airflow.security import permissions +from airflow.ti_deps.deps.base_ti_dep import BaseTIDep +from airflow.timetables.base import Timetable from airflow.utils.module_loading import qualname from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests.test_utils.config import conf_vars @@ -60,6 +62,26 @@ class MockView(BaseView): } +class CustomTIDep(BaseTIDep): + pass + + +ti_dep = CustomTIDep() + + +class CustomTimetable(Timetable): + def infer_manual_data_interval(self, *, run_after): + pass + + def next_dagrun_info( + self, + *, + last_automated_data_interval, + restriction, + ): + pass + + class MockPlugin(AirflowPlugin): name = "mock_plugin" flask_blueprints = [bp] @@ -69,6 +91,9 @@ class MockPlugin(AirflowPlugin): operator_extra_links = [MockOperatorLink()] hooks = [PluginHook] macros = [plugin_macro] + ti_deps = [ti_dep] + timetables = [CustomTimetable] + listeners = [pytest] # using pytest here because we need a module(just for test) @pytest.fixture(scope="module") @@ -120,6 +145,9 @@ def test_get_plugins_return_200(self): "operator_extra_links": [f"<{qualname(MockOperatorLink().__class__)} object>"], "source": None, "name": "test_plugin", + "timetables": [qualname(CustomTimetable)], + "ti_deps": [str(ti_dep)], + "listeners": [pytest.__name__], } ], "total_entries": 1, diff --git a/tests/api_connexion/schemas/test_plugin_schema.py b/tests/api_connexion/schemas/test_plugin_schema.py index 179a318fe5021..1472fd2db7b21 100644 --- a/tests/api_connexion/schemas/test_plugin_schema.py +++ b/tests/api_connexion/schemas/test_plugin_schema.py @@ -91,6 +91,9 @@ def test_serialize(self): "operator_extra_links": [str(MockOperatorLink())], "source": None, "name": "test_plugin", + "ti_deps": [], + "listeners": [], + "timetables": [], } @@ -112,6 +115,9 @@ def test_serialize(self): "operator_extra_links": [str(MockOperatorLink())], "source": None, "name": "test_plugin", + "ti_deps": [], + "listeners": [], + "timetables": [], }, { "appbuilder_menu_items": [appbuilder_menu_items], @@ -124,6 +130,9 @@ def test_serialize(self): "operator_extra_links": [str(MockOperatorLink())], "source": None, "name": "test_plugin_2", + "ti_deps": [], + "listeners": [], + "timetables": [], }, ], "total_entries": 2, From ecaac07d41fb78ac1b8ecf4c69c9ea1429b391d3 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 13 Oct 2023 13:02:11 +0100 Subject: [PATCH 2/4] fixup! Add extra fields to plugins endpoint --- tests/cli/commands/test_plugins_command.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/cli/commands/test_plugins_command.py b/tests/cli/commands/test_plugins_command.py index 049fab4354032..cbf6afeab9257 100644 --- a/tests/cli/commands/test_plugins_command.py +++ b/tests/cli/commands/test_plugins_command.py @@ -61,7 +61,9 @@ def test_should_display_one_plugins(self): assert info == [ { "name": "test_plugin", + "admin_views": [], "macros": ["tests.plugins.test_plugin.plugin_macro"], + "menu_links": [], "executors": ["tests.plugins.test_plugin.PluginExecutor"], "flask_blueprints": [ "" From f8b11d97cacf452bd358bfbed0cf61562c4296e2 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Wed, 18 Oct 2023 12:08:24 +0100 Subject: [PATCH 3/4] Remove nullable=true in the arrays --- airflow/api_connexion/openapi/v1.yaml | 3 --- airflow/www/static/js/types/api-generated.ts | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index f6892cf6c5398..8fd0a582e6555 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -3837,19 +3837,16 @@ components: type: array items: type: string - nullable: true description: The plugin task instance dependencies listeners: type: array items: type: string - nullable: true description: The plugin listeners timetables: type: array items: type: string - nullable: true description: The plugin timetables PluginCollection: diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 8865ea21ffed7..e3368f8a79967 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1598,11 +1598,11 @@ export interface components { /** @description The plugin source */ source?: string | null; /** @description The plugin task instance dependencies */ - ti_deps?: (string | null)[]; + ti_deps?: string[]; /** @description The plugin listeners */ - listeners?: (string | null)[]; + listeners?: string[]; /** @description The plugin timetables */ - timetables?: (string | null)[]; + timetables?: string[]; }; /** * @description A collection of plugin. From e1ca400830d478c405f254953bb37e86958d9d1c Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Thu, 19 Oct 2023 09:45:54 +0100 Subject: [PATCH 4/4] update the listener serialization --- .../api_connexion/endpoints/test_plugin_endpoint.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/api_connexion/endpoints/test_plugin_endpoint.py b/tests/api_connexion/endpoints/test_plugin_endpoint.py index 42de5efc4aaaa..6a941d69977e3 100644 --- a/tests/api_connexion/endpoints/test_plugin_endpoint.py +++ b/tests/api_connexion/endpoints/test_plugin_endpoint.py @@ -16,6 +16,8 @@ # under the License. from __future__ import annotations +import inspect + import pytest from flask import Blueprint from flask_appbuilder import BaseView @@ -82,6 +84,10 @@ def next_dagrun_info( pass +class MyCustomListener: + pass + + class MockPlugin(AirflowPlugin): name = "mock_plugin" flask_blueprints = [bp] @@ -93,7 +99,7 @@ class MockPlugin(AirflowPlugin): macros = [plugin_macro] ti_deps = [ti_dep] timetables = [CustomTimetable] - listeners = [pytest] # using pytest here because we need a module(just for test) + listeners = [pytest, MyCustomListener()] # using pytest here because we need a module(just for test) @pytest.fixture(scope="module") @@ -147,7 +153,10 @@ def test_get_plugins_return_200(self): "name": "test_plugin", "timetables": [qualname(CustomTimetable)], "ti_deps": [str(ti_dep)], - "listeners": [pytest.__name__], + "listeners": [ + d.__name__ if inspect.ismodule(d) else qualname(d) + for d in [pytest, MyCustomListener()] + ], } ], "total_entries": 1,