From d3c65233344e37a30bc55c5eb446d7f1b62736d7 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 15 May 2024 16:56:27 +0530 Subject: [PATCH 1/8] UI changes for DAG Reparsing feature releated: https://github.com/apache/airflow/pull/39138 --- airflow/www/static/js/main.js | 15 +++++++++++++++ airflow/www/templates/airflow/dag.html | 6 ++++++ airflow/www/templates/airflow/dags.html | 4 ++++ airflow/www/views.py | 5 +++++ 4 files changed, 30 insertions(+) diff --git a/airflow/www/static/js/main.js b/airflow/www/static/js/main.js index 95861ea6c13a6..98222bafff5da 100644 --- a/airflow/www/static/js/main.js +++ b/airflow/www/static/js/main.js @@ -287,3 +287,18 @@ $(document).ready(() => { // Global Tooltip selector $(".js-tooltip").tooltip(); }); + +$(".reparse_dag").click((event) => { + console.log(event); + event.preventDefault(); + $.ajax({ + url: event.currentTarget.attributes.href.value, + type: "PUT", + done(response) { + console.log(response); + }, + fail(response) { + console.log(response); + }, + }); +}); diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index c91d4b4454a44..6a180bc5e591d 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -235,6 +235,12 @@

+ + cached + diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html index 1bb5ac25abf4d..77e96768a9753 100644 --- a/airflow/www/templates/airflow/dags.html +++ b/airflow/www/templates/airflow/dags.html @@ -398,6 +398,10 @@

{{ page_title }}

class="btn btn-sm btn-default btn-icon-only{{ ' disabled' if not dag.can_delete }}"> + + cached + {% endif %} diff --git a/airflow/www/views.py b/airflow/www/views.py index fbc5296f733db..2ce7ee6829dea 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -48,6 +48,7 @@ Response, abort, before_render_template, + current_app, flash, g, has_request_context, @@ -67,6 +68,7 @@ from flask_appbuilder.urltools import get_order_args, get_page_args, get_page_size_args from flask_appbuilder.widgets import FormWidget from flask_babel import lazy_gettext +from itsdangerous import URLSafeSerializer from jinja2.utils import htmlsafe_json_dumps, pformat # type: ignore from markupsafe import Markup, escape from pendulum.datetime import DateTime @@ -163,6 +165,7 @@ SENSITIVE_FIELD_PLACEHOLDER = "RATHER_LONG_SENSITIVE_FIELD_PLACEHOLDER" logger = logging.getLogger(__name__) +url_serializer = URLSafeSerializer(current_app.config["SECRET_KEY"]) def sanitize_args(args: dict[str, Any]) -> dict[str, Any]: @@ -937,6 +940,7 @@ def index(self): dag.can_delete = get_auth_manager().is_authorized_dag( method="DELETE", details=DagDetails(id=dag.dag_id), user=g.user ) + dag.file_token = url_serializer.dumps(dag.fileloc) dagtags = session.execute(select(func.distinct(DagTag.name)).order_by(DagTag.name)).all() tags = [ @@ -2846,6 +2850,7 @@ def grid(self, dag_id: str, session: Session = NEW_SESSION): color_log_warning_keywords = conf.get("logging", "color_log_warning_keywords", fallback="") dag = get_airflow_app().dag_bag.get_dag(dag_id, session=session) + dag.file_token = url_serializer.dumps(dag.fileloc) dag_model = DagModel.get_dagmodel(dag_id, session=session) if not dag: flash(f'DAG "{dag_id}" seems to be missing from DagBag.', "error") From f68f72afde942bd00b823eafdbd94a8d54b9a0cd Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 15 May 2024 17:47:50 +0530 Subject: [PATCH 2/8] Add correct permission for UI --- airflow/www/templates/airflow/dags.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html index 77e96768a9753..6acff5f70b326 100644 --- a/airflow/www/templates/airflow/dags.html +++ b/airflow/www/templates/airflow/dags.html @@ -399,7 +399,7 @@

{{ page_title }}

+ class="btn btn-sm btn-default btn-icon-only{{ ' disabled' if not dag.can_edit }} reparse_dag"> cached {% endif %} From c50c749c26556d75edee5743fe8da1d79d56cdcf Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 15 May 2024 17:54:48 +0530 Subject: [PATCH 3/8] Fix JS --- airflow/www/static/js/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/www/static/js/main.js b/airflow/www/static/js/main.js index 98222bafff5da..ccaf7556db8e6 100644 --- a/airflow/www/static/js/main.js +++ b/airflow/www/static/js/main.js @@ -294,10 +294,10 @@ $(".reparse_dag").click((event) => { $.ajax({ url: event.currentTarget.attributes.href.value, type: "PUT", - done(response) { + success: (response) => { console.log(response); }, - fail(response) { + error: (response) => { console.log(response); }, }); From e4fedf7d67cf72f552119ff9d10fc323e215bd0a Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Thu, 16 May 2024 12:33:46 +0530 Subject: [PATCH 4/8] Update airflow/www/static/js/main.js --- airflow/www/static/js/main.js | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/www/static/js/main.js b/airflow/www/static/js/main.js index ccaf7556db8e6..05fabdc432468 100644 --- a/airflow/www/static/js/main.js +++ b/airflow/www/static/js/main.js @@ -289,7 +289,6 @@ $(document).ready(() => { }); $(".reparse_dag").click((event) => { - console.log(event); event.preventDefault(); $.ajax({ url: event.currentTarget.attributes.href.value, From f4366b272ba042e5cb7632cff549fb0ff3819d6f Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Fri, 17 May 2024 18:10:37 +0530 Subject: [PATCH 5/8] Add a wrapper arounf REST DAG Reparsing API --- airflow/www/static/js/main.js | 14 -------------- airflow/www/templates/airflow/dag.html | 12 ++++++------ airflow/www/templates/airflow/dags.html | 8 ++++---- airflow/www/views.py | 19 ++++++++++++++++++- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/airflow/www/static/js/main.js b/airflow/www/static/js/main.js index 05fabdc432468..95861ea6c13a6 100644 --- a/airflow/www/static/js/main.js +++ b/airflow/www/static/js/main.js @@ -287,17 +287,3 @@ $(document).ready(() => { // Global Tooltip selector $(".js-tooltip").tooltip(); }); - -$(".reparse_dag").click((event) => { - event.preventDefault(); - $.ajax({ - url: event.currentTarget.attributes.href.value, - type: "PUT", - success: (response) => { - console.log(response); - }, - error: (response) => { - console.log(response); - }, - }); -}); diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 6a180bc5e591d..9be36949c2936 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -228,6 +228,12 @@

{{ page_title }}

{% endif %} + + cached + - - cached - {% endif %} diff --git a/airflow/www/views.py b/airflow/www/views.py index 2ce7ee6829dea..d187c18f294b8 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -165,7 +165,6 @@ SENSITIVE_FIELD_PLACEHOLDER = "RATHER_LONG_SENSITIVE_FIELD_PLACEHOLDER" logger = logging.getLogger(__name__) -url_serializer = URLSafeSerializer(current_app.config["SECRET_KEY"]) def sanitize_args(args: dict[str, Any]) -> dict[str, Any]: @@ -940,6 +939,7 @@ def index(self): dag.can_delete = get_auth_manager().is_authorized_dag( method="DELETE", details=DagDetails(id=dag.dag_id), user=g.user ) + url_serializer = URLSafeSerializer(current_app.config["SECRET_KEY"]) dag.file_token = url_serializer.dumps(dag.fileloc) dagtags = session.execute(select(func.distinct(DagTag.name)).order_by(DagTag.name)).all() @@ -2850,6 +2850,7 @@ def grid(self, dag_id: str, session: Session = NEW_SESSION): color_log_warning_keywords = conf.get("logging", "color_log_warning_keywords", fallback="") dag = get_airflow_app().dag_bag.get_dag(dag_id, session=session) + url_serializer = URLSafeSerializer(current_app.config["SECRET_KEY"]) dag.file_token = url_serializer.dumps(dag.fileloc) dag_model = DagModel.get_dagmodel(dag_id, session=session) if not dag: @@ -3562,6 +3563,22 @@ def audit_log(self, dag_id: str): return redirect(url_for("Airflow.grid", **kwargs)) + @expose("/parseDagFile/") + def parse_dag(self, file_token: str): + from airflow.api_connexion.endpoints.dag_parsing import reparse_dag_file + + with create_session() as session: + response = reparse_dag_file(file_token=file_token, session=session) + response_messages = { + 201: ["Reparsing request submitted successfully", "info"], + 401: ["Unauthenticated request", "error"], + 403: ["Permission Denied", "error"], + 404: ["DAG not found", "error"], + } + flash(response_messages[response.status_code][0], response_messages[response.status_code][1]) + redirect_url = get_safe_url(request.values.get("redirect_url")) + return redirect(redirect_url) + class ConfigurationView(AirflowBaseView): """View to show Airflow Configurations.""" From e4b9d8673c948d68a7e1b410cdcb28b02f198138 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 20 May 2024 16:42:06 +0530 Subject: [PATCH 6/8] Move file_token from dag object to a separate field --- airflow/www/templates/airflow/dag.html | 2 +- airflow/www/templates/airflow/dags.html | 2 +- airflow/www/views.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 337f989db8de7..9c43ff094cc52 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -231,7 +231,7 @@