From 4c25c0becd38b99d931770f43a33cdeb527ffced Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Tue, 5 Mar 2024 17:18:32 -0500 Subject: [PATCH 1/2] Add TaskFail entries to Gantt chart --- airflow/www/static/js/api/index.ts | 2 + airflow/www/static/js/api/useTaskFails.ts | 60 ++++++++++++++++ .../www/static/js/dag/details/gantt/Row.tsx | 70 +++++++++++++++++-- .../js/dag/details/task/TaskDuration.tsx | 6 +- airflow/www/templates/airflow/dag.html | 1 + airflow/www/views.py | 25 +++++++ 6 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 airflow/www/static/js/api/useTaskFails.ts diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index b04da5259a99c..4f883b48251ce 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -52,6 +52,7 @@ import useHistoricalMetricsData from "./useHistoricalMetricsData"; import { useTaskXcomEntry, useTaskXcomCollection } from "./useTaskXcom"; import useEventLogs from "./useEventLogs"; import useCalendarData from "./useCalendarData"; +import useTaskFails from "./useTaskFails"; axios.interceptors.request.use((config) => { config.paramsSerializer = { @@ -100,4 +101,5 @@ export { useTaskXcomCollection, useEventLogs, useCalendarData, + useTaskFails, }; diff --git a/airflow/www/static/js/api/useTaskFails.ts b/airflow/www/static/js/api/useTaskFails.ts new file mode 100644 index 0000000000000..75628400d348d --- /dev/null +++ b/airflow/www/static/js/api/useTaskFails.ts @@ -0,0 +1,60 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useQuery } from "react-query"; +import axios, { AxiosResponse } from "axios"; + +import { getMetaValue } from "src/utils"; + +const DAG_ID_PARAM = "dag_id"; +const RUN_ID_PARAM = "run_id"; +const TASK_ID_PARAM = "task_id"; + +const dagId = getMetaValue(DAG_ID_PARAM); +const taskFailsUrl = getMetaValue("task_fails_url"); + +export interface TaskFail { + runId: string; + taskId: string; + mapIndex?: number; + startDate?: string; + endDate?: string; +} + +interface Props { + runId?: string; + taskId?: string; + enabled?: boolean; +} + +const useTaskFails = ({ runId, taskId, enabled = true }: Props) => + useQuery( + ["taskFails", runId, taskId], + async () => { + const params = { + [DAG_ID_PARAM]: dagId, + [RUN_ID_PARAM]: runId, + [TASK_ID_PARAM]: taskId, + }; + return axios.get(taskFailsUrl, { params }); + }, + { enabled } + ); + +export default useTaskFails; diff --git a/airflow/www/static/js/dag/details/gantt/Row.tsx b/airflow/www/static/js/dag/details/gantt/Row.tsx index 1526e1713f0ad..08bc132d8817d 100644 --- a/airflow/www/static/js/dag/details/gantt/Row.tsx +++ b/airflow/www/static/js/dag/details/gantt/Row.tsx @@ -18,13 +18,17 @@ */ import React from "react"; -import { Box, Tooltip, Flex } from "@chakra-ui/react"; +import { Box, Tooltip, Flex, Text } from "@chakra-ui/react"; + import useSelection from "src/dag/useSelection"; import { getDuration } from "src/datetime_utils"; -import { SimpleStatus } from "src/dag/StatusBox"; +import { SimpleStatus, boxSize } from "src/dag/StatusBox"; import { useContainerRef } from "src/context/containerRef"; import { hoverDelay } from "src/utils"; import type { Task } from "src/types"; +import { useTaskFails } from "src/api"; +import Time from "src/components/Time"; + import GanttTooltip from "./GanttTooltip"; interface Props { @@ -59,6 +63,17 @@ const Row = ({ : true); const isOpen = openGroupIds.includes(task.id || ""); + const { data: taskFails } = useTaskFails({ + taskId: task.id || undefined, + runId: runId || undefined, + enabled: !!(instance?.tryNumber && instance?.tryNumber > 1) && !!task.id, // Only try to look up task fails if it even has a try number > 1 + }); + + const pastFails = (taskFails || []).filter( + (tf) => + instance?.startDate && tf?.startDate && tf.startDate < instance?.startDate + ); + // Calculate durations in ms const taskDuration = getDuration(instance?.startDate, instance?.endDate); const queuedDuration = hasValidQueuedDttm @@ -84,10 +99,12 @@ const Row = ({ return (
{instance ? ( { onSelect({ runId: instance.runId, @@ -132,6 +151,49 @@ const Row = ({ ) : ( )} + {pastFails.map((tf) => { + const duration = getDuration(tf?.startDate, tf?.endDate); + const percent = duration / runDuration; + const failWidth = ganttWidth * percent; + + const startOffset = getDuration(ganttStartDate, tf?.startDate); + const offsetLeft = (startOffset / runDuration) * ganttWidth; + + return ( + + Task Fail + {tf?.startDate && ( + + Start: + )} + {instance?.endDate && ( + + End: + )} + + } + hasArrow + portalProps={{ containerRef }} + placement="top" + openDelay={hoverDelay} + key={`${tf.taskId}-${tf.startDate}`} + top="4px" + > + + + + + ); + })} {isOpen && !!task.children && diff --git a/airflow/www/static/js/dag/details/task/TaskDuration.tsx b/airflow/www/static/js/dag/details/task/TaskDuration.tsx index 4b7eed9f3d365..27cd8b33ec94e 100644 --- a/airflow/www/static/js/dag/details/task/TaskDuration.tsx +++ b/airflow/www/static/js/dag/details/task/TaskDuration.tsx @@ -22,7 +22,7 @@ import React from "react"; import useSelection from "src/dag/useSelection"; -import { useGridData } from "src/api"; +import { useGridData, useTaskFails } from "src/api"; import { startCase } from "lodash"; import { getDuration, formatDateTime, defaultFormat } from "src/datetime_utils"; import ReactECharts, { ReactEChartsProps } from "src/components/ReactECharts"; @@ -45,6 +45,10 @@ const TaskDuration = () => { onSelect, } = useSelection(); + const { data: taskFails } = useTaskFails({ taskId: taskId || undefined }); + + console.log(taskFails); + const { data: { dagRuns, groups, ordering }, } = useGridData(); diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index c69251e71c9b3..6f5a187f61f68 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -57,6 +57,7 @@ + diff --git a/airflow/www/views.py b/airflow/www/views.py index 5cffd28aaa3b3..743947cdd397b 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -3505,6 +3505,31 @@ def graph_data(self): {"Content-Type": "application/json; charset=utf-8"}, ) + @expose("/object/task_fails") + @auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE) + @provide_session + def task_fails(self, session): + """Return task fails.""" + dag_id = request.args.get("dag_id") + task_id = request.args.get("task_id") + run_id = request.args.get("run_id") + + query = select( + TaskFail.task_id, TaskFail.run_id, TaskFail.map_index, TaskFail.start_date, TaskFail.end_date + ).where(TaskFail.dag_id == dag_id) + + if run_id: + query = query.where(TaskFail.run_id == run_id) + if task_id: + query = query.where(TaskFail.task_id == task_id) + + task_fails = [dict(tf) for tf in session.execute(query).all()] + + return ( + htmlsafe_json_dumps(task_fails, separators=(",", ":"), dumps=flask.json.dumps), + {"Content-Type": "application/json; charset=utf-8"}, + ) + @expose("/object/task_instances") @auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE) def task_instances(self): From febee38f55a44e060edd4419dfab22056ee3412e Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Thu, 7 Mar 2024 16:28:29 -0500 Subject: [PATCH 2/2] Fix some autorefresh --- airflow/www/static/js/api/useTaskFails.ts | 13 ++- .../www/static/js/dag/details/gantt/Row.tsx | 73 ++++----------- .../static/js/dag/details/gantt/TaskFail.tsx | 91 +++++++++++++++++++ 3 files changed, 121 insertions(+), 56 deletions(-) create mode 100644 airflow/www/static/js/dag/details/gantt/TaskFail.tsx diff --git a/airflow/www/static/js/api/useTaskFails.ts b/airflow/www/static/js/api/useTaskFails.ts index 75628400d348d..f256db8f1fbde 100644 --- a/airflow/www/static/js/api/useTaskFails.ts +++ b/airflow/www/static/js/api/useTaskFails.ts @@ -21,6 +21,7 @@ import { useQuery } from "react-query"; import axios, { AxiosResponse } from "axios"; import { getMetaValue } from "src/utils"; +import { useAutoRefresh } from "src/context/autorefresh"; const DAG_ID_PARAM = "dag_id"; const RUN_ID_PARAM = "run_id"; @@ -43,8 +44,10 @@ interface Props { enabled?: boolean; } -const useTaskFails = ({ runId, taskId, enabled = true }: Props) => - useQuery( +const useTaskFails = ({ runId, taskId, enabled = true }: Props) => { + const { isRefreshOn } = useAutoRefresh(); + + return useQuery( ["taskFails", runId, taskId], async () => { const params = { @@ -54,7 +57,11 @@ const useTaskFails = ({ runId, taskId, enabled = true }: Props) => }; return axios.get(taskFailsUrl, { params }); }, - { enabled } + { + enabled, + refetchInterval: isRefreshOn && (autoRefreshInterval || 1) * 1000, + } ); +}; export default useTaskFails; diff --git a/airflow/www/static/js/dag/details/gantt/Row.tsx b/airflow/www/static/js/dag/details/gantt/Row.tsx index 08bc132d8817d..fbee17e13921a 100644 --- a/airflow/www/static/js/dag/details/gantt/Row.tsx +++ b/airflow/www/static/js/dag/details/gantt/Row.tsx @@ -18,7 +18,7 @@ */ import React from "react"; -import { Box, Tooltip, Flex, Text } from "@chakra-ui/react"; +import { Box, Tooltip, Flex } from "@chakra-ui/react"; import useSelection from "src/dag/useSelection"; import { getDuration } from "src/datetime_utils"; @@ -27,9 +27,9 @@ import { useContainerRef } from "src/context/containerRef"; import { hoverDelay } from "src/utils"; import type { Task } from "src/types"; import { useTaskFails } from "src/api"; -import Time from "src/components/Time"; import GanttTooltip from "./GanttTooltip"; +import TaskFail from "./TaskFail"; interface Props { ganttWidth?: number; @@ -69,11 +69,6 @@ const Row = ({ enabled: !!(instance?.tryNumber && instance?.tryNumber > 1) && !!task.id, // Only try to look up task fails if it even has a try number > 1 }); - const pastFails = (taskFails || []).filter( - (tf) => - instance?.startDate && tf?.startDate && tf.startDate < instance?.startDate - ); - // Calculate durations in ms const taskDuration = getDuration(instance?.startDate, instance?.endDate); const queuedDuration = hasValidQueuedDttm @@ -106,7 +101,7 @@ const Row = ({ width={ganttWidth} height={`${boxSize + 9}px`} > - {instance ? ( + {instance && ( } hasArrow @@ -148,52 +143,24 @@ const Row = ({ /> - ) : ( - )} - {pastFails.map((tf) => { - const duration = getDuration(tf?.startDate, tf?.endDate); - const percent = duration / runDuration; - const failWidth = ganttWidth * percent; - - const startOffset = getDuration(ganttStartDate, tf?.startDate); - const offsetLeft = (startOffset / runDuration) * ganttWidth; - - return ( - - Task Fail - {tf?.startDate && ( - - Start: - )} - {instance?.endDate && ( - - End: - )} - - } - hasArrow - portalProps={{ containerRef }} - placement="top" - openDelay={hoverDelay} - key={`${tf.taskId}-${tf.startDate}`} - top="4px" - > - - - - - ); - })} + {/* Only show fails before the most recent task instance */} + {(taskFails || []) + .filter( + (tf) => + tf.startDate !== instance?.startDate && + // @ts-ignore + moment(tf.startDate).isAfter(ganttStartDate) + ) + .map((taskFail) => ( + + ))} {isOpen && !!task.children && diff --git a/airflow/www/static/js/dag/details/gantt/TaskFail.tsx b/airflow/www/static/js/dag/details/gantt/TaskFail.tsx new file mode 100644 index 0000000000000..1d217d3bfb6d0 --- /dev/null +++ b/airflow/www/static/js/dag/details/gantt/TaskFail.tsx @@ -0,0 +1,91 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from "react"; +import { Box, Tooltip, Text } from "@chakra-ui/react"; + +import { getDuration } from "src/datetime_utils"; +import { SimpleStatus } from "src/dag/StatusBox"; +import { useContainerRef } from "src/context/containerRef"; +import { hoverDelay } from "src/utils"; +import Time from "src/components/Time"; + +import type { TaskFail as TaskFailType } from "src/api/useTaskFails"; + +interface Props { + taskFail: TaskFailType; + runDuration: number; + ganttWidth: number; + ganttStartDate?: string | null; +} + +const TaskFail = ({ + taskFail, + runDuration, + ganttWidth, + ganttStartDate, +}: Props) => { + const containerRef = useContainerRef(); + + const duration = getDuration(taskFail?.startDate, taskFail?.endDate); + const percent = duration / runDuration; + const failWidth = ganttWidth * percent; + + const startOffset = getDuration(ganttStartDate, taskFail?.startDate); + const offsetLeft = (startOffset / runDuration) * ganttWidth; + + return ( + + Task Fail + {taskFail?.startDate && ( + + Start: + )} + {taskFail?.endDate && ( + + End: + )} + + Can only show previous Task Fails, other tries are not yet saved. + + + } + hasArrow + portalProps={{ containerRef }} + placement="top" + openDelay={hoverDelay} + top="4px" + > + + + + + ); +}; + +export default TaskFail;