From d3bab834ae0c27620300ce084d0eb5fbe7bb503b Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Thu, 18 Jul 2024 13:50:39 -0400 Subject: [PATCH 1/3] Update Datasets page UX --- airflow/www/jest-setup.js | 2 + airflow/www/static/js/api/useTaskInstance.ts | 13 +- .../details => components}/BreadcrumbText.tsx | 0 .../InstanceTooltip.test.tsx | 0 .../{dag => components}/InstanceTooltip.tsx | 68 ++-- .../static/js/components/Table/CardList.tsx | 225 +++++++++++++ airflow/www/static/js/context/autorefresh.tsx | 19 +- airflow/www/static/js/dag/StatusBox.tsx | 2 +- airflow/www/static/js/dag/details/Header.tsx | 3 +- .../static/js/dag/details/graph/DagNode.tsx | 2 +- airflow/www/static/js/dag/details/index.tsx | 4 +- .../js/dag/details/taskInstance/index.tsx | 4 +- .../www/static/js/datasets/DatasetDetails.tsx | 137 ++++++++ .../static/js/datasets/DatasetEventCard.tsx | 144 +++++++++ .../js/datasets/DatasetEventDetails.tsx | 55 ++++ .../www/static/js/datasets/DatasetEvents.tsx | 104 ++++-- .../{List.test.tsx => DatasetsList.test.tsx} | 6 +- .../datasets/{List.tsx => DatasetsList.tsx} | 42 +-- airflow/www/static/js/datasets/Details.tsx | 97 ------ airflow/www/static/js/datasets/Extra.tsx | 58 ++++ airflow/www/static/js/datasets/Graph/Node.tsx | 4 +- .../www/static/js/datasets/Graph/index.tsx | 15 +- airflow/www/static/js/datasets/Main.tsx | 302 ++++++++++++------ airflow/www/static/js/datasets/SearchBar.tsx | 6 +- .../static/js/datasets/SourceTaskInstance.tsx | 73 +++++ airflow/www/static/js/datasets/types.ts | 24 ++ airflow/www/templates/airflow/datasets.html | 2 + airflow/www/views.py | 1 + 28 files changed, 1094 insertions(+), 318 deletions(-) rename airflow/www/static/js/{dag/details => components}/BreadcrumbText.tsx (100%) rename airflow/www/static/js/{dag => components}/InstanceTooltip.test.tsx (100%) rename airflow/www/static/js/{dag => components}/InstanceTooltip.tsx (70%) create mode 100644 airflow/www/static/js/components/Table/CardList.tsx create mode 100644 airflow/www/static/js/datasets/DatasetDetails.tsx create mode 100644 airflow/www/static/js/datasets/DatasetEventCard.tsx create mode 100644 airflow/www/static/js/datasets/DatasetEventDetails.tsx rename airflow/www/static/js/datasets/{List.test.tsx => DatasetsList.test.tsx} (96%) rename airflow/www/static/js/datasets/{List.tsx => DatasetsList.tsx} (85%) delete mode 100644 airflow/www/static/js/datasets/Details.tsx create mode 100644 airflow/www/static/js/datasets/Extra.tsx create mode 100644 airflow/www/static/js/datasets/SourceTaskInstance.tsx create mode 100644 airflow/www/static/js/datasets/types.ts diff --git a/airflow/www/jest-setup.js b/airflow/www/jest-setup.js index f5a269cfa2a5d..ecf79db5cb19f 100644 --- a/airflow/www/jest-setup.js +++ b/airflow/www/jest-setup.js @@ -67,3 +67,5 @@ global.filtersOptions = { global.moment = moment; global.standaloneDagProcessor = true; + +global.autoRefreshInterval = undefined; diff --git a/airflow/www/static/js/api/useTaskInstance.ts b/airflow/www/static/js/api/useTaskInstance.ts index 84d44ca02dc07..8e7c4faf32425 100644 --- a/airflow/www/static/js/api/useTaskInstance.ts +++ b/airflow/www/static/js/api/useTaskInstance.ts @@ -19,7 +19,7 @@ import axios, { AxiosResponse } from "axios"; import type { API } from "src/types"; -import { useQuery } from "react-query"; +import { useQuery, UseQueryOptions } from "react-query"; import { useAutoRefresh } from "src/context/autorefresh"; import { getMetaValue } from "src/utils"; @@ -29,7 +29,7 @@ const taskInstanceApi = getMetaValue("task_instance_api"); interface Props extends SetOptional { - enabled?: boolean; + options?: UseQueryOptions; } const useTaskInstance = ({ @@ -37,13 +37,14 @@ const useTaskInstance = ({ dagRunId, taskId, mapIndex, - enabled, + options, }: Props) => { let url: string = ""; if (taskInstanceApi) { url = taskInstanceApi + .replace("_DAG_ID_", dagId) .replace("_DAG_RUN_ID_", dagRunId) - .replace("_TASK_ID_", taskId || ""); + .replace("_TASK_ID_", taskId); } if (mapIndex !== undefined && mapIndex >= 0) { @@ -52,12 +53,12 @@ const useTaskInstance = ({ const { isRefreshOn } = useAutoRefresh(); - return useQuery( + return useQuery( ["taskInstance", dagId, dagRunId, taskId, mapIndex], () => axios.get(url), { refetchInterval: isRefreshOn && (autoRefreshInterval || 1) * 1000, - enabled, + ...options, } ); }; diff --git a/airflow/www/static/js/dag/details/BreadcrumbText.tsx b/airflow/www/static/js/components/BreadcrumbText.tsx similarity index 100% rename from airflow/www/static/js/dag/details/BreadcrumbText.tsx rename to airflow/www/static/js/components/BreadcrumbText.tsx diff --git a/airflow/www/static/js/dag/InstanceTooltip.test.tsx b/airflow/www/static/js/components/InstanceTooltip.test.tsx similarity index 100% rename from airflow/www/static/js/dag/InstanceTooltip.test.tsx rename to airflow/www/static/js/components/InstanceTooltip.test.tsx diff --git a/airflow/www/static/js/dag/InstanceTooltip.tsx b/airflow/www/static/js/components/InstanceTooltip.tsx similarity index 70% rename from airflow/www/static/js/dag/InstanceTooltip.tsx rename to airflow/www/static/js/components/InstanceTooltip.tsx index cfdd0be13ec83..f7d83f347c022 100644 --- a/airflow/www/static/js/dag/InstanceTooltip.tsx +++ b/airflow/www/static/js/components/InstanceTooltip.tsx @@ -26,9 +26,22 @@ import { formatDuration, getDuration } from "src/datetime_utils"; import type { TaskInstance, Task } from "src/types"; import Time from "src/components/Time"; +type Instance = Pick< + TaskInstance, + | "taskId" + | "startDate" + | "endDate" + | "state" + | "runId" + | "mappedStates" + | "note" + | "tryNumber" +>; + interface Props { - group: Task; - instance: TaskInstance; + group?: Task; + instance: Instance; + dagId?: string; } const InstanceTooltip = ({ @@ -43,38 +56,43 @@ const InstanceTooltip = ({ note, tryNumber, }, + dagId, }: Props) => { - if (!group) return null; - const isGroup = !!group.children; - const { isMapped } = group; + const isGroup = !!group?.children; + const isMapped = !!group?.isMapped; const summary: React.ReactNode[] = []; - const { totalTasks, childTaskMap } = getGroupAndMapSummary({ - group, - runId, - mappedStates, - }); + let totalTasks = 1; + if (group) { + const { totalTasks: total, childTaskMap } = getGroupAndMapSummary({ + group, + runId, + mappedStates, + }); + totalTasks = total; - childTaskMap.forEach((key, val) => { - const childState = snakeCase(val); - if (key > 0) { - summary.push( - - {childState} - {": "} - {key} - - ); - } - }); + childTaskMap.forEach((key, val) => { + const childState = snakeCase(val); + if (key > 0) { + summary.push( + + {childState} + {": "} + {key} + + ); + } + }); + } return ( + {!!dagId && DAG Id: {dagId}} Task Id: {taskId} - {!!group.setupTeardownType && ( + {!!group?.setupTeardownType && ( Type: {group.setupTeardownType} )} - {group.tooltip && {group.tooltip}} + {group?.tooltip && {group.tooltip}} {isMapped && totalTasks > 0 && ( {totalTasks} mapped task @@ -103,7 +121,7 @@ const InstanceTooltip = ({ )} {tryNumber && tryNumber > 1 && Try Number: {tryNumber}} - {group.triggerRule && Trigger Rule: {group.triggerRule}} + {group?.triggerRule && Trigger Rule: {group.triggerRule}} {note && Contains a note} ); diff --git a/airflow/www/static/js/components/Table/CardList.tsx b/airflow/www/static/js/components/Table/CardList.tsx new file mode 100644 index 0000000000000..222113327b9c4 --- /dev/null +++ b/airflow/www/static/js/components/Table/CardList.tsx @@ -0,0 +1,225 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Custom wrapper of react-table using Chakra UI components + */ + +import React, { useEffect } from "react"; +import { + Flex, + IconButton, + Text, + SimpleGrid, + Box, + Progress, + Skeleton, + BoxProps, + SimpleGridProps, +} from "@chakra-ui/react"; +import { + useTable, + useSortBy, + usePagination, + useRowSelect, + Column, + SortingRule, + Row, +} from "react-table"; +import { MdKeyboardArrowLeft, MdKeyboardArrowRight } from "react-icons/md"; +import { flexRender } from "@tanstack/react-table"; + +export interface CardDef { + card: (props: { row: TData }) => any; + gridProps?: SimpleGridProps; + meta?: { + customSkeleton?: JSX.Element; + }; +} + +interface TableProps extends BoxProps { + data: object[]; + columns: Column[]; + manualPagination?: { + totalEntries: number; + offset: number; + setOffset: (offset: number) => void; + }; + manualSort?: { + sortBy: SortingRule[]; + setSortBy: (sortBy: SortingRule[]) => void; + initialSortBy?: SortingRule[]; + }; + pageSize?: number; + isLoading?: boolean; + selectRows?: (selectedRows: number[]) => void; + onRowClicked?: (row: Row, e: unknown) => void; + cardDef: CardDef; +} + +export const CardList = ({ + data, + cardDef, + columns, + manualPagination, + manualSort, + pageSize = 25, + isLoading = false, + selectRows, + onRowClicked, + ...otherProps +}: TableProps) => { + const { totalEntries, offset, setOffset } = manualPagination || {}; + + const pageCount = totalEntries + ? Math.ceil(totalEntries / pageSize) || 1 + : data.length; + + const lowerCount = (offset || 0) + 1; + const upperCount = lowerCount + data.length - 1; + + // Don't show row selection if selectRows doesn't exist + const selectProps = selectRows ? [useRowSelect] : []; + + const { + prepareRow, + page, + canPreviousPage, + canNextPage, + nextPage, + previousPage, + selectedFlatRows, + state: { pageIndex, sortBy, selectedRowIds }, + } = useTable( + { + columns, + data, + pageCount, + manualPagination: !!manualPagination, + manualSortBy: !!manualSort, + disableMultiSort: !!manualSort, // API only supporting ordering by a single column + initialState: { + pageIndex: offset ? offset / pageSize : 0, + pageSize, + sortBy: manualSort?.initialSortBy || [], + }, + }, + useSortBy, + usePagination, + ...selectProps + ); + + const handleNext = () => { + nextPage(); + if (setOffset) setOffset((pageIndex + 1) * pageSize); + }; + + const handlePrevious = () => { + previousPage(); + if (setOffset) setOffset((pageIndex - 1 || 0) * pageSize); + }; + + // When the sortBy state changes we need to manually call setSortBy + useEffect(() => { + if (manualSort) { + manualSort.setSortBy(sortBy); + } + }, [sortBy, manualSort]); + + useEffect(() => { + if (selectRows) { + // @ts-ignore + selectRows(selectedFlatRows.map((row) => row.original.mapIndex)); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedRowIds, selectRows]); + + const defaultGridProps = { column: { base: 1 }, spacing: 0 }; + + return ( + <> + + + {!isLoading && !page.length && ( + No data found + )} + + {page.map((row) => { + prepareRow(row); + return ( + onRowClicked(row, e) + : undefined + } + > + {isLoading && ( + + )} + {!isLoading && + flexRender(cardDef.card, { + row: row.original as unknown as TData, + })} + + ); + })} + + + {(canPreviousPage || canNextPage) && ( + + } + /> + } + /> + + {lowerCount}-{upperCount} + {" of "} + {totalEntries || data.length} + + + )} + + ); +}; + +export * from "./Cells"; diff --git a/airflow/www/static/js/context/autorefresh.tsx b/airflow/www/static/js/context/autorefresh.tsx index b1ad380cb91b1..45f84068a4b69 100644 --- a/airflow/www/static/js/context/autorefresh.tsx +++ b/airflow/www/static/js/context/autorefresh.tsx @@ -17,7 +17,7 @@ * under the License. */ -/* global localStorage, document */ +/* global localStorage, document, autoRefreshInterval */ import React, { useMemo, @@ -37,9 +37,19 @@ const isRefreshDisabled = JSON.parse( localStorage.getItem(autoRefreshKey) || "false" ); -const AutoRefreshContext = React.createContext({ +type RefreshContext = { + isRefreshOn: boolean; + isPaused: boolean; + refetchInterval: number | false; + toggleRefresh: () => void; + stopRefresh: () => void; + startRefresh: () => void; +}; + +const AutoRefreshContext = React.createContext({ isRefreshOn: false, isPaused: true, + refetchInterval: false, toggleRefresh: () => {}, stopRefresh: () => {}, startRefresh: () => {}, @@ -60,6 +70,8 @@ export const AutoRefreshProvider = ({ children }: PropsWithChildren) => { [isRefreshAllowed, setRefresh] ); + const refetchInterval = isRefreshOn && (autoRefreshInterval || 1) * 1000; + const toggleRefresh = useCallback( (updateStorage = false) => { if (updateStorage) { @@ -99,12 +111,13 @@ export const AutoRefreshProvider = ({ children }: PropsWithChildren) => { const value = useMemo( () => ({ isRefreshOn, + refetchInterval, toggleRefresh, stopRefresh, startRefresh, isPaused, }), - [isPaused, isRefreshOn, startRefresh, toggleRefresh] + [isPaused, isRefreshOn, startRefresh, toggleRefresh, refetchInterval] ); return ( diff --git a/airflow/www/static/js/dag/StatusBox.tsx b/airflow/www/static/js/dag/StatusBox.tsx index 85a84dc133a87..06bcb7ed7c2ea 100644 --- a/airflow/www/static/js/dag/StatusBox.tsx +++ b/airflow/www/static/js/dag/StatusBox.tsx @@ -27,7 +27,7 @@ import type { SelectionProps } from "src/dag/useSelection"; import { getStatusBackgroundColor, hoverDelay } from "src/utils"; import Tooltip from "src/components/Tooltip"; -import InstanceTooltip from "./InstanceTooltip"; +import InstanceTooltip from "src/components/InstanceTooltip"; export const boxSize = 10; export const boxSizePx = `${boxSize}px`; diff --git a/airflow/www/static/js/dag/details/Header.tsx b/airflow/www/static/js/dag/details/Header.tsx index 0e1ff59ec8966..172443ca47b36 100644 --- a/airflow/www/static/js/dag/details/Header.tsx +++ b/airflow/www/static/js/dag/details/Header.tsx @@ -30,8 +30,7 @@ import useSelection from "src/dag/useSelection"; import Time from "src/components/Time"; import { useGridData } from "src/api"; import RunTypeIcon from "src/components/RunTypeIcon"; - -import BreadcrumbText from "./BreadcrumbText"; +import BreadcrumbText from "src/components/BreadcrumbText"; const dagDisplayName = getMetaValue("dag_display_name"); diff --git a/airflow/www/static/js/dag/details/graph/DagNode.tsx b/airflow/www/static/js/dag/details/graph/DagNode.tsx index 7a448da88058c..c2f9b01296c35 100644 --- a/airflow/www/static/js/dag/details/graph/DagNode.tsx +++ b/airflow/www/static/js/dag/details/graph/DagNode.tsx @@ -25,7 +25,7 @@ import { SimpleStatus } from "src/dag/StatusBox"; import useSelection from "src/dag/useSelection"; import { getGroupAndMapSummary, hoverDelay } from "src/utils"; import Tooltip from "src/components/Tooltip"; -import InstanceTooltip from "src/dag/InstanceTooltip"; +import InstanceTooltip from "src/components/InstanceTooltip"; import { useContainerRef } from "src/context/containerRef"; import TaskName from "src/dag/TaskName"; diff --git a/airflow/www/static/js/dag/details/index.tsx b/airflow/www/static/js/dag/details/index.tsx index c76744df07e32..99e25462b85b4 100644 --- a/airflow/www/static/js/dag/details/index.tsx +++ b/airflow/www/static/js/dag/details/index.tsx @@ -235,7 +235,9 @@ const Details = ({ dagRunId: runId || "", taskId: taskId || "", mapIndex, - enabled: mapIndex !== undefined, + options: { + enabled: mapIndex !== undefined, + }, }); const instance = diff --git a/airflow/www/static/js/dag/details/taskInstance/index.tsx b/airflow/www/static/js/dag/details/taskInstance/index.tsx index f33f3e23adbf0..74f317867aea3 100644 --- a/airflow/www/static/js/dag/details/taskInstance/index.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/index.tsx @@ -65,7 +65,9 @@ const TaskInstance = ({ taskId, runId, mapIndex }: Props) => { dagRunId: runId, taskId, mapIndex, - enabled: (!isGroup && !isMapped) || isMapIndexDefined, + options: { + enabled: (!isGroup && !isMapped) || isMapIndexDefined, + }, }); const showTaskSchedulingDependencies = diff --git a/airflow/www/static/js/datasets/DatasetDetails.tsx b/airflow/www/static/js/datasets/DatasetDetails.tsx new file mode 100644 index 0000000000000..8e489e6f7ac95 --- /dev/null +++ b/airflow/www/static/js/datasets/DatasetDetails.tsx @@ -0,0 +1,137 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from "react"; +import { + Spinner, + Flex, + IconButton, + useDisclosure, + Grid, + GridItem, + Heading, + Link, + Box, +} from "@chakra-ui/react"; +import { MdPlayArrow } from "react-icons/md"; + +import { useDataset } from "src/api"; +import { useContainerRef } from "src/context/containerRef"; +import Tooltip from "src/components/Tooltip"; +import { getMetaValue } from "src/utils"; + +import CreateDatasetEventModal from "./CreateDatasetEvent"; +import Events from "./DatasetEvents"; +import Extra from "./Extra"; + +const gridUrl = getMetaValue("grid_url"); + +interface Props { + uri: string; + // onSelect: (props: OnSelectProps) => void; +} + +const DatasetDetails = ({ uri }: Props) => { + const { data: dataset, isLoading } = useDataset({ uri }); + const { isOpen, onToggle, onClose } = useDisclosure(); + const containerRef = useContainerRef(); + + const hasProducingTasks = !!dataset?.producingTasks?.length; + const hasConsumingDags = !!dataset?.consumingDags?.length; + + return ( + + {isLoading && } + + {hasProducingTasks && ( + + Tasks that update this Dataset + {dataset?.producingTasks?.map((task) => { + if (!task.taskId || !task.dagId) return null; + const url = `${gridUrl?.replace( + "__DAG_ID__", + task.dagId + )}?&task_id=${encodeURIComponent(task.taskId)}`; + return ( + + {task.dagId}.{task.taskId} + + ); + })} + + )} + {hasConsumingDags && ( + + DAGs that consume this Dataset + {dataset?.consumingDags?.map((dag) => { + if (!dag.dagId) return null; + const url = gridUrl?.replace("__DAG_ID__", dag.dagId); + return ( + + {dag.dagId} + + ); + })} + + )} + + + + + + + + + + + + + {dataset && dataset.id && } + + {dataset && ( + + )} + + ); +}; + +export default DatasetDetails; diff --git a/airflow/www/static/js/datasets/DatasetEventCard.tsx b/airflow/www/static/js/datasets/DatasetEventCard.tsx new file mode 100644 index 0000000000000..bdd9e374100a9 --- /dev/null +++ b/airflow/www/static/js/datasets/DatasetEventCard.tsx @@ -0,0 +1,144 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from "react"; +import { isEmpty } from "lodash"; +import { TbApi } from "react-icons/tb"; + +import type { DatasetEvent } from "src/types/api-generated"; +import { Box, Flex, Tooltip, Text, Grid, GridItem } from "@chakra-ui/react"; +import { HiDatabase } from "react-icons/hi"; + +import Time from "src/components/Time"; +import { useContainerRef } from "src/context/containerRef"; +import { SimpleStatus } from "src/dag/StatusBox"; +import { formatDuration, getDuration } from "src/datetime_utils"; + +import SourceTaskInstance from "./SourceTaskInstance"; +import Extra from "./Extra"; + +type CardProps = { + datasetEvent: DatasetEvent; +}; + +const MAX_RUNS = 9; + +const DatasetEventCard = ({ datasetEvent }: CardProps) => { + const containerRef = useContainerRef(); + + const { fromRestApi, ...extra } = datasetEvent?.extra as Record< + string, + string + >; + + return ( + + + + + Source: + {fromRestApi && ( + + + + + + )} + {!!datasetEvent.sourceTaskId && ( + + )} + + + {!!datasetEvent?.createdDagruns?.length && ( + <> + Triggered Dag Runs: + + {(datasetEvent?.createdDagruns || []) + ?.slice(0, MAX_RUNS) + .map((run) => ( + + DAG Id: {run.dagId} + Status: {run.state || "no status"} + + Duration:{" "} + {formatDuration( + getDuration(run.startDate, run.endDate) + )} + + + Start Date: + {run.endDate && ( + + End Date: + )} + + } + portalProps={{ containerRef }} + hasArrow + placement="top" + > + + + + + ))} + {datasetEvent?.createdDagruns.length > MAX_RUNS && ( + + +{(datasetEvent.createdDagruns.length || 0) - MAX_RUNS} + + )} + + + )} + + {!isEmpty(extra) && ( + + + + )} + + ); +}; + +export default DatasetEventCard; diff --git a/airflow/www/static/js/datasets/DatasetEventDetails.tsx b/airflow/www/static/js/datasets/DatasetEventDetails.tsx new file mode 100644 index 0000000000000..b1ec5748053dd --- /dev/null +++ b/airflow/www/static/js/datasets/DatasetEventDetails.tsx @@ -0,0 +1,55 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React, { useRef } from "react"; +import { Box } from "@chakra-ui/react"; + +import { useOffsetTop } from "src/utils"; + +import Graph from "./Graph"; + +interface Props { + uri: string; + timestamp: string; +} + +const DatasetEventDetails = ({ uri, timestamp }: Props) => { + console.log(timestamp); + const graphRef = useRef(null); + const contentRef = useRef(null); + const offsetTop = useOffsetTop(contentRef); + const height = `calc(100vh - ${offsetTop + 80}px)`; + return ( + + + + + t + + ); +}; + +export default DatasetEventDetails; diff --git a/airflow/www/static/js/datasets/DatasetEvents.tsx b/airflow/www/static/js/datasets/DatasetEvents.tsx index b58006ac8193c..255c11d3a99ad 100644 --- a/airflow/www/static/js/datasets/DatasetEvents.tsx +++ b/airflow/www/static/js/datasets/DatasetEvents.tsx @@ -20,17 +20,26 @@ import React, { useMemo, useState } from "react"; import { snakeCase } from "lodash"; import type { SortingRule } from "react-table"; +import { Box, Flex, Heading, Select } from "@chakra-ui/react"; import { useDatasetEvents } from "src/api"; -import { - Table, - TimeCell, - TaskInstanceLink, - TriggeredRuns, - CodeCell, -} from "src/components/Table"; -const Events = ({ datasetId }: { datasetId: number }) => { +import { CardList, type CardDef } from "src/components/Table/CardList"; +import type { DatasetEvent } from "src/types/api-generated"; +import DatasetEventCard from "./DatasetEventCard"; +// import type { OnSelectProps } from "./types"; + +type Props = { + datasetId?: number; + // onSelect?: (props: OnSelectProps) => void; + showLabel?: boolean; +}; + +const cardDef: CardDef = { + card: ({ row }) => , +}; + +const Events = ({ datasetId, showLabel }: Props) => { const limit = 25; const [offset, setOffset] = useState(0); const [sortBy, setSortBy] = useState[]>([ @@ -52,26 +61,25 @@ const Events = ({ datasetId }: { datasetId: number }) => { const columns = useMemo( () => [ - { - Header: "Source Task Instance", - accessor: "sourceTaskId", - Cell: TaskInstanceLink, - }, { Header: "When", accessor: "timestamp", - Cell: TimeCell, + }, + { + Header: "Dataset", + accessor: "datasetUri", + }, + { + Header: "Source Task Instance", + accessor: "sourceTaskId", }, { Header: "Triggered Runs", accessor: "createdDagruns", - Cell: TriggeredRuns, }, { Header: "Extra", accessor: "extra", - Cell: CodeCell, - disableSortBy: true, }, ], [] @@ -79,25 +87,51 @@ const Events = ({ datasetId }: { datasetId: number }) => { const data = useMemo(() => datasetEvents, [datasetEvents]); - const memoSort = useMemo(() => sortBy, [sortBy]); - return ( - + + + {showLabel && "Events"} + + Sort: + + + + { + // if (onSelect) + // onSelect({ + // uri: datasetUri, + // timestamp, + // }); + // }} + /> + ); }; diff --git a/airflow/www/static/js/datasets/List.test.tsx b/airflow/www/static/js/datasets/DatasetsList.test.tsx similarity index 96% rename from airflow/www/static/js/datasets/List.test.tsx rename to airflow/www/static/js/datasets/DatasetsList.test.tsx index c445cc2d68d8f..89d0e9f490573 100644 --- a/airflow/www/static/js/datasets/List.test.tsx +++ b/airflow/www/static/js/datasets/DatasetsList.test.tsx @@ -27,7 +27,7 @@ import { Wrapper } from "src/utils/testUtils"; import type { UseQueryResult } from "react-query"; import type { DatasetListItem } from "src/types"; -import DatasetsList from "./List"; +import DatasetsList from "./DatasetsList"; const datasets = [ { @@ -87,7 +87,7 @@ describe("Test Datasets List", () => { .mockImplementation(() => returnValue); const { getByText, queryAllByTestId } = render( - {}} />, + {}} />, { wrapper: Wrapper } ); @@ -111,7 +111,7 @@ describe("Test Datasets List", () => { .mockImplementation(() => emptyReturnValue); const { getByText, queryAllByTestId, getByTestId } = render( - {}} />, + {}} />, { wrapper: Wrapper } ); diff --git a/airflow/www/static/js/datasets/List.tsx b/airflow/www/static/js/datasets/DatasetsList.tsx similarity index 85% rename from airflow/www/static/js/datasets/List.tsx rename to airflow/www/static/js/datasets/DatasetsList.tsx index b8753dfcd76b9..b2050155764e6 100644 --- a/airflow/www/static/js/datasets/List.tsx +++ b/airflow/www/static/js/datasets/DatasetsList.tsx @@ -18,15 +18,7 @@ */ import React, { useMemo, useState } from "react"; -import { - Box, - Heading, - Flex, - Text, - Link, - ButtonGroup, - Button, -} from "@chakra-ui/react"; +import { Box, Flex, Text, Link, ButtonGroup, Button } from "@chakra-ui/react"; import { snakeCase } from "lodash"; import type { Row, SortingRule } from "react-table"; import { useSearchParams } from "react-router-dom"; @@ -36,14 +28,10 @@ import { Table, TimeCell } from "src/components/Table"; import type { API } from "src/types"; import { getMetaValue } from "src/utils"; import type { DateOption } from "src/api/useDatasetsSummary"; -import type { DatasetDependencies } from "src/api/useDatasetDependencies"; -import SearchBar from "./SearchBar"; +import type { OnSelectProps } from "./types"; interface Props { - datasetDependencies?: DatasetDependencies; - selectedDagId?: string; - selectedUri?: string; - onSelectNode: (id: string, type: string) => void; + onSelect: (props: OnSelectProps) => void; } interface CellProps { @@ -78,12 +66,7 @@ const dateOptions: Record = { hour: { count: 1, unit: "hour" }, }; -const DatasetsList = ({ - datasetDependencies, - onSelectNode, - selectedDagId, - selectedUri, -}: Props) => { +const DatasetsList = ({ onSelect }: Props) => { const limit = 25; const [offset, setOffset] = useState(0); @@ -128,18 +111,13 @@ const DatasetsList = ({ const memoSort = useMemo(() => sortBy, [sortBy]); const onDatasetSelect = (row: Row) => { - if (row.original.uri) onSelectNode(row.original.uri, "dataset"); + if (row.original.uri) onSelect({ uri: row.original.uri }); }; const docsUrl = getMetaValue("datasets_docs"); return ( - - - - Datasets - - + <> {!datasets.length && !isLoading && !dateFilter && ( Looks like you do not have any datasets yet. Check out the{" "} @@ -185,12 +163,6 @@ const DatasetsList = ({ })} -
- + ); }; diff --git a/airflow/www/static/js/datasets/Details.tsx b/airflow/www/static/js/datasets/Details.tsx deleted file mode 100644 index cb0f83fd459dc..0000000000000 --- a/airflow/www/static/js/datasets/Details.tsx +++ /dev/null @@ -1,97 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import React from "react"; -import { - Box, - Heading, - Flex, - Spinner, - Button, - IconButton, - useDisclosure, -} from "@chakra-ui/react"; -import { MdPlayArrow } from "react-icons/md"; - -import { useDataset } from "src/api"; -import { ClipboardButton } from "src/components/Clipboard"; -import InfoTooltip from "src/components/InfoTooltip"; -import { useContainerRef } from "src/context/containerRef"; -import Tooltip from "src/components/Tooltip"; - -import CreateDatasetEventModal from "./CreateDatasetEvent"; -import Events from "./DatasetEvents"; - -interface Props { - uri: string; - onBack: () => void; -} - -const DatasetDetails = ({ uri, onBack }: Props) => { - const { data: dataset, isLoading } = useDataset({ uri }); - const { isOpen, onToggle, onClose } = useDisclosure(); - const containerRef = useContainerRef(); - return ( - - - - - - - - - - {isLoading && } - - - Dataset: {uri} - - - - - - History - - - - {dataset && dataset.id && } - {dataset && ( - - )} - - ); -}; - -export default DatasetDetails; diff --git a/airflow/www/static/js/datasets/Extra.tsx b/airflow/www/static/js/datasets/Extra.tsx new file mode 100644 index 0000000000000..b20be1d16e456 --- /dev/null +++ b/airflow/www/static/js/datasets/Extra.tsx @@ -0,0 +1,58 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from "react"; +import { + TableContainer, + Table, + Thead, + Tbody, + Tr, + Td, + Th, +} from "@chakra-ui/react"; +import type { Dataset, DatasetEvent } from "src/types/api-generated"; + +type Props = { + extra: Dataset["extra"] | DatasetEvent["extra"]; +}; + +const Extra = ({ extra }: Props) => + extra ? ( + +
+ + + + + + + + {Object.entries(extra).map(([key, value]) => ( + + + + + ))} + +
KeyValue
{key}{JSON.stringify(value)}
+ + ) : null; + +export default Extra; diff --git a/airflow/www/static/js/datasets/Graph/Node.tsx b/airflow/www/static/js/datasets/Graph/Node.tsx index 425f25e3c9e3f..72b5505ffb061 100644 --- a/airflow/www/static/js/datasets/Graph/Node.tsx +++ b/airflow/www/static/js/datasets/Graph/Node.tsx @@ -32,7 +32,7 @@ export interface CustomNodeProps { width?: number; isSelected?: boolean; isHighlighted?: boolean; - onSelect: (datasetUri: string, type: string) => void; + onSelect: () => void; isOpen?: boolean; isActive?: boolean; } @@ -62,7 +62,7 @@ const BaseNode = ({ onClick={(e) => { e.preventDefault(); e.stopPropagation(); - onSelect(label, "dataset"); + onSelect(); }} cursor="pointer" fontSize={16} diff --git a/airflow/www/static/js/datasets/Graph/index.tsx b/airflow/www/static/js/datasets/Graph/index.tsx index 69486031120b9..9157a8a1a2617 100644 --- a/airflow/www/static/js/datasets/Graph/index.tsx +++ b/airflow/www/static/js/datasets/Graph/index.tsx @@ -38,16 +38,17 @@ import { useDatasetGraphs } from "src/api/useDatasetDependencies"; import Node, { CustomNodeProps } from "./Node"; import Legend from "./Legend"; +import type { OnSelectProps } from "../types"; interface Props { - selectedNodeId: string | null; - onSelectNode: (id: string, type: string) => void; + selectedNodeId?: string; + onSelect?: (props: OnSelectProps) => void; } const nodeTypes = { custom: Node }; const edgeTypes = { custom: Edge }; -const Graph = ({ selectedNodeId, onSelectNode }: Props) => { +const Graph = ({ selectedNodeId, onSelect }: Props) => { const { colors } = useTheme(); const { setCenter } = useReactFlow(); const containerRef = useContainerRef(); @@ -84,7 +85,13 @@ const Graph = ({ selectedNodeId, onSelectNode }: Props) => { type: c.value.class, width: c.width, height: c.height, - onSelect: onSelectNode, + onSelect: () => { + if (onSelect) { + if (c.value.class === "dataset") onSelect({ uri: c.value.label }); + else if (c.value.class === "dag") + onSelect({ dagId: c.value.label }); + } + }, isSelected: selectedNodeId === c.value.label, isHighlighted: edges.some( (e) => e.data.rest.isSelected && e.id.includes(c.id) diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index b73c4fca2ec83..c93e34fec475e 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -17,138 +17,240 @@ * under the License. */ -import React, { useCallback, useEffect, useRef } from "react"; +import React, { useCallback, useRef } from "react"; import { useSearchParams } from "react-router-dom"; -import { Flex, Box, Spinner } from "@chakra-ui/react"; +import { + Box, + Breadcrumb, + BreadcrumbLink, + BreadcrumbItem, + Heading, + Tabs, + Spinner, + Tab, + TabList, + TabPanel, + TabPanels, + Text, +} from "@chakra-ui/react"; +import { HiDatabase } from "react-icons/hi"; +import { MdEvent, MdAccountTree, MdDetails } from "react-icons/md"; +import Time from "src/components/Time"; +import BreadcrumbText from "src/components/BreadcrumbText"; import { useOffsetTop } from "src/utils"; import { useDatasetDependencies } from "src/api"; +import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; -import DatasetsList from "./List"; -import DatasetDetails from "./Details"; +import DatasetEvents from "./DatasetEvents"; +import DatasetsList from "./DatasetsList"; +import DatasetDetails from "./DatasetDetails"; +// import DatasetEventDetails from "./DatasetEventDetails"; +import type { OnSelectProps } from "./types"; import Graph from "./Graph"; +import SearchBar from "./SearchBar"; const DATASET_URI_PARAM = "uri"; const DAG_ID_PARAM = "dag_id"; -const minPanelWidth = 300; +const TIMESTAMP_PARAM = "timestamp"; +const TAB_PARAM = "tab"; + +const tabToIndex = (tab?: string) => { + switch (tab) { + case "graph": + return 1; + case "datasets": + return 2; + case "details": + case "events": + default: + return 0; + } +}; + +const indexToTab = (index: number, uri?: string) => { + switch (index) { + case 0: + return uri ? "details" : "events"; + case 1: + return "graph"; + case 2: + if (!uri) return "datasets"; + return undefined; + default: + return undefined; + } +}; const Datasets = () => { - const [searchParams, setSearchParams] = useSearchParams(); const contentRef = useRef(null); const offsetTop = useOffsetTop(contentRef); - const listRef = useRef(null); - const graphRef = useRef(null); - // 60px for footer height - const height = `calc(100vh - ${offsetTop + 60}px)`; + const height = `calc(100vh - ${offsetTop + 100}px)`; - const resizeRef = useRef(null); + const { data: datasetDependencies, isLoading } = useDatasetDependencies(); + const [searchParams, setSearchParams] = useSearchParams(); const selectedUri = decodeURIComponent( searchParams.get(DATASET_URI_PARAM) || "" ); - const selectedDagId = searchParams.get(DAG_ID_PARAM); - // We need to load in the raw dependencies in order to generate the list of dagIds - const { data: datasetDependencies, isLoading } = useDatasetDependencies(); - - const resize = useCallback( - (e: MouseEvent) => { - const listEl = listRef.current; - if ( - listEl && - e.x > minPanelWidth && - e.x < window.innerWidth - minPanelWidth - ) { - const width = `${e.x}px`; - listEl.style.width = width; - } - }, - [listRef] + const selectedTimestamp = decodeURIComponent( + searchParams.get(TIMESTAMP_PARAM) || "" ); + const selectedDagId = searchParams.get(DAG_ID_PARAM) || undefined; - useEffect(() => { - const resizeEl = resizeRef.current; - if (resizeEl) { - resizeEl.addEventListener("mousedown", (e) => { - e.preventDefault(); - document.addEventListener("mousemove", resize); - }); - - document.addEventListener("mouseup", () => { - document.removeEventListener("mousemove", resize); - }); - - return () => { - resizeEl?.removeEventListener("mousedown", resize); - document.removeEventListener("mouseup", resize); - }; - } - return () => {}; - }, [resize]); + const tab = searchParams.get(TAB_PARAM) || undefined; + const tabIndex = tabToIndex(tab); - const selectedNodeId = selectedUri || selectedDagId; + const onChangeTab = useCallback( + (index: number) => { + const params = new URLSearchParamsWrapper(searchParams); + const newTab = indexToTab(index, selectedUri); + if (newTab) params.set(TAB_PARAM, newTab); + else params.delete(TAB_PARAM); + setSearchParams(params); + }, + [setSearchParams, searchParams, selectedUri] + ); - const onSelectNode = (id: string, type: string) => { - if (type === "dag") { - if (id === selectedDagId) searchParams.delete(DAG_ID_PARAM); - else searchParams.set(DAG_ID_PARAM, id); + const onSelect = ({ uri, timestamp, dagId }: OnSelectProps = {}) => { + if (dagId) { + if (dagId === selectedDagId) searchParams.delete(DAG_ID_PARAM); + searchParams.set(DAG_ID_PARAM, dagId); searchParams.delete(DATASET_URI_PARAM); - } - if (type === "dataset") { - if (id === selectedUri) searchParams.delete(DATASET_URI_PARAM); - else searchParams.set(DATASET_URI_PARAM, id); + } else if (uri) { + searchParams.set(DATASET_URI_PARAM, uri); + if (timestamp) searchParams.set(TIMESTAMP_PARAM, timestamp); + else searchParams.delete(TIMESTAMP_PARAM); + searchParams.delete(DAG_ID_PARAM); + if (tab === "datasets") searchParams.delete(TAB_PARAM); + } else { + searchParams.delete(DATASET_URI_PARAM); + searchParams.delete(TIMESTAMP_PARAM); searchParams.delete(DAG_ID_PARAM); } setSearchParams(searchParams); }; return ( - - + + / + + } > - {selectedUri ? ( - onSelectNode(selectedUri, "dataset")} - /> - ) : ( - + + onSelect()} + isCurrentPage={!selectedUri} + > + + Datasets + + + + + {selectedUri && ( + + onSelect({ uri: selectedUri })}> + + + )} - - - - {isLoading && } - - - + + {selectedTimestamp && ( + + + } + /> + + + )} + + + + {!selectedUri && ( + + + + Dataset Events + + + )} + {!!selectedUri && ( + + + + Details + + + )} + + + + Dependency Graph + + + {!selectedUri && ( + + + + Datasets + + + )} + + + {!selectedUri && ( + + + + )} + {!!selectedUri && ( + + + + )} + + {isLoading && } + {/* the graph needs a defined height to render properly */} + + + {height && ( + + )} + + + {!selectedUri && ( + + + + )} + + + ); }; diff --git a/airflow/www/static/js/datasets/SearchBar.tsx b/airflow/www/static/js/datasets/SearchBar.tsx index 4702035bf2a6d..fc47215389a57 100644 --- a/airflow/www/static/js/datasets/SearchBar.tsx +++ b/airflow/www/static/js/datasets/SearchBar.tsx @@ -21,12 +21,13 @@ import React from "react"; import { Select, SingleValue, useChakraSelectProps } from "chakra-react-select"; import type { DatasetDependencies } from "src/api/useDatasetDependencies"; +import type { OnSelectProps } from "./types"; interface Props { datasetDependencies?: DatasetDependencies; selectedDagId?: string; selectedUri?: string; - onSelectNode: (id: string, type: string) => void; + onSelectNode: (props: OnSelectProps) => void; } interface Option { @@ -54,7 +55,8 @@ const SearchBar = ({ if (option) { if (option.value.startsWith("dataset:")) type = "dataset"; else if (option.value.startsWith("dag:")) type = "dag"; - if (type) onSelectNode(option.label, type); + if (type === "dag") onSelectNode({ dagId: option.label }); + else if (type === "dataset") onSelectNode({ uri: option.label }); } }; diff --git a/airflow/www/static/js/datasets/SourceTaskInstance.tsx b/airflow/www/static/js/datasets/SourceTaskInstance.tsx new file mode 100644 index 0000000000000..ed5fe32001174 --- /dev/null +++ b/airflow/www/static/js/datasets/SourceTaskInstance.tsx @@ -0,0 +1,73 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from "react"; +import { Box, Tooltip } from "@chakra-ui/react"; + +import { useTaskInstance } from "src/api"; +import type { DatasetEvent } from "src/types/api-generated"; +import { useContainerRef } from "src/context/containerRef"; +import { SimpleStatus } from "src/dag/StatusBox"; +import InstanceTooltip from "src/components/InstanceTooltip"; +import type { TaskInstance } from "src/types"; + +type SourceTIProps = { + datasetEvent: DatasetEvent; +}; + +const SourceTaskInstance = ({ datasetEvent }: SourceTIProps) => { + const containerRef = useContainerRef(); + const { sourceDagId, sourceRunId, sourceTaskId, sourceMapIndex } = + datasetEvent; + + const { data: taskInstance } = useTaskInstance({ + dagId: sourceDagId || "", + dagRunId: sourceRunId || "", + taskId: sourceTaskId || "", + mapIndex: sourceMapIndex || undefined, + options: { + enabled: !!(sourceDagId && sourceRunId && sourceTaskId), + refetchInterval: false, + }, + }); + + return ( + + {!!taskInstance && ( + + } + portalProps={{ containerRef }} + hasArrow + placement="top" + > + + + + + )} + + ); +}; + +export default SourceTaskInstance; diff --git a/airflow/www/static/js/datasets/types.ts b/airflow/www/static/js/datasets/types.ts new file mode 100644 index 0000000000000..98e7613d14cde --- /dev/null +++ b/airflow/www/static/js/datasets/types.ts @@ -0,0 +1,24 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export type OnSelectProps = { + uri?: string; + timestamp?: string; + dagId?: string; +}; diff --git a/airflow/www/templates/airflow/datasets.html b/airflow/www/templates/airflow/datasets.html index ea9295c05703f..64e08510e4a4f 100644 --- a/airflow/www/templates/airflow/datasets.html +++ b/airflow/www/templates/airflow/datasets.html @@ -30,6 +30,7 @@ + {% endblock %} {% block content %} @@ -44,6 +45,7 @@ {{ super()}} {% endblock %} diff --git a/airflow/www/views.py b/airflow/www/views.py index 6d54cf93c08f4..0e0923aa130ba 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -1161,6 +1161,7 @@ def datasets(self): state_color_mapping["null"] = state_color_mapping.pop(None) return self.render_template( "airflow/datasets.html", + auto_refresh_interval=conf.getint("webserver", "auto_refresh_interval"), state_color_mapping=state_color_mapping, ) From 1035cdb47bc6555424fbaf3b213761590a1fb4fc Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Thu, 18 Jul 2024 14:59:22 -0400 Subject: [PATCH 2/3] Cleanup selected dataset event code --- .../www/static/js/datasets/DatasetDetails.tsx | 1 - .../static/js/datasets/DatasetEventCard.tsx | 2 +- .../js/datasets/DatasetEventDetails.tsx | 55 ------------------- .../www/static/js/datasets/DatasetEvents.tsx | 9 --- .../www/static/js/datasets/DatasetsList.tsx | 1 - airflow/www/static/js/datasets/Main.tsx | 1 - 6 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 airflow/www/static/js/datasets/DatasetEventDetails.tsx diff --git a/airflow/www/static/js/datasets/DatasetDetails.tsx b/airflow/www/static/js/datasets/DatasetDetails.tsx index 8e489e6f7ac95..626fbadca9453 100644 --- a/airflow/www/static/js/datasets/DatasetDetails.tsx +++ b/airflow/www/static/js/datasets/DatasetDetails.tsx @@ -44,7 +44,6 @@ const gridUrl = getMetaValue("grid_url"); interface Props { uri: string; - // onSelect: (props: OnSelectProps) => void; } const DatasetDetails = ({ uri }: Props) => { diff --git a/airflow/www/static/js/datasets/DatasetEventCard.tsx b/airflow/www/static/js/datasets/DatasetEventCard.tsx index bdd9e374100a9..d5377a3bc48e2 100644 --- a/airflow/www/static/js/datasets/DatasetEventCard.tsx +++ b/airflow/www/static/js/datasets/DatasetEventCard.tsx @@ -134,7 +134,7 @@ const DatasetEventCard = ({ datasetEvent }: CardProps) => { {!isEmpty(extra) && ( - + )} diff --git a/airflow/www/static/js/datasets/DatasetEventDetails.tsx b/airflow/www/static/js/datasets/DatasetEventDetails.tsx deleted file mode 100644 index b1ec5748053dd..0000000000000 --- a/airflow/www/static/js/datasets/DatasetEventDetails.tsx +++ /dev/null @@ -1,55 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import React, { useRef } from "react"; -import { Box } from "@chakra-ui/react"; - -import { useOffsetTop } from "src/utils"; - -import Graph from "./Graph"; - -interface Props { - uri: string; - timestamp: string; -} - -const DatasetEventDetails = ({ uri, timestamp }: Props) => { - console.log(timestamp); - const graphRef = useRef(null); - const contentRef = useRef(null); - const offsetTop = useOffsetTop(contentRef); - const height = `calc(100vh - ${offsetTop + 80}px)`; - return ( - - - - - t - - ); -}; - -export default DatasetEventDetails; diff --git a/airflow/www/static/js/datasets/DatasetEvents.tsx b/airflow/www/static/js/datasets/DatasetEvents.tsx index 255c11d3a99ad..26a5a3ae690fc 100644 --- a/airflow/www/static/js/datasets/DatasetEvents.tsx +++ b/airflow/www/static/js/datasets/DatasetEvents.tsx @@ -27,11 +27,9 @@ import { useDatasetEvents } from "src/api"; import { CardList, type CardDef } from "src/components/Table/CardList"; import type { DatasetEvent } from "src/types/api-generated"; import DatasetEventCard from "./DatasetEventCard"; -// import type { OnSelectProps } from "./types"; type Props = { datasetId?: number; - // onSelect?: (props: OnSelectProps) => void; showLabel?: boolean; }; @@ -123,13 +121,6 @@ const Events = ({ datasetId, showLabel }: Props) => { pageSize={limit} isLoading={isEventsLoading} cardDef={cardDef} - // onRowClicked={({ original: { datasetUri, timestamp } }: any) => { - // if (onSelect) - // onSelect({ - // uri: datasetUri, - // timestamp, - // }); - // }} /> ); diff --git a/airflow/www/static/js/datasets/DatasetsList.tsx b/airflow/www/static/js/datasets/DatasetsList.tsx index b2050155764e6..28e8f4b38e048 100644 --- a/airflow/www/static/js/datasets/DatasetsList.tsx +++ b/airflow/www/static/js/datasets/DatasetsList.tsx @@ -87,7 +87,6 @@ const DatasetsList = ({ onSelect }: Props) => { limit, offset, order, - // uri, updatedAfter: dateFilter ? dateOptions[dateFilter] : undefined, }); diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index c93e34fec475e..04380e2665dc7 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -45,7 +45,6 @@ import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; import DatasetEvents from "./DatasetEvents"; import DatasetsList from "./DatasetsList"; import DatasetDetails from "./DatasetDetails"; -// import DatasetEventDetails from "./DatasetEventDetails"; import type { OnSelectProps } from "./types"; import Graph from "./Graph"; import SearchBar from "./SearchBar"; From 4280c7a8fe2a4d2643920acd4dafc518cf7d4528 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Tue, 23 Jul 2024 13:58:00 -0400 Subject: [PATCH 3/3] Use DatasetEventCard everywhere, use RenderedJson for extra, add links --- .../static/js/components/DatasetEventCard.tsx | 183 ++++++++++++++++++ .../js/components/RenderedJsonField.tsx | 20 +- .../SourceTaskInstance.tsx | 28 ++- .../static/js/components/Table/CardList.tsx | 2 - .../static/js/components/Table/Cells.test.tsx | 96 --------- .../www/static/js/components/Table/Cells.tsx | 141 +------------- .../www/static/js/components/Table/index.tsx | 1 + .../details/dagRun/DatasetTriggerEvents.tsx | 36 ++-- .../taskInstance/DatasetUpdateEvents.tsx | 36 ++-- .../details/taskInstance/MappedInstances.tsx | 10 +- .../www/static/js/datasets/DatasetDetails.tsx | 14 +- .../static/js/datasets/DatasetEventCard.tsx | 144 -------------- .../www/static/js/datasets/DatasetEvents.tsx | 4 +- .../www/static/js/datasets/DatasetsList.tsx | 14 +- airflow/www/static/js/datasets/Extra.tsx | 58 ------ airflow/www/templates/airflow/dag.html | 2 +- 16 files changed, 284 insertions(+), 505 deletions(-) create mode 100644 airflow/www/static/js/components/DatasetEventCard.tsx rename airflow/www/static/js/{datasets => components}/SourceTaskInstance.tsx (76%) delete mode 100644 airflow/www/static/js/components/Table/Cells.test.tsx delete mode 100644 airflow/www/static/js/datasets/DatasetEventCard.tsx delete mode 100644 airflow/www/static/js/datasets/Extra.tsx diff --git a/airflow/www/static/js/components/DatasetEventCard.tsx b/airflow/www/static/js/components/DatasetEventCard.tsx new file mode 100644 index 0000000000000..e6c5b6bb9cbad --- /dev/null +++ b/airflow/www/static/js/components/DatasetEventCard.tsx @@ -0,0 +1,183 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from "react"; +import { isEmpty } from "lodash"; +import { TbApi } from "react-icons/tb"; + +import type { DatasetEvent } from "src/types/api-generated"; +import { + Box, + Flex, + Tooltip, + Text, + Grid, + GridItem, + Link, +} from "@chakra-ui/react"; +import { HiDatabase } from "react-icons/hi"; +import { FiLink } from "react-icons/fi"; +import { useSearchParams } from "react-router-dom"; + +import { getMetaValue } from "src/utils"; +import Time from "src/components/Time"; +import { useContainerRef } from "src/context/containerRef"; +import { SimpleStatus } from "src/dag/StatusBox"; +import { formatDuration, getDuration } from "src/datetime_utils"; +import RenderedJsonField from "src/components/RenderedJsonField"; + +import SourceTaskInstance from "./SourceTaskInstance"; + +type CardProps = { + datasetEvent: DatasetEvent; +}; + +const gridUrl = getMetaValue("grid_url"); +const datasetsUrl = getMetaValue("datasets_url"); + +const DatasetEventCard = ({ datasetEvent }: CardProps) => { + const [searchParams] = useSearchParams(); + + const selectedUri = decodeURIComponent(searchParams.get("uri") || ""); + const containerRef = useContainerRef(); + + const { fromRestApi, ...extra } = datasetEvent?.extra as Record< + string, + string + >; + + return ( + + + + + + Source: + {fromRestApi && ( + + + + + + )} + {!!datasetEvent.sourceTaskId && ( + + )} + + + {!!datasetEvent?.createdDagruns?.length && ( + <> + Triggered Dag Runs: + + {datasetEvent?.createdDagruns.map((run) => { + const runId = (run as any).dagRunId; // For some reason the type is wrong here + const url = `${gridUrl?.replace( + "__DAG_ID__", + run.dagId || "" + )}?dag_run_id=${encodeURIComponent(runId)}`; + + return ( + + DAG Id: {run.dagId} + Status: {run.state || "no status"} + + Duration:{" "} + {formatDuration( + getDuration(run.startDate, run.endDate) + )} + + + Start Date: + {run.endDate && ( + + End Date: + )} + + } + portalProps={{ containerRef }} + hasArrow + placement="top" + > + + + + + + + + ); + })} + + + )} + + + {!isEmpty(extra) && ( + + )} + + ); +}; + +export default DatasetEventCard; diff --git a/airflow/www/static/js/components/RenderedJsonField.tsx b/airflow/www/static/js/components/RenderedJsonField.tsx index a5e216f64d127..7000dc17ad3c5 100644 --- a/airflow/www/static/js/components/RenderedJsonField.tsx +++ b/airflow/www/static/js/components/RenderedJsonField.tsx @@ -19,7 +19,7 @@ import React from "react"; -import ReactJson from "react-json-view"; +import ReactJson, { ReactJsonViewProps } from "react-json-view"; import { Flex, @@ -32,15 +32,20 @@ import { } from "@chakra-ui/react"; interface Props extends FlexProps { - content: string; + content: string | object; + jsonProps?: Omit; } -const JsonParse = (content: string) => { +const JsonParse = (content: string | object) => { let contentJson = null; let contentFormatted = ""; let isJson = false; try { - contentJson = JSON.parse(content); + if (typeof content === "string") { + contentJson = JSON.parse(content); + } else { + contentJson = content; + } contentFormatted = JSON.stringify(contentJson, null, 4); isJson = true; } catch (e) { @@ -49,7 +54,7 @@ const JsonParse = (content: string) => { return [isJson, contentJson, contentFormatted]; }; -const RenderedJsonField = ({ content, ...rest }: Props) => { +const RenderedJsonField = ({ content, jsonProps, ...rest }: Props) => { const [isJson, contentJson, contentFormatted] = JsonParse(content); const { onCopy, hasCopied } = useClipboard(contentFormatted); const theme = useTheme(); @@ -69,14 +74,15 @@ const RenderedJsonField = ({ content, ...rest }: Props) => { fontSize: theme.fontSizes.md, font: theme.fonts.mono, }} + {...jsonProps} /> - ) : ( - {content} + {content as string} ); }; diff --git a/airflow/www/static/js/datasets/SourceTaskInstance.tsx b/airflow/www/static/js/components/SourceTaskInstance.tsx similarity index 76% rename from airflow/www/static/js/datasets/SourceTaskInstance.tsx rename to airflow/www/static/js/components/SourceTaskInstance.tsx index ed5fe32001174..3dcae30d32501 100644 --- a/airflow/www/static/js/datasets/SourceTaskInstance.tsx +++ b/airflow/www/static/js/components/SourceTaskInstance.tsx @@ -18,7 +18,8 @@ */ import React from "react"; -import { Box, Tooltip } from "@chakra-ui/react"; +import { Box, Link, Tooltip, Flex } from "@chakra-ui/react"; +import { FiLink } from "react-icons/fi"; import { useTaskInstance } from "src/api"; import type { DatasetEvent } from "src/types/api-generated"; @@ -26,11 +27,14 @@ import { useContainerRef } from "src/context/containerRef"; import { SimpleStatus } from "src/dag/StatusBox"; import InstanceTooltip from "src/components/InstanceTooltip"; import type { TaskInstance } from "src/types"; +import { getMetaValue } from "src/utils"; type SourceTIProps = { datasetEvent: DatasetEvent; }; +const gridUrl = getMetaValue("grid_url"); + const SourceTaskInstance = ({ datasetEvent }: SourceTIProps) => { const containerRef = useContainerRef(); const { sourceDagId, sourceRunId, sourceTaskId, sourceMapIndex } = @@ -47,6 +51,21 @@ const SourceTaskInstance = ({ datasetEvent }: SourceTIProps) => { }, }); + let url = `${gridUrl?.replace( + "__DAG_ID__", + sourceDagId || "" + )}?dag_run_id=${encodeURIComponent( + sourceRunId || "" + )}&task_id=${encodeURIComponent(sourceTaskId || "")}`; + + if ( + sourceMapIndex !== null && + sourceMapIndex !== undefined && + sourceMapIndex > -1 + ) { + url = `${url}&map_index=${sourceMapIndex}`; + } + return ( {!!taskInstance && ( @@ -61,9 +80,12 @@ const SourceTaskInstance = ({ datasetEvent }: SourceTIProps) => { hasArrow placement="top" > - + - + + + + )} diff --git a/airflow/www/static/js/components/Table/CardList.tsx b/airflow/www/static/js/components/Table/CardList.tsx index 222113327b9c4..668566737d616 100644 --- a/airflow/www/static/js/components/Table/CardList.tsx +++ b/airflow/www/static/js/components/Table/CardList.tsx @@ -221,5 +221,3 @@ export const CardList = ({ ); }; - -export * from "./Cells"; diff --git a/airflow/www/static/js/components/Table/Cells.test.tsx b/airflow/www/static/js/components/Table/Cells.test.tsx deleted file mode 100644 index ac1273b3e10c7..0000000000000 --- a/airflow/www/static/js/components/Table/Cells.test.tsx +++ /dev/null @@ -1,96 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* global describe, test, expect */ - -import React from "react"; -import "@testing-library/jest-dom"; -import { render } from "@testing-library/react"; - -import { ChakraWrapper } from "src/utils/testUtils"; -import * as utils from "src/utils"; -import { TaskInstanceLink } from "./Cells"; - -const taskId = "task_id"; -const sourceDagId = "source_dag_id"; -const sourceRunId = "source_run_id"; -const originalDagId = "og_dag_id"; - -describe("Test TaskInstanceLink", () => { - test("Replaces __DAG_ID__ url param correctly", async () => { - jest.spyOn(utils, "getMetaValue").mockImplementation((meta) => { - if (meta === "grid_url") return "/dags/__DAG_ID__/grid"; - return ""; - }); - - const { getByText } = render( - , - { wrapper: ChakraWrapper } - ); - - const link = getByText(`${sourceDagId}.${taskId}`); - expect(link).toBeInTheDocument(); - expect(link).toHaveAttribute( - "href", - `/dags/${sourceDagId}/grid?dag_run_id=${sourceRunId}&task_id=${taskId}` - ); - }); - - test("Replaces existing dag id url param correctly", async () => { - jest.spyOn(utils, "getMetaValue").mockImplementation((meta) => { - if (meta === "dag_id") return originalDagId; - if (meta === "grid_url") return `/dags/${originalDagId}/grid`; - return ""; - }); - - const { getByText } = render( - , - { wrapper: ChakraWrapper } - ); - - const link = getByText(`${sourceDagId}.${taskId}`); - expect(link).toBeInTheDocument(); - expect(link).toHaveAttribute( - "href", - `/dags/${sourceDagId}/grid?dag_run_id=${sourceRunId}&task_id=${taskId}` - ); - }); -}); diff --git a/airflow/www/static/js/components/Table/Cells.tsx b/airflow/www/static/js/components/Table/Cells.tsx index 9f75e0e484503..ec39d89087341 100644 --- a/airflow/www/static/js/components/Table/Cells.tsx +++ b/airflow/www/static/js/components/Table/Cells.tsx @@ -17,27 +17,9 @@ * under the License. */ -import React, { useMemo } from "react"; -import { - Flex, - Code, - Link, - Box, - Text, - useDisclosure, - ModalCloseButton, - Modal, - ModalContent, - ModalOverlay, - ModalBody, - ModalHeader, -} from "@chakra-ui/react"; +import React from "react"; -import { Table } from "src/components/Table"; import Time from "src/components/Time"; -import { getMetaValue } from "src/utils"; -import { useContainerRef } from "src/context/containerRef"; -import { SimpleStatus } from "src/dag/StatusBox"; export interface CellProps { cell: { @@ -53,124 +35,3 @@ export interface CellProps { export const TimeCell = ({ cell: { value } }: CellProps) => (