From 04c286e09bf2fb6d140e629c1db3d81f8ff2bc42 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 11:27:54 +0900 Subject: [PATCH 01/43] feat(api_connexion): rename dataset_endpoint module as asset_endpoint --- ...{dataset_endpoint.py => asset_endpoint.py} | 0 airflow/api_connexion/openapi/v1.yaml | 22 +++++++++---------- airflow/www/templates/airflow/dag.html | 2 +- airflow/www/templates/airflow/datasets.html | 6 ++--- airflow/www/templates/airflow/grid.html | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) rename airflow/api_connexion/endpoints/{dataset_endpoint.py => asset_endpoint.py} (100%) diff --git a/airflow/api_connexion/endpoints/dataset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py similarity index 100% rename from airflow/api_connexion/endpoints/dataset_endpoint.py rename to airflow/api_connexion/endpoints/asset_endpoint.py diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 15ad6fd8a4f63..e6acf91f188f7 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1256,7 +1256,7 @@ paths: Get a queued Dataset event for a DAG. *New in version 2.9.0* - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_dag_dataset_queued_event parameters: - $ref: "#/components/parameters/Before" @@ -1281,7 +1281,7 @@ paths: Delete a queued Dataset event for a DAG. *New in version 2.9.0* - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: delete_dag_dataset_queued_event parameters: - $ref: "#/components/parameters/Before" @@ -1308,7 +1308,7 @@ paths: Get queued Dataset events for a DAG. *New in version 2.9.0* - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_dag_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" @@ -1333,7 +1333,7 @@ paths: Delete queued Dataset events for a DAG. *New in version 2.9.0* - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: delete_dag_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" @@ -1381,7 +1381,7 @@ paths: Get queued Dataset events for a Dataset *New in version 2.9.0* - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" @@ -1406,7 +1406,7 @@ paths: Delete queued Dataset events for a Dataset. *New in version 2.9.0* - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: delete_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" @@ -2519,8 +2519,8 @@ paths: /datasets: get: - summary: List datasets - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + summary: List assets + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_datasets tags: [Dataset] parameters: @@ -2561,7 +2561,7 @@ paths: get: summary: Get a dataset description: Get a dataset by uri. - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_dataset tags: [Dataset] responses: @@ -2582,7 +2582,7 @@ paths: get: summary: Get dataset events description: Get dataset events - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_dataset_events tags: [Dataset] parameters: @@ -2610,7 +2610,7 @@ paths: post: summary: Create dataset event description: Create dataset event - x-openapi-router-controller: airflow.api_connexion.endpoints.dataset_endpoint + x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: create_dataset_event tags: [Dataset] requestBody: diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 0d3a2cf1770ca..69ed4afd3b79b 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -81,7 +81,7 @@ - + diff --git a/airflow/www/templates/airflow/datasets.html b/airflow/www/templates/airflow/datasets.html index 64e08510e4a4f..f692958ec5e92 100644 --- a/airflow/www/templates/airflow/datasets.html +++ b/airflow/www/templates/airflow/datasets.html @@ -24,9 +24,9 @@ {% block head_meta %} {{ super() }} - - - + + + diff --git a/airflow/www/templates/airflow/grid.html b/airflow/www/templates/airflow/grid.html index 2c2cf163e9e07..accf0675d0759 100644 --- a/airflow/www/templates/airflow/grid.html +++ b/airflow/www/templates/airflow/grid.html @@ -27,7 +27,7 @@ - + From 3a989d4d9fe0b5428dfff365ec11b9c098a1e7bf Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 11:30:10 +0900 Subject: [PATCH 02/43] feat(api_connexion/openapi): rename tag Dataset as Asset --- airflow/api_connexion/openapi/v1.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index e6acf91f188f7..bb91dd473dc60 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1260,7 +1260,7 @@ paths: operationId: get_dag_dataset_queued_event parameters: - $ref: "#/components/parameters/Before" - tags: [Dataset] + tags: [Asset] responses: "200": description: Success. @@ -1285,7 +1285,7 @@ paths: operationId: delete_dag_dataset_queued_event parameters: - $ref: "#/components/parameters/Before" - tags: [Dataset] + tags: [Asset] responses: "204": description: Success. @@ -1312,7 +1312,7 @@ paths: operationId: get_dag_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" - tags: [Dataset] + tags: [Asset] responses: "200": description: Success. @@ -1337,7 +1337,7 @@ paths: operationId: delete_dag_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" - tags: [Dataset] + tags: [Asset] responses: "204": description: Success. @@ -1385,7 +1385,7 @@ paths: operationId: get_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" - tags: [Dataset] + tags: [Asset] responses: "200": description: Success. @@ -1410,7 +1410,7 @@ paths: operationId: delete_dataset_queued_events parameters: - $ref: "#/components/parameters/Before" - tags: [Dataset] + tags: [Asset] responses: "204": description: Success. @@ -2522,7 +2522,7 @@ paths: summary: List assets x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_datasets - tags: [Dataset] + tags: [Asset] parameters: - $ref: "#/components/parameters/PageLimit" - $ref: "#/components/parameters/PageOffset" @@ -2563,7 +2563,7 @@ paths: description: Get a dataset by uri. x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_dataset - tags: [Dataset] + tags: [Asset] responses: "200": description: Success. @@ -2584,7 +2584,7 @@ paths: description: Get dataset events x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_dataset_events - tags: [Dataset] + tags: [Asset] parameters: - $ref: "#/components/parameters/PageLimit" - $ref: "#/components/parameters/PageOffset" @@ -2612,7 +2612,7 @@ paths: description: Create dataset event x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: create_dataset_event - tags: [Dataset] + tags: [Asset] requestBody: required: true content: From 45faf056577ee9b4821c27634deaa2157d3bb51c Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 11:34:51 +0900 Subject: [PATCH 03/43] feat(api_connexion): rename create_dataset_event as create_asset_event --- airflow/api_connexion/endpoints/asset_endpoint.py | 4 ++-- airflow/api_connexion/openapi/v1.yaml | 6 +++--- airflow/www/static/js/api/useCreateDatasetEvent.ts | 4 ++-- airflow/www/static/js/types/api-generated.ts | 6 +++--- airflow/www/templates/airflow/datasets.html | 2 +- tests/api_connexion/endpoints/test_dataset_endpoint.py | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 95c3bead3da52..b902d8d7e0b30 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -325,7 +325,7 @@ def delete_dataset_queued_events( @security.requires_access_asset("POST") @provide_session @action_logging -def create_dataset_event(session: Session = NEW_SESSION) -> APIResponse: +def create_asset_event(session: Session = NEW_SESSION) -> APIResponse: """Create asset event.""" body = get_json_request_dict() try: @@ -341,7 +341,7 @@ def create_dataset_event(session: Session = NEW_SESSION) -> APIResponse: extra = json_body.get("extra", {}) extra["from_rest_api"] = True asset_event = asset_manager.register_asset_change( - asset=Asset(uri), + asset=Asset(uri=uri), timestamp=timestamp, extra=extra, session=session, diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index bb91dd473dc60..84082c10d699c 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2608,10 +2608,10 @@ paths: "404": $ref: "#/components/responses/NotFound" post: - summary: Create dataset event - description: Create dataset event + summary: Create asset event + description: Create asset event x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: create_dataset_event + operationId: create_asset_event tags: [Asset] requestBody: required: true diff --git a/airflow/www/static/js/api/useCreateDatasetEvent.ts b/airflow/www/static/js/api/useCreateDatasetEvent.ts index f14b35ee375fe..73fdd0d531aef 100644 --- a/airflow/www/static/js/api/useCreateDatasetEvent.ts +++ b/airflow/www/static/js/api/useCreateDatasetEvent.ts @@ -29,7 +29,7 @@ interface Props { uri?: string; } -const createDatasetUrl = getMetaValue("create_dataset_event_api"); +const createAssetUrl = getMetaValue("create_asset_event_api"); export default function useCreateDatasetEvent({ datasetId, uri }: Props) { const queryClient = useQueryClient(); @@ -39,7 +39,7 @@ export default function useCreateDatasetEvent({ datasetId, uri }: Props) { ["createDatasetEvent", uri], (extra?: API.DatasetEvent["extra"]) => axios.post( - createDatasetUrl, + createAssetUrl, { dataset_uri: uri, extra: extra || {}, diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 3616be30a1fac..be390abb5f2b8 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -799,7 +799,7 @@ export interface paths { /** Get dataset events */ get: operations["get_dataset_events"]; /** Create dataset event */ - post: operations["create_dataset_event"]; + post: operations["create_asset_event"]; }; "/config": { get: operations["get_config"]; @@ -5140,7 +5140,7 @@ export interface operations { }; }; /** Create dataset event */ - create_dataset_event: { + create_asset_event: { responses: { /** Success. */ 200: { @@ -5797,7 +5797,7 @@ export type GetDatasetEventsVariables = CamelCasedPropertiesDeep< operations["get_dataset_events"]["parameters"]["query"] >; export type CreateDatasetEventVariables = CamelCasedPropertiesDeep< - operations["create_dataset_event"]["requestBody"]["content"]["application/json"] + operations["create_asset_event"]["requestBody"]["content"]["application/json"] >; export type GetConfigVariables = CamelCasedPropertiesDeep< operations["get_config"]["parameters"]["query"] diff --git a/airflow/www/templates/airflow/datasets.html b/airflow/www/templates/airflow/datasets.html index f692958ec5e92..37583df29527c 100644 --- a/airflow/www/templates/airflow/datasets.html +++ b/airflow/www/templates/airflow/datasets.html @@ -26,7 +26,7 @@ - + diff --git a/tests/api_connexion/endpoints/test_dataset_endpoint.py b/tests/api_connexion/endpoints/test_dataset_endpoint.py index 76c164654c9d8..8f7b6f90809a0 100644 --- a/tests/api_connexion/endpoints/test_dataset_endpoint.py +++ b/tests/api_connexion/endpoints/test_dataset_endpoint.py @@ -602,7 +602,7 @@ def test_should_respond_200(self, session): _check_last_log( session, dag_id=None, - event="api.create_dataset_event", + event="api.create_asset_event", execution_date=None, expected_extra=event_payload, ) @@ -620,7 +620,7 @@ def test_should_mask_sensitive_extra_logs(self, session): _check_last_log( session, dag_id=None, - event="api.create_dataset_event", + event="api.create_asset_event", execution_date=None, expected_extra=expected_extra, ) From 5d9cb5c519955e4eeb8143b44da7ee94c51c5b70 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 11:46:56 +0900 Subject: [PATCH 04/43] feat(api_connexion): rename schema CreateDatasetEvent as CreateAssetEvent --- .../api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 12 +++++------ airflow/api_connexion/schemas/asset_schema.py | 2 +- airflow/www/static/js/api/index.ts | 4 ++-- ...DatasetEvent.ts => useCreateAssetEvent.ts} | 15 ++++++-------- ...eDatasetEvent.tsx => CreateAssetEvent.tsx} | 10 +++++----- airflow/www/static/js/datasets/Main.tsx | 4 ++-- airflow/www/static/js/types/api-generated.ts | 20 +++++++++---------- 8 files changed, 33 insertions(+), 36 deletions(-) rename airflow/www/static/js/api/{useCreateDatasetEvent.ts => useCreateAssetEvent.ts} (84%) rename airflow/www/static/js/datasets/{CreateDatasetEvent.tsx => CreateAssetEvent.tsx} (89%) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index b902d8d7e0b30..4d3eb15db8acb 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -333,7 +333,7 @@ def create_asset_event(session: Session = NEW_SESSION) -> APIResponse: except ValidationError as err: raise BadRequest(detail=str(err)) - uri = json_body["dataset_uri"] + uri = json_body["asset_uri"] asset = session.scalar(select(AssetModel).where(AssetModel.uri == uri).limit(1)) if not asset: raise NotFound(title="Asset not found", detail=f"Asset with uri: '{uri}' not found") diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 84082c10d699c..48adcc4087e8d 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2618,7 +2618,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CreateDatasetEvent' + $ref: '#/components/schemas/CreateAssetEvent' responses: '200': description: Success. @@ -4644,18 +4644,18 @@ components: description: The dataset event creation time nullable: false - CreateDatasetEvent: + CreateAssetEvent: type: object required: - - dataset_uri + - asset_uri properties: - dataset_uri: + asset_uri: type: string - description: The URI of the dataset + description: The URI of the asset nullable: false extra: type: object - description: The dataset event extra + description: The asset event extra nullable: true QueuedEvent: diff --git a/airflow/api_connexion/schemas/asset_schema.py b/airflow/api_connexion/schemas/asset_schema.py index 791941f42016d..282c7818c56ca 100644 --- a/airflow/api_connexion/schemas/asset_schema.py +++ b/airflow/api_connexion/schemas/asset_schema.py @@ -164,7 +164,7 @@ class AssetEventCollectionSchema(Schema): class CreateAssetEventSchema(Schema): """Create Asset Event Schema.""" - dataset_uri = fields.String() + asset_uri = fields.String() extra = JsonObjectField() diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index a4a45a08bfef8..09557dd55cc2a 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -53,7 +53,7 @@ import useHistoricalMetricsData from "./useHistoricalMetricsData"; import { useTaskXcomEntry, useTaskXcomCollection } from "./useTaskXcom"; import useEventLogs from "./useEventLogs"; import useCalendarData from "./useCalendarData"; -import useCreateDatasetEvent from "./useCreateDatasetEvent"; +import useCreateAssetEvent from "./useCreateAssetEvent"; import useRenderedK8s from "./useRenderedK8s"; import useTaskDetail from "./useTaskDetail"; import useTIHistory from "./useTIHistory"; @@ -112,7 +112,7 @@ export { useTaskFailedDependency, useEventLogs, useCalendarData, - useCreateDatasetEvent, + useCreateAssetEvent, useRenderedK8s, useTaskDetail, useTIHistory, diff --git a/airflow/www/static/js/api/useCreateDatasetEvent.ts b/airflow/www/static/js/api/useCreateAssetEvent.ts similarity index 84% rename from airflow/www/static/js/api/useCreateDatasetEvent.ts rename to airflow/www/static/js/api/useCreateAssetEvent.ts index 73fdd0d531aef..e0fea7a2cf679 100644 --- a/airflow/www/static/js/api/useCreateDatasetEvent.ts +++ b/airflow/www/static/js/api/useCreateAssetEvent.ts @@ -31,20 +31,17 @@ interface Props { const createAssetUrl = getMetaValue("create_asset_event_api"); -export default function useCreateDatasetEvent({ datasetId, uri }: Props) { +export default function useCreateAssetEvent({ datasetId, uri }: Props) { const queryClient = useQueryClient(); const errorToast = useErrorToast(); return useMutation( - ["createDatasetEvent", uri], + ["createAssetEvent", uri], (extra?: API.DatasetEvent["extra"]) => - axios.post( - createAssetUrl, - { - dataset_uri: uri, - extra: extra || {}, - } - ), + axios.post(createAssetUrl, { + asset_uri: uri, + extra: extra || {}, + }), { onSuccess: () => { queryClient.invalidateQueries(["datasets-events", datasetId]); diff --git a/airflow/www/static/js/datasets/CreateDatasetEvent.tsx b/airflow/www/static/js/datasets/CreateAssetEvent.tsx similarity index 89% rename from airflow/www/static/js/datasets/CreateDatasetEvent.tsx rename to airflow/www/static/js/datasets/CreateAssetEvent.tsx index 39de1143715ad..69a78de8b7586 100644 --- a/airflow/www/static/js/datasets/CreateDatasetEvent.tsx +++ b/airflow/www/static/js/datasets/CreateAssetEvent.tsx @@ -34,7 +34,7 @@ import { } from "@chakra-ui/react"; import { useContainerRef } from "src/context/containerRef"; -import { useCreateDatasetEvent, useDataset } from "src/api"; +import { useCreateAssetEvent, useDataset } from "src/api"; interface Props { isOpen: boolean; @@ -51,7 +51,7 @@ function checkJsonString(str: string) { return true; } -const CreateDatasetEventModal = ({ uri, isOpen, onClose }: Props) => { +const CreateAssetEventModal = ({ uri, isOpen, onClose }: Props) => { const containerRef = useContainerRef(); const [extra, setExtra] = useState(""); const { data: dataset } = useDataset({ uri }); @@ -59,13 +59,13 @@ const CreateDatasetEventModal = ({ uri, isOpen, onClose }: Props) => { const isJson = checkJsonString(extra); const isDisabled = !!extra && !isJson; - const { mutate: createDatasetEvent, isLoading } = useCreateDatasetEvent({ + const { mutate: createAssetEvent, isLoading } = useCreateAssetEvent({ datasetId: dataset?.id, uri: dataset?.uri, }); const onSubmit = () => { - createDatasetEvent(extra ? JSON.parse(extra) : undefined); + createAssetEvent(extra ? JSON.parse(extra) : undefined); onClose(); }; @@ -110,4 +110,4 @@ const CreateDatasetEventModal = ({ uri, isOpen, onClose }: Props) => { ); }; -export default CreateDatasetEventModal; +export default CreateAssetEventModal; diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index 3d9b5e51c382d..3d165a7ebfe3d 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -53,7 +53,7 @@ import DatasetDetails from "./DatasetDetails"; import type { OnSelectProps } from "./types"; import Graph from "./Graph"; import SearchBar from "./SearchBar"; -import CreateDatasetEventModal from "./CreateDatasetEvent"; +import CreateAssetEventModal from "./CreateAssetEvent"; const DATASET_URI_PARAM = "uri"; const DAG_ID_PARAM = "dag_id"; @@ -281,7 +281,7 @@ const Datasets = () => { {selectedUri && ( - ; -export type CreateDatasetEvent = CamelCasedPropertiesDeep< - components["schemas"]["CreateDatasetEvent"] +export type CreateAssetEvent = CamelCasedPropertiesDeep< + components["schemas"]["CreateAssetEvent"] >; export type QueuedEvent = CamelCasedPropertiesDeep< components["schemas"]["QueuedEvent"] @@ -5796,7 +5796,7 @@ export type GetDatasetVariables = CamelCasedPropertiesDeep< export type GetDatasetEventsVariables = CamelCasedPropertiesDeep< operations["get_dataset_events"]["parameters"]["query"] >; -export type CreateDatasetEventVariables = CamelCasedPropertiesDeep< +export type CreateAssetEventVariables = CamelCasedPropertiesDeep< operations["create_asset_event"]["requestBody"]["content"]["application/json"] >; export type GetConfigVariables = CamelCasedPropertiesDeep< From cb76782fc8365b4a2fe30ae7d57ed3e70ab3af32 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 15:03:24 +0900 Subject: [PATCH 05/43] test(api_connexion): rename test_dataset_endpoint as test_asset_endpoint --- ...test_dataset_endpoint.py => test_asset_endpoint.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename tests/api_connexion/endpoints/{test_dataset_endpoint.py => test_asset_endpoint.py} (98%) diff --git a/tests/api_connexion/endpoints/test_dataset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py similarity index 98% rename from tests/api_connexion/endpoints/test_dataset_endpoint.py rename to tests/api_connexion/endpoints/test_asset_endpoint.py index 8f7b6f90809a0..59d1b4208def9 100644 --- a/tests/api_connexion/endpoints/test_dataset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -580,7 +580,7 @@ def time_freezer(self) -> Generator: @pytest.mark.usefixtures("time_freezer") def test_should_respond_200(self, session): self._create_dataset(session) - event_payload = {"dataset_uri": "s3://bucket/key", "extra": {"foo": "bar"}} + event_payload = {"asset_uri": "s3://bucket/key", "extra": {"foo": "bar"}} response = self.client.post( "/api/v1/datasets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} ) @@ -590,7 +590,7 @@ def test_should_respond_200(self, session): assert response_data == { "id": ANY, "created_dagruns": [], - "dataset_uri": event_payload["dataset_uri"], + "dataset_uri": event_payload["asset_uri"], "dataset_id": ANY, "extra": {"foo": "bar", "from_rest_api": True}, "source_dag_id": None, @@ -610,7 +610,7 @@ def test_should_respond_200(self, session): @pytest.mark.enable_redact def test_should_mask_sensitive_extra_logs(self, session): self._create_dataset(session) - event_payload = {"dataset_uri": "s3://bucket/key", "extra": {"password": "bar"}} + event_payload = {"asset_uri": "s3://bucket/key", "extra": {"password": "bar"}} response = self.client.post( "/api/v1/datasets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} ) @@ -627,7 +627,7 @@ def test_should_mask_sensitive_extra_logs(self, session): def test_order_by_raises_400_for_invalid_attr(self, session): self._create_dataset(session) - event_invalid_payload = {"dataset_uri": "TEST_DATASET_URI", "extra": {"foo": "bar"}, "fake": {}} + event_invalid_payload = {"asset_uri": "TEST_ASSET_URI", "extra": {"foo": "bar"}, "fake": {}} response = self.client.post( "/api/v1/datasets/events", json=event_invalid_payload, environ_overrides={"REMOTE_USER": "test"} ) @@ -635,7 +635,7 @@ def test_order_by_raises_400_for_invalid_attr(self, session): def test_should_raises_401_unauthenticated(self, session): self._create_dataset(session) - response = self.client.post("/api/v1/datasets/events", json={"dataset_uri": "TEST_DATASET_URI"}) + response = self.client.post("/api/v1/datasets/events", json={"asset_uri": "TEST_ASSET_URI"}) assert_401(response) From 246eff57f5d1c36fee5777f848d015abf0202870 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 15:56:00 +0900 Subject: [PATCH 06/43] feat(api_connexion): rename delete_dataset_queued_events as delete_asset_queued_events --- airflow/api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 2 +- airflow/www/static/js/types/api-generated.ts | 8 ++++---- tests/api_connexion/endpoints/test_asset_endpoint.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 4d3eb15db8acb..e5d185d0de433 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -303,7 +303,7 @@ def get_dataset_queued_events( @security.requires_access_asset("DELETE") @action_logging @provide_session -def delete_dataset_queued_events( +def delete_asset_queued_events( *, uri: str, before: str | None = None, session: Session = NEW_SESSION ) -> APIResponse: """Delete queued asset events for an asset.""" diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 48adcc4087e8d..bb3d1d25357b8 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1407,7 +1407,7 @@ paths: *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: delete_dataset_queued_events + operationId: delete_asset_queued_events parameters: - $ref: "#/components/parameters/Before" tags: [Asset] diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 78f9134258157..ccfeaaa4b7c9a 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -362,7 +362,7 @@ export interface paths { * * *New in version 2.9.0* */ - delete: operations["delete_dataset_queued_events"]; + delete: operations["delete_asset_queued_events"]; parameters: { path: { /** The encoded Dataset URI */ @@ -3812,7 +3812,7 @@ export interface operations { * * *New in version 2.9.0* */ - delete_dataset_queued_events: { + delete_asset_queued_events: { parameters: { path: { /** The encoded Dataset URI */ @@ -5664,8 +5664,8 @@ export type GetDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< operations["get_dataset_queued_events"]["parameters"]["query"] >; export type DeleteDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< - operations["delete_dataset_queued_events"]["parameters"]["path"] & - operations["delete_dataset_queued_events"]["parameters"]["query"] + operations["delete_asset_queued_events"]["parameters"]["path"] & + operations["delete_asset_queued_events"]["parameters"]["query"] >; export type GetEventLogsVariables = CamelCasedPropertiesDeep< operations["get_event_logs"]["parameters"]["query"] diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index 59d1b4208def9..f4902ab7d3908 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -846,7 +846,7 @@ def test_should_raise_403_forbidden(self): assert response.status_code == 403 -class TestDeleteDatasetQueuedEvents(TestQueuedEventEndpoint): +class TestDeleteAssetQueuedEvents(TestQueuedEventEndpoint): def test_should_raises_401_unauthenticated(self): dataset_uri = "not_exists" From 2067636c6a48c163b35ad8c833afd28f95a7eaf5 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 16:54:45 +0900 Subject: [PATCH 07/43] feat(api_connexion): rename get_dataset_queued_events as get_asset_queued_events --- airflow/api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 2 +- airflow/www/static/js/types/api-generated.ts | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index e5d185d0de433..341598f2bde0f 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -271,7 +271,7 @@ def delete_dag_dataset_queued_events( @security.requires_access_asset("GET") @provide_session -def get_dataset_queued_events( +def get_asset_queued_events( *, uri: str, before: str | None = None, session: Session = NEW_SESSION ) -> APIResponse: """Get queued asset events for an asset.""" diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index bb3d1d25357b8..a9884f6fa6a44 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1382,7 +1382,7 @@ paths: *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: get_dataset_queued_events + operationId: get_asset_queued_events parameters: - $ref: "#/components/parameters/Before" tags: [Asset] diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index ccfeaaa4b7c9a..13a05cc6894bb 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -356,7 +356,7 @@ export interface paths { * * *New in version 2.9.0* */ - get: operations["get_dataset_queued_events"]; + get: operations["get_asset_queued_events"]; /** * Delete queued Dataset events for a Dataset. * @@ -3784,7 +3784,7 @@ export interface operations { * * *New in version 2.9.0* */ - get_dataset_queued_events: { + get_asset_queued_events: { parameters: { path: { /** The encoded Dataset URI */ @@ -5659,11 +5659,11 @@ export type DeleteDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< export type ReparseDagFileVariables = CamelCasedPropertiesDeep< operations["reparse_dag_file"]["parameters"]["path"] >; -export type GetDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< - operations["get_dataset_queued_events"]["parameters"]["path"] & - operations["get_dataset_queued_events"]["parameters"]["query"] +export type GetAssetQueuedEventsVariables = CamelCasedPropertiesDeep< + operations["get_asset_queued_events"]["parameters"]["path"] & + operations["get_asset_queued_events"]["parameters"]["query"] >; -export type DeleteDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< +export type DeleteAssetQueuedEventsVariables = CamelCasedPropertiesDeep< operations["delete_asset_queued_events"]["parameters"]["path"] & operations["delete_asset_queued_events"]["parameters"]["query"] >; From 9a054c17ed352d1f53aacb6deee861f26c4509f5 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 17:27:52 +0900 Subject: [PATCH 08/43] feat(api_connexion): rename delete_dag_dataset_queued_events as delete_dag_asset_queued_events --- airflow/api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 6 +++--- airflow/www/static/js/types/api-generated.ts | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 341598f2bde0f..1a6e411c0dd2e 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -253,7 +253,7 @@ def get_dag_dataset_queued_events( @security.requires_access_dag("GET") @action_logging @provide_session -def delete_dag_dataset_queued_events( +def delete_dag_asset_queued_events( *, dag_id: str, before: str | None = None, session: Session = NEW_SESSION ) -> APIResponse: """Delete queued asset events for a DAG.""" diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index a9884f6fa6a44..121a76e6c207d 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1328,13 +1328,13 @@ paths: $ref: "#/components/responses/NotFound" delete: - summary: Delete queued Dataset events for a DAG. + summary: Delete queued Asset events for a DAG. description: | - Delete queued Dataset events for a DAG. + Delete queued Asset events for a DAG. *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: delete_dag_dataset_queued_events + operationId: delete_dag_asset_queued_events parameters: - $ref: "#/components/parameters/Before" tags: [Asset] diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 13a05cc6894bb..cff6646cace43 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -328,7 +328,7 @@ export interface paths { * * *New in version 2.9.0* */ - delete: operations["delete_dag_dataset_queued_events"]; + delete: operations["delete_dag_asset_queued_events"]; parameters: { path: { /** The DAG ID. */ @@ -3739,7 +3739,7 @@ export interface operations { * * *New in version 2.9.0* */ - delete_dag_dataset_queued_events: { + delete_dag_asset_queued_events: { parameters: { path: { /** The DAG ID. */ @@ -5653,8 +5653,8 @@ export type GetDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< operations["get_dag_dataset_queued_events"]["parameters"]["query"] >; export type DeleteDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< - operations["delete_dag_dataset_queued_events"]["parameters"]["path"] & - operations["delete_dag_dataset_queued_events"]["parameters"]["query"] + operations["delete_dag_asset_queued_events"]["parameters"]["path"] & + operations["delete_dag_asset_queued_events"]["parameters"]["query"] >; export type ReparseDagFileVariables = CamelCasedPropertiesDeep< operations["reparse_dag_file"]["parameters"]["path"] From 39e6f4a9b1d2aa52b92c8f5bb71c631fdaef1126 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 17:30:01 +0900 Subject: [PATCH 09/43] feat(api_connexion): rename delete_dag_dataset_queued_event as delete_dag_asset_queued_event --- airflow/api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 6 +++--- airflow/www/static/js/types/api-generated.ts | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 1a6e411c0dd2e..326ec5e4b9069 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -206,7 +206,7 @@ def get_dag_dataset_queued_event( @security.requires_access_dag("GET") @provide_session @action_logging -def delete_dag_dataset_queued_event( +def delete_dag_asset_queued_event( *, dag_id: str, uri: str, before: str | None = None, session: Session = NEW_SESSION ) -> APIResponse: """Delete a queued asset event for a DAG.""" diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 121a76e6c207d..a89128092245a 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1276,13 +1276,13 @@ paths: $ref: "#/components/responses/NotFound" delete: - summary: Delete a queued Dataset event for a DAG. + summary: Delete a queued Asset event for a DAG. description: | - Delete a queued Dataset event for a DAG. + Delete a queued Asset event for a DAG. *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: delete_dag_dataset_queued_event + operationId: delete_dag_asset_queued_event parameters: - $ref: "#/components/parameters/Before" tags: [Asset] diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index cff6646cace43..a59a8f105b63d 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -306,7 +306,7 @@ export interface paths { * * *New in version 2.9.0* */ - delete: operations["delete_dag_dataset_queued_event"]; + delete: operations["delete_dag_asset_queued_event"]; parameters: { path: { /** The DAG ID. */ @@ -3684,7 +3684,7 @@ export interface operations { * * *New in version 2.9.0* */ - delete_dag_dataset_queued_event: { + delete_dag_asset_queued_event: { parameters: { path: { /** The DAG ID. */ @@ -5645,8 +5645,8 @@ export type GetDagDatasetQueuedEventVariables = CamelCasedPropertiesDeep< operations["get_dag_dataset_queued_event"]["parameters"]["query"] >; export type DeleteDagDatasetQueuedEventVariables = CamelCasedPropertiesDeep< - operations["delete_dag_dataset_queued_event"]["parameters"]["path"] & - operations["delete_dag_dataset_queued_event"]["parameters"]["query"] + operations["delete_dag_asset_queued_event"]["parameters"]["path"] & + operations["delete_dag_asset_queued_event"]["parameters"]["query"] >; export type GetDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< operations["get_dag_dataset_queued_events"]["parameters"]["path"] & From 08afa578ee83b7e041d486d72d9e502021441252 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 17:32:29 +0900 Subject: [PATCH 10/43] feat(api_connexion): rename get_dag_dataset_queued_events as get_dag_asset_queued_events --- airflow/api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 6 +++--- airflow/www/static/js/types/api-generated.ts | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 326ec5e4b9069..efc0843adb8de 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -224,7 +224,7 @@ def delete_dag_asset_queued_event( @security.requires_access_asset("GET") @security.requires_access_dag("GET") @provide_session -def get_dag_dataset_queued_events( +def get_dag_asset_queued_events( *, dag_id: str, before: str | None = None, session: Session = NEW_SESSION ) -> APIResponse: """Get queued asset events for a DAG.""" diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index a89128092245a..f1428d08f14db 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1303,13 +1303,13 @@ paths: - $ref: "#/components/parameters/DAGID" get: - summary: Get queued Dataset events for a DAG. + summary: Get queued Asset events for a DAG. description: | - Get queued Dataset events for a DAG. + Get queued Asset events for a DAG. *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: get_dag_dataset_queued_events + operationId: get_dag_asset_queued_events parameters: - $ref: "#/components/parameters/Before" tags: [Asset] diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index a59a8f105b63d..904ca8b8aaa38 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -322,7 +322,7 @@ export interface paths { * * *New in version 2.9.0* */ - get: operations["get_dag_dataset_queued_events"]; + get: operations["get_dag_asset_queued_events"]; /** * Delete queued Dataset events for a DAG. * @@ -3711,7 +3711,7 @@ export interface operations { * * *New in version 2.9.0* */ - get_dag_dataset_queued_events: { + get_dag_asset_queued_events: { parameters: { path: { /** The DAG ID. */ @@ -5649,8 +5649,8 @@ export type DeleteDagDatasetQueuedEventVariables = CamelCasedPropertiesDeep< operations["delete_dag_asset_queued_event"]["parameters"]["query"] >; export type GetDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< - operations["get_dag_dataset_queued_events"]["parameters"]["path"] & - operations["get_dag_dataset_queued_events"]["parameters"]["query"] + operations["get_dag_asset_queued_events"]["parameters"]["path"] & + operations["get_dag_asset_queued_events"]["parameters"]["query"] >; export type DeleteDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< operations["delete_dag_asset_queued_events"]["parameters"]["path"] & From 1c328ea2fd2836ad98c510bcad1882a6170c24ac Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 17:35:02 +0900 Subject: [PATCH 11/43] feat(api_connexion): rename get_dag_dataset_queued_event as get_dag_asset_queued_event --- airflow/api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 2 +- airflow/www/static/js/types/api-generated.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index efc0843adb8de..d39468876e734 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -183,7 +183,7 @@ def _generate_queued_event_where_clause( @security.requires_access_asset("GET") @security.requires_access_dag("GET") @provide_session -def get_dag_dataset_queued_event( +def get_dag_asset_queued_event( *, dag_id: str, uri: str, before: str | None = None, session: Session = NEW_SESSION ) -> APIResponse: """Get a queued asset event for a DAG.""" diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index f1428d08f14db..6ac6c869ea76a 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1257,7 +1257,7 @@ paths: *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: get_dag_dataset_queued_event + operationId: get_dag_asset_queued_event parameters: - $ref: "#/components/parameters/Before" tags: [Asset] diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 904ca8b8aaa38..1a2e4362b4d54 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -300,7 +300,7 @@ export interface paths { * * *New in version 2.9.0* */ - get: operations["get_dag_dataset_queued_event"]; + get: operations["get_dag_asset_queued_event"]; /** * Delete a queued Dataset event for a DAG. * @@ -3654,7 +3654,7 @@ export interface operations { * * *New in version 2.9.0* */ - get_dag_dataset_queued_event: { + get_dag_asset_queued_event: { parameters: { path: { /** The DAG ID. */ @@ -5641,8 +5641,8 @@ export type SetDagRunNoteVariables = CamelCasedPropertiesDeep< operations["set_dag_run_note"]["requestBody"]["content"]["application/json"] >; export type GetDagDatasetQueuedEventVariables = CamelCasedPropertiesDeep< - operations["get_dag_dataset_queued_event"]["parameters"]["path"] & - operations["get_dag_dataset_queued_event"]["parameters"]["query"] + operations["get_dag_asset_queued_event"]["parameters"]["path"] & + operations["get_dag_asset_queued_event"]["parameters"]["query"] >; export type DeleteDagDatasetQueuedEventVariables = CamelCasedPropertiesDeep< operations["delete_dag_asset_queued_event"]["parameters"]["path"] & From 30c4b3bfc39930f2b8d595ee53329e1e998ff659 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 17:35:54 +0900 Subject: [PATCH 12/43] refactor(api_connexion): remove unused dataset_id in _generate_queued_event_where_clause --- airflow/api_connexion/endpoints/asset_endpoint.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index d39468876e734..8f44c2d643257 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -156,7 +156,6 @@ def get_dataset_events( def _generate_queued_event_where_clause( *, dag_id: str | None = None, - dataset_id: int | None = None, uri: str | None = None, before: str | None = None, permitted_dag_ids: set[str] | None = None, @@ -165,8 +164,6 @@ def _generate_queued_event_where_clause( where_clause = [] if dag_id is not None: where_clause.append(AssetDagRunQueue.target_dag_id == dag_id) - if dataset_id is not None: - where_clause.append(AssetDagRunQueue.dataset_id == dataset_id) if uri is not None: where_clause.append( AssetDagRunQueue.dataset_id.in_( From 48b42336913558de27f03b1295432b5941e6ecb2 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 17:42:03 +0900 Subject: [PATCH 13/43] feat(api_connexion): rename get_dataset_events as get_asset_events --- airflow/api_connexion/endpoints/asset_endpoint.py | 8 ++++---- airflow/api_connexion/openapi/v1.yaml | 10 +++++----- airflow/www/static/js/types/api-generated.ts | 10 +++++----- airflow/www/templates/airflow/datasets.html | 2 +- airflow/www/templates/airflow/grid.html | 2 +- clients/python/README.md | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 8f44c2d643257..2ee91d113df08 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -115,12 +115,12 @@ def get_datasets( @security.requires_access_asset("GET") @provide_session @format_parameters({"limit": check_limit}) -def get_dataset_events( +def get_asset_events( *, limit: int, offset: int = 0, order_by: str = "timestamp", - dataset_id: int | None = None, + asset_id: int | None = None, source_dag_id: str | None = None, source_task_id: str | None = None, source_run_id: str | None = None, @@ -132,8 +132,8 @@ def get_dataset_events( query = select(AssetEvent) - if dataset_id: - query = query.where(AssetEvent.dataset_id == dataset_id) + if asset_id: + query = query.where(AssetEvent.dataset_id == asset_id) if source_dag_id: query = query.where(AssetEvent.source_dag_id == source_dag_id) if source_task_id: diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 6ac6c869ea76a..195106c4c1b2a 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2583,13 +2583,13 @@ paths: summary: Get dataset events description: Get dataset events x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: get_dataset_events + operationId: get_asset_events tags: [Asset] parameters: - $ref: "#/components/parameters/PageLimit" - $ref: "#/components/parameters/PageOffset" - $ref: "#/components/parameters/OrderBy" - - $ref: "#/components/parameters/FilterDatasetID" + - $ref: "#/components/parameters/FilterAssetID" - $ref: "#/components/parameters/FilterSourceDAGID" - $ref: "#/components/parameters/FilterSourceTaskID" - $ref: "#/components/parameters/FilterSourceRunID" @@ -5733,12 +5733,12 @@ components: *New in version 2.2.0* - FilterDatasetID: + FilterAssetID: in: query - name: dataset_id + name: asset_id schema: type: integer - description: The Dataset ID that updated the dataset. + description: The Asset ID that updated the asset. FilterSourceDAGID: in: query diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 1a2e4362b4d54..d964d1b31b549 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -797,7 +797,7 @@ export interface paths { }; "/datasets/events": { /** Get dataset events */ - get: operations["get_dataset_events"]; + get: operations["get_asset_events"]; /** Create asset event */ post: operations["create_asset_event"]; }; @@ -2622,7 +2622,7 @@ export interface components { */ FilterTags: string[]; /** @description The Dataset ID that updated the dataset. */ - FilterDatasetID: number; + FilterAssetID: number; /** @description The DAG ID that updated the dataset. */ FilterSourceDAGID: string; /** @description The task ID that updated the dataset. */ @@ -5101,7 +5101,7 @@ export interface operations { }; }; /** Get dataset events */ - get_dataset_events: { + get_asset_events: { parameters: { query: { /** The numbers of items to return. */ @@ -5116,7 +5116,7 @@ export interface operations { */ order_by?: components["parameters"]["OrderBy"]; /** The Dataset ID that updated the dataset. */ - dataset_id?: components["parameters"]["FilterDatasetID"]; + dataset_id?: components["parameters"]["FilterAssetID"]; /** The DAG ID that updated the dataset. */ source_dag_id?: components["parameters"]["FilterSourceDAGID"]; /** The task ID that updated the dataset. */ @@ -5794,7 +5794,7 @@ export type GetDatasetVariables = CamelCasedPropertiesDeep< operations["get_dataset"]["parameters"]["path"] >; export type GetDatasetEventsVariables = CamelCasedPropertiesDeep< - operations["get_dataset_events"]["parameters"]["query"] + operations["get_asset_events"]["parameters"]["query"] >; export type CreateAssetEventVariables = CamelCasedPropertiesDeep< operations["create_asset_event"]["requestBody"]["content"]["application/json"] diff --git a/airflow/www/templates/airflow/datasets.html b/airflow/www/templates/airflow/datasets.html index 37583df29527c..2cf90e5b95c39 100644 --- a/airflow/www/templates/airflow/datasets.html +++ b/airflow/www/templates/airflow/datasets.html @@ -25,7 +25,7 @@ {{ super() }} - + diff --git a/airflow/www/templates/airflow/grid.html b/airflow/www/templates/airflow/grid.html index accf0675d0759..834724b92e879 100644 --- a/airflow/www/templates/airflow/grid.html +++ b/airflow/www/templates/airflow/grid.html @@ -27,7 +27,7 @@ - + diff --git a/clients/python/README.md b/clients/python/README.md index 055aa4d24d885..d2a458fe7eeb1 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -349,7 +349,7 @@ Class | Method | HTTP request | Description *DAGRunApi* | [**update_dag_run_state**](docs/DAGRunApi.md#update_dag_run_state) | **PATCH** /dags/{dag_id}/dagRuns/{dag_run_id} | Modify a DAG run *DagWarningApi* | [**get_dag_warnings**](docs/DagWarningApi.md#get_dag_warnings) | **GET** /dagWarnings | List dag warnings *DatasetApi* | [**get_dataset**](docs/DatasetApi.md#get_dataset) | **GET** /datasets/{uri} | Get a dataset -*DatasetApi* | [**get_dataset_events**](docs/DatasetApi.md#get_dataset_events) | **GET** /datasets/events | Get dataset events +*AssetApi* | [**get_asset_events**](docs/DatasetApi.md#get_asset_events) | **GET** /datasets/events | Get asset events *DatasetApi* | [**get_datasets**](docs/DatasetApi.md#get_datasets) | **GET** /datasets | List datasets *DatasetApi* | [**get_upstream_dataset_events**](docs/DatasetApi.md#get_upstream_dataset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents | Get dataset events for a DAG run *EventLogApi* | [**get_event_log**](docs/EventLogApi.md#get_event_log) | **GET** /eventLogs/{event_log_id} | Get a log entry From f50ea72cc6e0034c33c0dd90ad7353986ba4757f Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 19:14:46 +0900 Subject: [PATCH 14/43] feat(api_connexion): rename get_datasets as get_assets --- .../api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 4 +- airflow/www/static/js/api/useDatasetEvents.ts | 10 ++--- .../www/static/js/datasets/DatasetDetails.tsx | 2 +- .../www/static/js/datasets/DatasetEvents.tsx | 6 +-- airflow/www/static/js/types/api-generated.ts | 38 +++++++++---------- airflow/www/templates/airflow/dag.html | 2 +- clients/python/README.md | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 2ee91d113df08..a1edba826f195 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -80,7 +80,7 @@ def get_dataset(*, uri: str, session: Session = NEW_SESSION) -> APIResponse: @security.requires_access_asset("GET") @format_parameters({"limit": check_limit}) @provide_session -def get_datasets( +def get_assets( *, limit: int, offset: int = 0, diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 195106c4c1b2a..27c461e784f57 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2521,7 +2521,7 @@ paths: get: summary: List assets x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: get_datasets + operationId: get_assets tags: [Asset] parameters: - $ref: "#/components/parameters/PageLimit" @@ -2533,7 +2533,7 @@ paths: type: string required: false description: | - If set, only return datasets with uris matching this pattern. + If set, only return assets with uris matching this pattern. - name: dag_ids in: query schema: diff --git a/airflow/www/static/js/api/useDatasetEvents.ts b/airflow/www/static/js/api/useDatasetEvents.ts index 30e4670a87d3e..13b8a78b40636 100644 --- a/airflow/www/static/js/api/useDatasetEvents.ts +++ b/airflow/www/static/js/api/useDatasetEvents.ts @@ -24,15 +24,15 @@ import { getMetaValue } from "src/utils"; import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; import type { DatasetEventCollection, - GetDatasetEventsVariables, + GetAssetEventsVariables, } from "src/types/api-generated"; -interface Props extends GetDatasetEventsVariables { +interface Props extends GetAssetEventsVariables { options?: UseQueryOptions; } const useDatasetEvents = ({ - datasetId, + assetId, sourceDagId, sourceRunId, sourceTaskId, @@ -45,7 +45,7 @@ const useDatasetEvents = ({ const query = useQuery( [ "datasets-events", - datasetId, + assetId, sourceDagId, sourceRunId, sourceTaskId, @@ -62,7 +62,7 @@ const useDatasetEvents = ({ if (limit) params.set("limit", limit.toString()); if (offset) params.set("offset", offset.toString()); if (orderBy) params.set("order_by", orderBy); - if (datasetId) params.set("dataset_id", datasetId.toString()); + if (assetId) params.set("asset_id", assetId.toString()); if (sourceDagId) params.set("source_dag_id", sourceDagId); if (sourceRunId) params.set("source_run_id", sourceRunId); if (sourceTaskId) params.set("source_task_id", sourceTaskId); diff --git a/airflow/www/static/js/datasets/DatasetDetails.tsx b/airflow/www/static/js/datasets/DatasetDetails.tsx index 715c2bf308683..8479d97a5bfe4 100644 --- a/airflow/www/static/js/datasets/DatasetDetails.tsx +++ b/airflow/www/static/js/datasets/DatasetDetails.tsx @@ -102,7 +102,7 @@ const DatasetDetails = ({ uri }: Props) => { /> )} - {dataset && dataset.id && } + {dataset && dataset.id && } ); diff --git a/airflow/www/static/js/datasets/DatasetEvents.tsx b/airflow/www/static/js/datasets/DatasetEvents.tsx index 5cec880308249..26830485daf71 100644 --- a/airflow/www/static/js/datasets/DatasetEvents.tsx +++ b/airflow/www/static/js/datasets/DatasetEvents.tsx @@ -29,7 +29,7 @@ import type { DatasetEvent } from "src/types/api-generated"; import DatasetEventCard from "src/components/DatasetEventCard"; type Props = { - datasetId?: number; + assetId?: number; showLabel?: boolean; }; @@ -37,7 +37,7 @@ const cardDef: CardDef = { card: ({ row }) => , }; -const Events = ({ datasetId, showLabel }: Props) => { +const Events = ({ assetId, showLabel }: Props) => { const limit = 25; const [offset, setOffset] = useState(0); const [sortBy, setSortBy] = useState[]>([ @@ -51,7 +51,7 @@ const Events = ({ datasetId, showLabel }: Props) => { data: { datasetEvents = [], totalEntries = 0 }, isLoading: isEventsLoading, } = useDatasetEvents({ - datasetId, + assetId, limit, offset, orderBy, diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index d964d1b31b549..7e85b3cf95afa 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -302,7 +302,7 @@ export interface paths { */ get: operations["get_dag_asset_queued_event"]; /** - * Delete a queued Dataset event for a DAG. + * Delete a queued Asset event for a DAG. * * *New in version 2.9.0* */ @@ -318,13 +318,13 @@ export interface paths { }; "/dags/{dag_id}/datasets/queuedEvent": { /** - * Get queued Dataset events for a DAG. + * Get queued Asset events for a DAG. * * *New in version 2.9.0* */ get: operations["get_dag_asset_queued_events"]; /** - * Delete queued Dataset events for a DAG. + * Delete queued Asset events for a DAG. * * *New in version 2.9.0* */ @@ -783,7 +783,7 @@ export interface paths { get: operations["get_dag_warnings"]; }; "/datasets": { - get: operations["get_datasets"]; + get: operations["get_assets"]; }; "/datasets/{uri}": { /** Get a dataset by uri. */ @@ -2621,7 +2621,7 @@ export interface components { * *New in version 2.2.0* */ FilterTags: string[]; - /** @description The Dataset ID that updated the dataset. */ + /** @description The Asset ID that updated the asset. */ FilterAssetID: number; /** @description The DAG ID that updated the dataset. */ FilterSourceDAGID: string; @@ -3680,7 +3680,7 @@ export interface operations { }; }; /** - * Delete a queued Dataset event for a DAG. + * Delete a queued Asset event for a DAG. * * *New in version 2.9.0* */ @@ -3707,7 +3707,7 @@ export interface operations { }; }; /** - * Get queued Dataset events for a DAG. + * Get queued Asset events for a DAG. * * *New in version 2.9.0* */ @@ -3735,7 +3735,7 @@ export interface operations { }; }; /** - * Delete queued Dataset events for a DAG. + * Delete queued Asset events for a DAG. * * *New in version 2.9.0* */ @@ -5045,7 +5045,7 @@ export interface operations { 403: components["responses"]["PermissionDenied"]; }; }; - get_datasets: { + get_assets: { parameters: { query: { /** The numbers of items to return. */ @@ -5059,7 +5059,7 @@ export interface operations { * *New in version 2.1.0* */ order_by?: components["parameters"]["OrderBy"]; - /** If set, only return datasets with uris matching this pattern. */ + /** If set, only return assets with uris matching this pattern. */ uri_pattern?: string; /** * One or more DAG IDs separated by commas to filter datasets by associated DAGs either consuming or producing. @@ -5115,8 +5115,8 @@ export interface operations { * *New in version 2.1.0* */ order_by?: components["parameters"]["OrderBy"]; - /** The Dataset ID that updated the dataset. */ - dataset_id?: components["parameters"]["FilterAssetID"]; + /** The Asset ID that updated the asset. */ + asset_id?: components["parameters"]["FilterAssetID"]; /** The DAG ID that updated the dataset. */ source_dag_id?: components["parameters"]["FilterSourceDAGID"]; /** The task ID that updated the dataset. */ @@ -5640,19 +5640,19 @@ export type SetDagRunNoteVariables = CamelCasedPropertiesDeep< operations["set_dag_run_note"]["parameters"]["path"] & operations["set_dag_run_note"]["requestBody"]["content"]["application/json"] >; -export type GetDagDatasetQueuedEventVariables = CamelCasedPropertiesDeep< +export type GetDagAssetQueuedEventVariables = CamelCasedPropertiesDeep< operations["get_dag_asset_queued_event"]["parameters"]["path"] & operations["get_dag_asset_queued_event"]["parameters"]["query"] >; -export type DeleteDagDatasetQueuedEventVariables = CamelCasedPropertiesDeep< +export type DeleteDagAssetQueuedEventVariables = CamelCasedPropertiesDeep< operations["delete_dag_asset_queued_event"]["parameters"]["path"] & operations["delete_dag_asset_queued_event"]["parameters"]["query"] >; -export type GetDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< +export type GetDagAssetQueuedEventsVariables = CamelCasedPropertiesDeep< operations["get_dag_asset_queued_events"]["parameters"]["path"] & operations["get_dag_asset_queued_events"]["parameters"]["query"] >; -export type DeleteDagDatasetQueuedEventsVariables = CamelCasedPropertiesDeep< +export type DeleteDagAssetQueuedEventsVariables = CamelCasedPropertiesDeep< operations["delete_dag_asset_queued_events"]["parameters"]["path"] & operations["delete_dag_asset_queued_events"]["parameters"]["query"] >; @@ -5787,13 +5787,13 @@ export type GetDagSourceVariables = CamelCasedPropertiesDeep< export type GetDagWarningsVariables = CamelCasedPropertiesDeep< operations["get_dag_warnings"]["parameters"]["query"] >; -export type GetDatasetsVariables = CamelCasedPropertiesDeep< - operations["get_datasets"]["parameters"]["query"] +export type GetAssetsVariables = CamelCasedPropertiesDeep< + operations["get_assets"]["parameters"]["query"] >; export type GetDatasetVariables = CamelCasedPropertiesDeep< operations["get_dataset"]["parameters"]["path"] >; -export type GetDatasetEventsVariables = CamelCasedPropertiesDeep< +export type GetAssetEventsVariables = CamelCasedPropertiesDeep< operations["get_asset_events"]["parameters"]["query"] >; export type CreateAssetEventVariables = CamelCasedPropertiesDeep< diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 69ed4afd3b79b..0d10fd09d72fe 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -81,7 +81,7 @@ - + diff --git a/clients/python/README.md b/clients/python/README.md index d2a458fe7eeb1..8989363149118 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -350,7 +350,7 @@ Class | Method | HTTP request | Description *DagWarningApi* | [**get_dag_warnings**](docs/DagWarningApi.md#get_dag_warnings) | **GET** /dagWarnings | List dag warnings *DatasetApi* | [**get_dataset**](docs/DatasetApi.md#get_dataset) | **GET** /datasets/{uri} | Get a dataset *AssetApi* | [**get_asset_events**](docs/DatasetApi.md#get_asset_events) | **GET** /datasets/events | Get asset events -*DatasetApi* | [**get_datasets**](docs/DatasetApi.md#get_datasets) | **GET** /datasets | List datasets +*DatasetApi* | [**get_assets**](docs/DatasetApi.md#get_assets) | **GET** /datasets | List assets *DatasetApi* | [**get_upstream_dataset_events**](docs/DatasetApi.md#get_upstream_dataset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents | Get dataset events for a DAG run *EventLogApi* | [**get_event_log**](docs/EventLogApi.md#get_event_log) | **GET** /eventLogs/{event_log_id} | Get a log entry *EventLogApi* | [**get_event_logs**](docs/EventLogApi.md#get_event_logs) | **GET** /eventLogs | List log entries From 428d136ddbb437755c0d09a77640acc049fbcff1 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 19:19:16 +0900 Subject: [PATCH 15/43] feat(api_connexion): rename get_dataset as get_asset --- airflow/api_connexion/endpoints/asset_endpoint.py | 4 ++-- airflow/api_connexion/openapi/v1.yaml | 2 +- airflow/www/static/js/types/api-generated.ts | 8 ++++---- airflow/www/templates/airflow/datasets.html | 2 +- clients/python/README.md | 2 +- tests/api_connexion/endpoints/test_asset_endpoint.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index a1edba826f195..8c02e4e6f0d47 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -62,8 +62,8 @@ @security.requires_access_asset("GET") @provide_session -def get_dataset(*, uri: str, session: Session = NEW_SESSION) -> APIResponse: - """Get an asset .""" +def get_asset(*, uri: str, session: Session = NEW_SESSION) -> APIResponse: + """Get an asset.""" asset = session.scalar( select(AssetModel) .where(AssetModel.uri == uri) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 27c461e784f57..b6b12e59bb6b2 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2562,7 +2562,7 @@ paths: summary: Get a dataset description: Get a dataset by uri. x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint - operationId: get_dataset + operationId: get_asset tags: [Asset] responses: "200": diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 7e85b3cf95afa..8235a66f6748e 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -787,7 +787,7 @@ export interface paths { }; "/datasets/{uri}": { /** Get a dataset by uri. */ - get: operations["get_dataset"]; + get: operations["get_asset"]; parameters: { path: { /** The encoded Dataset URI */ @@ -5081,7 +5081,7 @@ export interface operations { }; }; /** Get a dataset by uri. */ - get_dataset: { + get_asset: { parameters: { path: { /** The encoded Dataset URI */ @@ -5790,8 +5790,8 @@ export type GetDagWarningsVariables = CamelCasedPropertiesDeep< export type GetAssetsVariables = CamelCasedPropertiesDeep< operations["get_assets"]["parameters"]["query"] >; -export type GetDatasetVariables = CamelCasedPropertiesDeep< - operations["get_dataset"]["parameters"]["path"] +export type GetAssetVariables = CamelCasedPropertiesDeep< + operations["get_asset"]["parameters"]["path"] >; export type GetAssetEventsVariables = CamelCasedPropertiesDeep< operations["get_asset_events"]["parameters"]["query"] diff --git a/airflow/www/templates/airflow/datasets.html b/airflow/www/templates/airflow/datasets.html index 2cf90e5b95c39..4a98b8f31d688 100644 --- a/airflow/www/templates/airflow/datasets.html +++ b/airflow/www/templates/airflow/datasets.html @@ -24,7 +24,7 @@ {% block head_meta %} {{ super() }} - + diff --git a/clients/python/README.md b/clients/python/README.md index 8989363149118..050054b383c9f 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -348,7 +348,7 @@ Class | Method | HTTP request | Description *DAGRunApi* | [**set_dag_run_note**](docs/DAGRunApi.md#set_dag_run_note) | **PATCH** /dags/{dag_id}/dagRuns/{dag_run_id}/setNote | Update the DagRun note. *DAGRunApi* | [**update_dag_run_state**](docs/DAGRunApi.md#update_dag_run_state) | **PATCH** /dags/{dag_id}/dagRuns/{dag_run_id} | Modify a DAG run *DagWarningApi* | [**get_dag_warnings**](docs/DagWarningApi.md#get_dag_warnings) | **GET** /dagWarnings | List dag warnings -*DatasetApi* | [**get_dataset**](docs/DatasetApi.md#get_dataset) | **GET** /datasets/{uri} | Get a dataset +*AssetApi* | [**get_asset**](docs/DatasetApi.md#get_asset) | **GET** /datasets/{uri} | Get an asset *AssetApi* | [**get_asset_events**](docs/DatasetApi.md#get_asset_events) | **GET** /datasets/events | Get asset events *DatasetApi* | [**get_assets**](docs/DatasetApi.md#get_assets) | **GET** /datasets | List assets *DatasetApi* | [**get_upstream_dataset_events**](docs/DatasetApi.md#get_upstream_dataset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents | Get dataset events for a DAG run diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index f4902ab7d3908..b3154e56b542b 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -416,7 +416,7 @@ def test_should_respond_200(self, session): @pytest.mark.parametrize( "attr, value", [ - ("dataset_id", "2"), + ("asset_id", "2"), ("source_dag_id", "dag2"), ("source_task_id", "task2"), ("source_run_id", "run2"), From 9bc33362fac3ab5ef92f91594aea37149102a197 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 30 Sep 2024 21:16:39 +0900 Subject: [PATCH 16/43] feat(api_connexion/openapi): update api docs --- airflow/api_connexion/openapi/v1.yaml | 14 +++++++------- airflow/www/static/js/types/api-generated.ts | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index b6b12e59bb6b2..4a822db5bc876 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1376,9 +1376,9 @@ paths: - $ref: "#/components/parameters/DatasetURI" get: - summary: Get queued Dataset events for a Dataset. + summary: Get queued Asset events for an Asset. description: | - Get queued Dataset events for a Dataset + Get queued Asset events for an Asset *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint @@ -1401,9 +1401,9 @@ paths: $ref: "#/components/responses/NotFound" delete: - summary: Delete queued Dataset events for a Dataset. + summary: Delete queued Asset events for an Asset. description: | - Delete queued Dataset events for a Dataset. + Delete queued Asset events for a Asset. *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint @@ -2540,7 +2540,7 @@ paths: type: string required: false description: | - One or more DAG IDs separated by commas to filter datasets by associated DAGs either consuming or producing. + One or more DAG IDs separated by commas to filter assets by associated DAGs either consuming or producing. *New in version 2.9.0* responses: @@ -2580,8 +2580,8 @@ paths: /datasets/events: get: - summary: Get dataset events - description: Get dataset events + summary: Get asset events + description: Get asset events x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_asset_events tags: [Asset] diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 8235a66f6748e..a0d121316e34a 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -352,13 +352,13 @@ export interface paths { }; "/datasets/queuedEvent/{uri}": { /** - * Get queued Dataset events for a Dataset + * Get queued Asset events for an Asset * * *New in version 2.9.0* */ get: operations["get_asset_queued_events"]; /** - * Delete queued Dataset events for a Dataset. + * Delete queued Asset events for a Asset. * * *New in version 2.9.0* */ @@ -796,7 +796,7 @@ export interface paths { }; }; "/datasets/events": { - /** Get dataset events */ + /** Get asset events */ get: operations["get_asset_events"]; /** Create asset event */ post: operations["create_asset_event"]; @@ -3780,7 +3780,7 @@ export interface operations { }; }; /** - * Get queued Dataset events for a Dataset + * Get queued Asset events for an Asset * * *New in version 2.9.0* */ @@ -3808,7 +3808,7 @@ export interface operations { }; }; /** - * Delete queued Dataset events for a Dataset. + * Delete queued Asset events for a Asset. * * *New in version 2.9.0* */ @@ -5062,7 +5062,7 @@ export interface operations { /** If set, only return assets with uris matching this pattern. */ uri_pattern?: string; /** - * One or more DAG IDs separated by commas to filter datasets by associated DAGs either consuming or producing. + * One or more DAG IDs separated by commas to filter assets by associated DAGs either consuming or producing. * * *New in version 2.9.0* */ @@ -5100,7 +5100,7 @@ export interface operations { 404: components["responses"]["NotFound"]; }; }; - /** Get dataset events */ + /** Get asset events */ get_asset_events: { parameters: { query: { From 7ed254358faf96cb1a34b8aa6a219d39d4054adb Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:17:02 +0900 Subject: [PATCH 17/43] feat(js): rename DatasetEvents as AssetEvents --- .../datasets/{DatasetEvents.tsx => AssetEvents.tsx} | 0 airflow/www/static/js/datasets/DatasetDetails.tsx | 2 +- airflow/www/static/js/datasets/Main.tsx | 2 +- tests/api_connexion/endpoints/test_asset_endpoint.py | 12 ++++++------ 4 files changed, 8 insertions(+), 8 deletions(-) rename airflow/www/static/js/datasets/{DatasetEvents.tsx => AssetEvents.tsx} (100%) diff --git a/airflow/www/static/js/datasets/DatasetEvents.tsx b/airflow/www/static/js/datasets/AssetEvents.tsx similarity index 100% rename from airflow/www/static/js/datasets/DatasetEvents.tsx rename to airflow/www/static/js/datasets/AssetEvents.tsx diff --git a/airflow/www/static/js/datasets/DatasetDetails.tsx b/airflow/www/static/js/datasets/DatasetDetails.tsx index 8479d97a5bfe4..39930b522d269 100644 --- a/airflow/www/static/js/datasets/DatasetDetails.tsx +++ b/airflow/www/static/js/datasets/DatasetDetails.tsx @@ -33,7 +33,7 @@ import { useDataset } from "src/api"; import { getMetaValue } from "src/utils"; import RenderedJsonField from "src/components/RenderedJsonField"; -import Events from "./DatasetEvents"; +import Events from "./AssetEvents"; const gridUrl = getMetaValue("grid_url"); diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index 3d165a7ebfe3d..ada7ea8062a53 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -47,7 +47,7 @@ import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; import Tooltip from "src/components/Tooltip"; import { useContainerRef } from "src/context/containerRef"; -import DatasetEvents from "./DatasetEvents"; +import DatasetEvents from "./AssetEvents"; import DatasetsList from "./DatasetsList"; import DatasetDetails from "./DatasetDetails"; import type { OnSelectProps } from "./types"; diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index b3154e56b542b..0b8d608018361 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -396,7 +396,7 @@ def test_should_respond_200(self, session): assert response.status_code == 200 response_data = response.json assert response_data == { - "dataset_events": [ + "asset_events": [ { "id": 1, "timestamp": self.default_time, @@ -460,7 +460,7 @@ def test_filtering(self, attr, value, session): assert response.status_code == 200 response_data = response.json assert response_data == { - "dataset_events": [ + "asset_events": [ { "id": 2, "dataset_id": 2, @@ -538,7 +538,7 @@ def test_includes_created_dagrun(self, session): assert response.status_code == 200 response_data = response.json assert response_data == { - "dataset_events": [ + "asset_events": [ { "id": 1, "dataset_id": 1, @@ -685,7 +685,7 @@ def test_limit_and_offset(self, url, expected_event_runids, session): response = self.client.get(url, environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 - event_runids = [event["source_run_id"] for event in response.json["dataset_events"]] + event_runids = [event["source_run_id"] for event in response.json["asset_events"]] assert event_runids == expected_event_runids def test_should_respect_page_size_limit_default(self, session): @@ -707,7 +707,7 @@ def test_should_respect_page_size_limit_default(self, session): response = self.client.get("/api/v1/datasets/events", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 - assert len(response.json["dataset_events"]) == 100 + assert len(response.json["asset_events"]) == 100 @conf_vars({("api", "maximum_page_limit"): "150"}) def test_should_return_conf_max_if_req_max_above_conf(self, session): @@ -731,7 +731,7 @@ def test_should_return_conf_max_if_req_max_above_conf(self, session): ) assert response.status_code == 200 - assert len(response.json["dataset_events"]) == 150 + assert len(response.json["asset_events"]) == 150 class TestQueuedEventEndpoint(TestDatasetEndpoint): From 7d16068910c163029ae24176556d42b16da40a56 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:19:24 +0900 Subject: [PATCH 18/43] feat(js): rename DatasetDetails as AssetDetails --- .../js/datasets/{DatasetDetails.tsx => AssetDetails.tsx} | 4 ++-- airflow/www/static/js/datasets/Main.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename airflow/www/static/js/datasets/{DatasetDetails.tsx => AssetDetails.tsx} (97%) diff --git a/airflow/www/static/js/datasets/DatasetDetails.tsx b/airflow/www/static/js/datasets/AssetDetails.tsx similarity index 97% rename from airflow/www/static/js/datasets/DatasetDetails.tsx rename to airflow/www/static/js/datasets/AssetDetails.tsx index 39930b522d269..351fb1fbbd220 100644 --- a/airflow/www/static/js/datasets/DatasetDetails.tsx +++ b/airflow/www/static/js/datasets/AssetDetails.tsx @@ -41,7 +41,7 @@ interface Props { uri: string; } -const DatasetDetails = ({ uri }: Props) => { +const AssetDetails = ({ uri }: Props) => { const { data: dataset, isLoading } = useDataset({ uri }); const hasProducingTasks = !!dataset?.producingTasks?.length; @@ -108,4 +108,4 @@ const DatasetDetails = ({ uri }: Props) => { ); }; -export default DatasetDetails; +export default AssetDetails; diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index ada7ea8062a53..f09c2abccfc93 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -49,7 +49,7 @@ import { useContainerRef } from "src/context/containerRef"; import DatasetEvents from "./AssetEvents"; import DatasetsList from "./DatasetsList"; -import DatasetDetails from "./DatasetDetails"; +import AssetDetails from "./AssetDetails"; import type { OnSelectProps } from "./types"; import Graph from "./Graph"; import SearchBar from "./SearchBar"; @@ -245,7 +245,7 @@ const Datasets = () => { )} {!!selectedUri && ( - + )} From 45db9c13afb7d6dcae7a7615d8292a372cbb9c05 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:20:56 +0900 Subject: [PATCH 19/43] feat(js): rename DatasetList as AssetList --- .../datasets/{DatasetsList.test.tsx => AssetList.test.tsx} | 6 +++--- .../static/js/datasets/{DatasetsList.tsx => AssetsList.tsx} | 4 ++-- airflow/www/static/js/datasets/Main.tsx | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename airflow/www/static/js/datasets/{DatasetsList.test.tsx => AssetList.test.tsx} (96%) rename airflow/www/static/js/datasets/{DatasetsList.tsx => AssetsList.tsx} (98%) diff --git a/airflow/www/static/js/datasets/DatasetsList.test.tsx b/airflow/www/static/js/datasets/AssetList.test.tsx similarity index 96% rename from airflow/www/static/js/datasets/DatasetsList.test.tsx rename to airflow/www/static/js/datasets/AssetList.test.tsx index 89d0e9f490573..2dadbc2ae2a73 100644 --- a/airflow/www/static/js/datasets/DatasetsList.test.tsx +++ b/airflow/www/static/js/datasets/AssetList.test.tsx @@ -27,7 +27,7 @@ import { Wrapper } from "src/utils/testUtils"; import type { UseQueryResult } from "react-query"; import type { DatasetListItem } from "src/types"; -import DatasetsList from "./DatasetsList"; +import AssetsList from "./AssetsList"; const datasets = [ { @@ -87,7 +87,7 @@ describe("Test Datasets List", () => { .mockImplementation(() => returnValue); const { getByText, queryAllByTestId } = render( - {}} />, + {}} />, { wrapper: Wrapper } ); @@ -111,7 +111,7 @@ describe("Test Datasets List", () => { .mockImplementation(() => emptyReturnValue); const { getByText, queryAllByTestId, getByTestId } = render( - {}} />, + {}} />, { wrapper: Wrapper } ); diff --git a/airflow/www/static/js/datasets/DatasetsList.tsx b/airflow/www/static/js/datasets/AssetsList.tsx similarity index 98% rename from airflow/www/static/js/datasets/DatasetsList.tsx rename to airflow/www/static/js/datasets/AssetsList.tsx index ffc0c4a08e7fc..05e9b065d6cb1 100644 --- a/airflow/www/static/js/datasets/DatasetsList.tsx +++ b/airflow/www/static/js/datasets/AssetsList.tsx @@ -56,7 +56,7 @@ const dateOptions: Record = { hour: { count: 1, unit: "hour" }, }; -const DatasetsList = ({ onSelect }: Props) => { +const AssetsList = ({ onSelect }: Props) => { const limit = 25; const [offset, setOffset] = useState(0); @@ -175,4 +175,4 @@ const DatasetsList = ({ onSelect }: Props) => { ); }; -export default DatasetsList; +export default AssetsList; diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index f09c2abccfc93..3c06922fb6c7a 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -48,7 +48,7 @@ import Tooltip from "src/components/Tooltip"; import { useContainerRef } from "src/context/containerRef"; import DatasetEvents from "./AssetEvents"; -import DatasetsList from "./DatasetsList"; +import AssetsList from "./AssetsList"; import AssetDetails from "./AssetDetails"; import type { OnSelectProps } from "./types"; import Graph from "./Graph"; @@ -275,7 +275,7 @@ const Datasets = () => { {!selectedUri && ( - + )} From 9c887dc0749d65a4f1fa1d2a9c1947558b3fbf7a Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:27:06 +0900 Subject: [PATCH 20/43] feat(js/api): rename useUpstreamDatasetEvents as useUpstreamAssetEvents --- airflow/www/static/js/api/index.ts | 4 ++-- ...{useUpstreamDatasetEvents.ts => useUpstreamAssetEvents.ts} | 4 ++-- .../www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx | 4 ++-- airflow/www/static/js/dag/details/graph/index.tsx | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) rename airflow/www/static/js/api/{useUpstreamDatasetEvents.ts => useUpstreamAssetEvents.ts} (93%) diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index 09557dd55cc2a..888fc30a08c57 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -39,7 +39,7 @@ import useDatasetDependencies from "./useDatasetDependencies"; import useDatasetEvents from "./useDatasetEvents"; import useSetDagRunNote from "./useSetDagRunNote"; import useSetTaskInstanceNote from "./useSetTaskInstanceNote"; -import useUpstreamDatasetEvents from "./useUpstreamDatasetEvents"; +import useUpstreamAssetEvents from "./useUpstreamAssetEvents"; import useTaskInstance from "./useTaskInstance"; import useTaskFailedDependency from "./useTaskFailedDependency"; import useDag from "./useDag"; @@ -105,7 +105,7 @@ export { useSetDagRunNote, useSetTaskInstanceNote, useTaskInstance, - useUpstreamDatasetEvents, + useUpstreamAssetEvents, useHistoricalMetricsData, useTaskXcomEntry, useTaskXcomCollection, diff --git a/airflow/www/static/js/api/useUpstreamDatasetEvents.ts b/airflow/www/static/js/api/useUpstreamAssetEvents.ts similarity index 93% rename from airflow/www/static/js/api/useUpstreamDatasetEvents.ts rename to airflow/www/static/js/api/useUpstreamAssetEvents.ts index 32d1c7aeff2d8..20086af0c3bff 100644 --- a/airflow/www/static/js/api/useUpstreamDatasetEvents.ts +++ b/airflow/www/static/js/api/useUpstreamAssetEvents.ts @@ -30,7 +30,7 @@ interface Props extends GetUpstreamDatasetEventsVariables { options?: UseQueryOptions; } -const useUpstreamDatasetEvents = ({ dagId, dagRunId, options }: Props) => { +const useUpstreamAssetEvents = ({ dagId, dagRunId, options }: Props) => { const upstreamEventsUrl = ( getMetaValue("upstream_dataset_events_api") || `api/v1/dags/${dagId}/dagRuns/_DAG_RUN_ID_/upstreamDatasetEvents` @@ -48,4 +48,4 @@ const useUpstreamDatasetEvents = ({ dagId, dagRunId, options }: Props) => { }; }; -export default useUpstreamDatasetEvents; +export default useUpstreamAssetEvents; diff --git a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx index 5fa585830b437..b3e94991ee75e 100644 --- a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx +++ b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx @@ -19,7 +19,7 @@ import React, { useMemo } from "react"; import { Box, Text } from "@chakra-ui/react"; -import { useUpstreamDatasetEvents } from "src/api"; +import { useUpstreamAssetEvents } from "src/api"; import type { DagRun as DagRunType } from "src/types"; import { CardDef, CardList } from "src/components/Table"; import type { DatasetEvent } from "src/types/api-generated"; @@ -42,7 +42,7 @@ const DatasetTriggerEvents = ({ runId }: Props) => { const { data: { datasetEvents = [] }, isLoading, - } = useUpstreamDatasetEvents({ dagRunId: runId, dagId }); + } = useUpstreamAssetEvents({ dagRunId: runId, dagId }); const columns = useMemo( () => [ diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index edafc99fe9c34..2df5fcd077ad3 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -36,7 +36,7 @@ import { useDatasets, useGraphData, useGridData, - useUpstreamDatasetEvents, + useUpstreamAssetEvents, } from "src/api"; import useSelection from "src/dag/useSelection"; import { getMetaValue, getTask, useOffsetTop } from "src/utils"; @@ -165,7 +165,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { const { data: { datasetEvents: upstreamDatasetEvents = [] }, - } = useUpstreamDatasetEvents({ + } = useUpstreamAssetEvents({ dagId, dagRunId: selected.runId || "", options: { From 4fcabb5c2ebaf67886fc26fa6f53a0bd4bb91933 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:29:39 +0900 Subject: [PATCH 21/43] feat(js/api): rename useDatasetsSummary as useAssetsSummary --- airflow/www/static/js/api/index.ts | 4 ++-- .../js/api/{useDatasetsSummary.ts => useAssetsSummary.ts} | 2 +- airflow/www/static/js/datasets/AssetList.test.tsx | 2 +- airflow/www/static/js/datasets/AssetsList.tsx | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) rename airflow/www/static/js/api/{useDatasetsSummary.ts => useAssetsSummary.ts} (98%) diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index 888fc30a08c57..6817e154e3d63 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -33,7 +33,7 @@ import useGraphData from "./useGraphData"; import useGridData from "./useGridData"; import useMappedInstances from "./useMappedInstances"; import useDatasets from "./useDatasets"; -import useDatasetsSummary from "./useDatasetsSummary"; +import useAssetsSummary from "./useAssetsSummary"; import useDataset from "./useDataset"; import useDatasetDependencies from "./useDatasetDependencies"; import useDatasetEvents from "./useDatasetEvents"; @@ -89,7 +89,7 @@ export { useDatasets, useDatasetDependencies, useDatasetEvents, - useDatasetsSummary, + useAssetsSummary, useExtraLinks, useGraphData, useGridData, diff --git a/airflow/www/static/js/api/useDatasetsSummary.ts b/airflow/www/static/js/api/useAssetsSummary.ts similarity index 98% rename from airflow/www/static/js/api/useDatasetsSummary.ts rename to airflow/www/static/js/api/useAssetsSummary.ts index 6f902946f6296..66b56ca9f6925 100644 --- a/airflow/www/static/js/api/useDatasetsSummary.ts +++ b/airflow/www/static/js/api/useAssetsSummary.ts @@ -42,7 +42,7 @@ interface Props { updatedAfter?: DateOption; } -export default function useDatasetsSummary({ +export default function useAssetsSummary({ limit, offset, order, diff --git a/airflow/www/static/js/datasets/AssetList.test.tsx b/airflow/www/static/js/datasets/AssetList.test.tsx index 2dadbc2ae2a73..a7e9915bef39e 100644 --- a/airflow/www/static/js/datasets/AssetList.test.tsx +++ b/airflow/www/static/js/datasets/AssetList.test.tsx @@ -22,7 +22,7 @@ import React from "react"; import { render } from "@testing-library/react"; -import * as useDatasetsModule from "src/api/useDatasetsSummary"; +import * as useDatasetsModule from "src/api/useAssetsSummary"; import { Wrapper } from "src/utils/testUtils"; import type { UseQueryResult } from "react-query"; diff --git a/airflow/www/static/js/datasets/AssetsList.tsx b/airflow/www/static/js/datasets/AssetsList.tsx index 05e9b065d6cb1..ff1867e44cc57 100644 --- a/airflow/www/static/js/datasets/AssetsList.tsx +++ b/airflow/www/static/js/datasets/AssetsList.tsx @@ -23,11 +23,11 @@ import { snakeCase } from "lodash"; import type { Row, SortingRule } from "react-table"; import { useSearchParams } from "react-router-dom"; -import { useDatasetsSummary } from "src/api"; +import { useAssetsSummary } from "src/api"; import { CellProps, Table, TimeCell } from "src/components/Table"; import type { API } from "src/types"; import { getMetaValue } from "src/utils"; -import type { DateOption } from "src/api/useDatasetsSummary"; +import type { DateOption } from "src/api/useAssetsSummary"; import type { OnSelectProps } from "./types"; @@ -73,7 +73,7 @@ const AssetsList = ({ onSelect }: Props) => { const { data: { datasets, totalEntries }, isLoading, - } = useDatasetsSummary({ + } = useAssetsSummary({ limit, offset, order, From 00cd6753d0773186b65e73b9228d438cf9c21d38 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:35:20 +0900 Subject: [PATCH 22/43] feat(js/api): rename useDatasetDependencies as useAssetDependencies --- airflow/www/static/js/api/index.ts | 4 ++-- .../{useDatasetDependencies.ts => useAssetDependencies.ts} | 4 ++-- airflow/www/static/js/datasets/Graph/index.tsx | 2 +- airflow/www/static/js/datasets/Main.tsx | 4 ++-- airflow/www/static/js/datasets/SearchBar.tsx | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename airflow/www/static/js/api/{useDatasetDependencies.ts => useAssetDependencies.ts} (96%) diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index 6817e154e3d63..5dca18bf3a2b4 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -35,7 +35,7 @@ import useMappedInstances from "./useMappedInstances"; import useDatasets from "./useDatasets"; import useAssetsSummary from "./useAssetsSummary"; import useDataset from "./useDataset"; -import useDatasetDependencies from "./useDatasetDependencies"; +import useAssetDependencies from "./useAssetDependencies"; import useDatasetEvents from "./useDatasetEvents"; import useSetDagRunNote from "./useSetDagRunNote"; import useSetTaskInstanceNote from "./useSetTaskInstanceNote"; @@ -87,7 +87,7 @@ export { useDags, useDataset, useDatasets, - useDatasetDependencies, + useAssetDependencies, useDatasetEvents, useAssetsSummary, useExtraLinks, diff --git a/airflow/www/static/js/api/useDatasetDependencies.ts b/airflow/www/static/js/api/useAssetDependencies.ts similarity index 96% rename from airflow/www/static/js/api/useDatasetDependencies.ts rename to airflow/www/static/js/api/useAssetDependencies.ts index d2ba627f64458..05597cdd0a36e 100644 --- a/airflow/www/static/js/api/useDatasetDependencies.ts +++ b/airflow/www/static/js/api/useAssetDependencies.ts @@ -82,7 +82,7 @@ const formatDependencies = async ({ edges, nodes }: DatasetDependencies) => { return graph as DatasetGraph; }; -export default function useDatasetDependencies() { +export default function useAssetDependencies() { return useQuery("datasetDependencies", async () => { const datasetDepsUrl = getMetaValue("dataset_dependencies_url"); return axios.get(datasetDepsUrl); @@ -90,7 +90,7 @@ export default function useDatasetDependencies() { } export const useDatasetGraphs = () => { - const { data: datasetDependencies } = useDatasetDependencies(); + const { data: datasetDependencies } = useAssetDependencies(); return useQuery(["datasetGraphs", datasetDependencies], () => { if (datasetDependencies) { return formatDependencies(datasetDependencies); diff --git a/airflow/www/static/js/datasets/Graph/index.tsx b/airflow/www/static/js/datasets/Graph/index.tsx index e960c48ff63a2..5725fb6f00cb9 100644 --- a/airflow/www/static/js/datasets/Graph/index.tsx +++ b/airflow/www/static/js/datasets/Graph/index.tsx @@ -34,7 +34,7 @@ import { RiFocus3Line } from "react-icons/ri"; import Edge from "src/components/Graph/Edge"; import { useContainerRef } from "src/context/containerRef"; -import { useDatasetGraphs } from "src/api/useDatasetDependencies"; +import { useDatasetGraphs } from "src/api/useAssetDependencies"; import Node, { CustomNodeProps } from "./Node"; import Legend from "./Legend"; diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index 3c06922fb6c7a..9f662d66b98e9 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -42,7 +42,7 @@ import { MdEvent, MdAccountTree, MdDetails, MdPlayArrow } from "react-icons/md"; import Time from "src/components/Time"; import BreadcrumbText from "src/components/BreadcrumbText"; import { useOffsetTop } from "src/utils"; -import { useDatasetDependencies } from "src/api"; +import { useAssetDependencies } from "src/api"; import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; import Tooltip from "src/components/Tooltip"; import { useContainerRef } from "src/context/containerRef"; @@ -92,7 +92,7 @@ const Datasets = () => { const offsetTop = useOffsetTop(contentRef); const height = `calc(100vh - ${offsetTop + 100}px)`; - const { data: datasetDependencies, isLoading } = useDatasetDependencies(); + const { data: datasetDependencies, isLoading } = useAssetDependencies(); const [searchParams, setSearchParams] = useSearchParams(); const { isOpen, onToggle, onClose } = useDisclosure(); diff --git a/airflow/www/static/js/datasets/SearchBar.tsx b/airflow/www/static/js/datasets/SearchBar.tsx index 33476b8419fdb..b54f4a66a885b 100644 --- a/airflow/www/static/js/datasets/SearchBar.tsx +++ b/airflow/www/static/js/datasets/SearchBar.tsx @@ -20,7 +20,7 @@ import React from "react"; import { Select, SingleValue, useChakraSelectProps } from "chakra-react-select"; -import type { DatasetDependencies } from "src/api/useDatasetDependencies"; +import type { DatasetDependencies } from "src/api/useAssetDependencies"; import type { OnSelectProps } from "./types"; interface Props { From 5931131361e06fb7959b2c58b4357e9cc9a9e1f1 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:36:36 +0900 Subject: [PATCH 23/43] feat(js/api): rename useDatasetEvents as useAssetEvents --- airflow/www/static/js/api/index.ts | 4 ++-- .../static/js/api/{useDatasetEvents.ts => useAssetEvents.ts} | 4 ++-- airflow/www/static/js/dag/details/graph/index.tsx | 4 ++-- .../js/dag/details/taskInstance/DatasetUpdateEvents.tsx | 4 ++-- airflow/www/static/js/datasets/AssetEvents.tsx | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) rename airflow/www/static/js/api/{useDatasetEvents.ts => useAssetEvents.ts} (97%) diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index 5dca18bf3a2b4..342175a966777 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -36,7 +36,7 @@ import useDatasets from "./useDatasets"; import useAssetsSummary from "./useAssetsSummary"; import useDataset from "./useDataset"; import useAssetDependencies from "./useAssetDependencies"; -import useDatasetEvents from "./useDatasetEvents"; +import useAssetEvents from "./useAssetEvents"; import useSetDagRunNote from "./useSetDagRunNote"; import useSetTaskInstanceNote from "./useSetTaskInstanceNote"; import useUpstreamAssetEvents from "./useUpstreamAssetEvents"; @@ -88,7 +88,7 @@ export { useDataset, useDatasets, useAssetDependencies, - useDatasetEvents, + useAssetEvents, useAssetsSummary, useExtraLinks, useGraphData, diff --git a/airflow/www/static/js/api/useDatasetEvents.ts b/airflow/www/static/js/api/useAssetEvents.ts similarity index 97% rename from airflow/www/static/js/api/useDatasetEvents.ts rename to airflow/www/static/js/api/useAssetEvents.ts index 13b8a78b40636..ff8425e5092a7 100644 --- a/airflow/www/static/js/api/useDatasetEvents.ts +++ b/airflow/www/static/js/api/useAssetEvents.ts @@ -31,7 +31,7 @@ interface Props extends GetAssetEventsVariables { options?: UseQueryOptions; } -const useDatasetEvents = ({ +const useAssetEvents = ({ assetId, sourceDagId, sourceRunId, @@ -84,4 +84,4 @@ const useDatasetEvents = ({ }; }; -export default useDatasetEvents; +export default useAssetEvents; diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index 2df5fcd077ad3..01220b9bb3b9c 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -32,7 +32,7 @@ import ReactFlow, { import { useDagDetails, - useDatasetEvents, + useAssetEvents, useDatasets, useGraphData, useGridData, @@ -176,7 +176,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { const { data: { datasetEvents: downstreamDatasetEvents = [] }, - } = useDatasetEvents({ + } = useAssetEvents({ sourceDagId: dagId, sourceRunId: selected.runId || undefined, options: { enabled: !!selected.runId && showDatasets }, diff --git a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx index 48c987a829601..3eb16380101a5 100644 --- a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx @@ -19,7 +19,7 @@ import React, { useMemo } from "react"; import { Box, Text } from "@chakra-ui/react"; -import { useDatasetEvents } from "src/api"; +import { useAssetEvents } from "src/api"; import type { DagRun as DagRunType } from "src/types"; import { getMetaValue } from "src/utils"; import { CardDef, CardList } from "src/components/Table"; @@ -41,7 +41,7 @@ const DatasetUpdateEvents = ({ runId, taskId }: Props) => { const { data: { datasetEvents = [] }, isLoading, - } = useDatasetEvents({ + } = useAssetEvents({ sourceDagId: dagId, sourceRunId: runId, sourceTaskId: taskId, diff --git a/airflow/www/static/js/datasets/AssetEvents.tsx b/airflow/www/static/js/datasets/AssetEvents.tsx index 26830485daf71..a651a9184843d 100644 --- a/airflow/www/static/js/datasets/AssetEvents.tsx +++ b/airflow/www/static/js/datasets/AssetEvents.tsx @@ -22,7 +22,7 @@ import { snakeCase } from "lodash"; import type { SortingRule } from "react-table"; import { Box, Flex, Heading, Select } from "@chakra-ui/react"; -import { useDatasetEvents } from "src/api"; +import { useAssetEvents } from "src/api"; import { CardList, type CardDef } from "src/components/Table"; import type { DatasetEvent } from "src/types/api-generated"; @@ -50,7 +50,7 @@ const Events = ({ assetId, showLabel }: Props) => { const { data: { datasetEvents = [], totalEntries = 0 }, isLoading: isEventsLoading, - } = useDatasetEvents({ + } = useAssetEvents({ assetId, limit, offset, From 93b52fc4cf7174a35cd0108177ce9fa39569058b Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:48:47 +0900 Subject: [PATCH 24/43] feat(js/api): rename useDatasets as useAssets --- airflow/www/static/js/api/index.ts | 4 ++-- .../static/js/api/{useDatasets.ts => useAssets.ts} | 2 +- airflow/www/static/js/dag/details/graph/index.tsx | 4 ++-- airflow/www/static/js/datasets/AssetList.test.tsx | 14 +++++++------- 4 files changed, 12 insertions(+), 12 deletions(-) rename airflow/www/static/js/api/{useDatasets.ts => useAssets.ts} (95%) diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index 342175a966777..838c91cb3cb9e 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -32,7 +32,7 @@ import useMarkTaskDryRun from "./useMarkTaskDryRun"; import useGraphData from "./useGraphData"; import useGridData from "./useGridData"; import useMappedInstances from "./useMappedInstances"; -import useDatasets from "./useDatasets"; +import useAssets from "./useAssets"; import useAssetsSummary from "./useAssetsSummary"; import useDataset from "./useDataset"; import useAssetDependencies from "./useAssetDependencies"; @@ -86,7 +86,7 @@ export { useDagRuns, useDags, useDataset, - useDatasets, + useAssets, useAssetDependencies, useAssetEvents, useAssetsSummary, diff --git a/airflow/www/static/js/api/useDatasets.ts b/airflow/www/static/js/api/useAssets.ts similarity index 95% rename from airflow/www/static/js/api/useDatasets.ts rename to airflow/www/static/js/api/useAssets.ts index db46415062c1a..b9af13ff65cb2 100644 --- a/airflow/www/static/js/api/useDatasets.ts +++ b/airflow/www/static/js/api/useAssets.ts @@ -28,7 +28,7 @@ interface Props { enabled?: boolean; } -export default function useDatasets({ dagIds, enabled = true }: Props) { +export default function useAssets({ dagIds, enabled = true }: Props) { return useQuery( ["datasets", dagIds], () => { diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index 01220b9bb3b9c..c3e9575ff5a4b 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -33,7 +33,7 @@ import ReactFlow, { import { useDagDetails, useAssetEvents, - useDatasets, + useAssets, useGraphData, useGridData, useUpstreamAssetEvents, @@ -150,7 +150,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { setArrange(data?.arrange || "LR"); }, [data?.arrange]); - const { data: datasetsCollection } = useDatasets({ + const { data: datasetsCollection } = useAssets({ dagIds: [dagId], }); diff --git a/airflow/www/static/js/datasets/AssetList.test.tsx b/airflow/www/static/js/datasets/AssetList.test.tsx index a7e9915bef39e..10bf0b45c82e9 100644 --- a/airflow/www/static/js/datasets/AssetList.test.tsx +++ b/airflow/www/static/js/datasets/AssetList.test.tsx @@ -22,7 +22,7 @@ import React from "react"; import { render } from "@testing-library/react"; -import * as useDatasetsModule from "src/api/useAssetsSummary"; +import * as useAssetsModule from "src/api/useAssetsSummary"; import { Wrapper } from "src/utils/testUtils"; import type { UseQueryResult } from "react-query"; @@ -59,8 +59,8 @@ const datasets = [ }, ]; -type UseDatasetsReturn = UseQueryResult & { - data: useDatasetsModule.DatasetsData; +type UseAssetsReturn = UseQueryResult & { + data: useAssetsModule.DatasetsData; }; const returnValue = { @@ -69,7 +69,7 @@ const returnValue = { totalEntries: datasets.length, }, isSuccess: true, -} as UseDatasetsReturn; +} as UseAssetsReturn; const emptyReturnValue = { data: { @@ -78,12 +78,12 @@ const emptyReturnValue = { }, isSuccess: true, isLoading: false, -} as UseDatasetsReturn; +} as UseAssetsReturn; describe("Test Datasets List", () => { test("Displays a list of datasets", () => { jest - .spyOn(useDatasetsModule, "default") + .spyOn(useAssetsModule, "default") .mockImplementation(() => returnValue); const { getByText, queryAllByTestId } = render( @@ -107,7 +107,7 @@ describe("Test Datasets List", () => { test("Empty state displays when there are no datasets", () => { jest - .spyOn(useDatasetsModule, "default") + .spyOn(useAssetsModule, "default") .mockImplementation(() => emptyReturnValue); const { getByText, queryAllByTestId, getByTestId } = render( From b9b28869a8b4443b6524021ace2a16aa4ad9d1d9 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 17:52:23 +0900 Subject: [PATCH 25/43] feat(js/api): rename useDataset as useAsset --- airflow/www/static/js/api/index.ts | 4 ++-- airflow/www/static/js/api/{useDataset.ts => useAsset.ts} | 2 +- airflow/www/static/js/api/useAssetDependencies.ts | 2 +- airflow/www/static/js/datasets/AssetDetails.tsx | 4 ++-- airflow/www/static/js/datasets/CreateAssetEvent.tsx | 4 ++-- airflow/www/static/js/datasets/Graph/index.tsx | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) rename airflow/www/static/js/api/{useDataset.ts => useAsset.ts} (95%) diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts index 838c91cb3cb9e..c2a9885b2c7ea 100644 --- a/airflow/www/static/js/api/index.ts +++ b/airflow/www/static/js/api/index.ts @@ -34,7 +34,7 @@ import useGridData from "./useGridData"; import useMappedInstances from "./useMappedInstances"; import useAssets from "./useAssets"; import useAssetsSummary from "./useAssetsSummary"; -import useDataset from "./useDataset"; +import useAsset from "./useAsset"; import useAssetDependencies from "./useAssetDependencies"; import useAssetEvents from "./useAssetEvents"; import useSetDagRunNote from "./useSetDagRunNote"; @@ -85,7 +85,7 @@ export { useDagDetails, useDagRuns, useDags, - useDataset, + useAsset, useAssets, useAssetDependencies, useAssetEvents, diff --git a/airflow/www/static/js/api/useDataset.ts b/airflow/www/static/js/api/useAsset.ts similarity index 95% rename from airflow/www/static/js/api/useDataset.ts rename to airflow/www/static/js/api/useAsset.ts index 4793464fac378..106be25aee3df 100644 --- a/airflow/www/static/js/api/useDataset.ts +++ b/airflow/www/static/js/api/useAsset.ts @@ -27,7 +27,7 @@ interface Props { uri: string; } -export default function useDataset({ uri }: Props) { +export default function useAsset({ uri }: Props) { return useQuery(["dataset", uri], () => { const datasetUrl = getMetaValue("dataset_api").replace( "__URI__", diff --git a/airflow/www/static/js/api/useAssetDependencies.ts b/airflow/www/static/js/api/useAssetDependencies.ts index 05597cdd0a36e..11e7219c53fc8 100644 --- a/airflow/www/static/js/api/useAssetDependencies.ts +++ b/airflow/www/static/js/api/useAssetDependencies.ts @@ -89,7 +89,7 @@ export default function useAssetDependencies() { }); } -export const useDatasetGraphs = () => { +export const useAssetGraphs = () => { const { data: datasetDependencies } = useAssetDependencies(); return useQuery(["datasetGraphs", datasetDependencies], () => { if (datasetDependencies) { diff --git a/airflow/www/static/js/datasets/AssetDetails.tsx b/airflow/www/static/js/datasets/AssetDetails.tsx index 351fb1fbbd220..d85637b3f4dc8 100644 --- a/airflow/www/static/js/datasets/AssetDetails.tsx +++ b/airflow/www/static/js/datasets/AssetDetails.tsx @@ -29,7 +29,7 @@ import { } from "@chakra-ui/react"; import { isEmpty } from "lodash"; -import { useDataset } from "src/api"; +import { useAsset } from "src/api"; import { getMetaValue } from "src/utils"; import RenderedJsonField from "src/components/RenderedJsonField"; @@ -42,7 +42,7 @@ interface Props { } const AssetDetails = ({ uri }: Props) => { - const { data: dataset, isLoading } = useDataset({ uri }); + const { data: dataset, isLoading } = useAsset({ uri }); const hasProducingTasks = !!dataset?.producingTasks?.length; const hasConsumingDags = !!dataset?.consumingDags?.length; diff --git a/airflow/www/static/js/datasets/CreateAssetEvent.tsx b/airflow/www/static/js/datasets/CreateAssetEvent.tsx index 69a78de8b7586..30e50721997fc 100644 --- a/airflow/www/static/js/datasets/CreateAssetEvent.tsx +++ b/airflow/www/static/js/datasets/CreateAssetEvent.tsx @@ -34,7 +34,7 @@ import { } from "@chakra-ui/react"; import { useContainerRef } from "src/context/containerRef"; -import { useCreateAssetEvent, useDataset } from "src/api"; +import { useCreateAssetEvent, useAsset } from "src/api"; interface Props { isOpen: boolean; @@ -54,7 +54,7 @@ function checkJsonString(str: string) { const CreateAssetEventModal = ({ uri, isOpen, onClose }: Props) => { const containerRef = useContainerRef(); const [extra, setExtra] = useState(""); - const { data: dataset } = useDataset({ uri }); + const { data: dataset } = useAsset({ uri }); const isJson = checkJsonString(extra); const isDisabled = !!extra && !isJson; diff --git a/airflow/www/static/js/datasets/Graph/index.tsx b/airflow/www/static/js/datasets/Graph/index.tsx index 5725fb6f00cb9..a7a4e95ec8fc9 100644 --- a/airflow/www/static/js/datasets/Graph/index.tsx +++ b/airflow/www/static/js/datasets/Graph/index.tsx @@ -34,7 +34,7 @@ import { RiFocus3Line } from "react-icons/ri"; import Edge from "src/components/Graph/Edge"; import { useContainerRef } from "src/context/containerRef"; -import { useDatasetGraphs } from "src/api/useAssetDependencies"; +import { useAssetGraphs } from "src/api/useAssetDependencies"; import Node, { CustomNodeProps } from "./Node"; import Legend from "./Legend"; @@ -53,7 +53,7 @@ const Graph = ({ selectedNodeId, onSelect }: Props) => { const { setCenter } = useReactFlow(); const containerRef = useContainerRef(); - const { data: graph } = useDatasetGraphs(); + const { data: graph } = useAssetGraphs(); const nodeColor = ({ data: { isSelected }, From 9899e465861da89b4e29936ed9d09f77d2690104 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 18:09:44 +0900 Subject: [PATCH 26/43] feat(api_connexion): rename get_upstream_dataset_events as get_upstream_asset_events --- .../api_connexion/endpoints/asset_endpoint.py | 2 +- .../endpoints/dag_run_endpoint.py | 8 +++----- airflow/api_connexion/openapi/v1.yaml | 18 +++++++++--------- airflow/api_connexion/schemas/asset_schema.py | 4 ++-- .../static/js/api/useUpstreamAssetEvents.ts | 8 ++++---- .../www/static/js/dag/details/graph/index.tsx | 6 +++--- airflow/www/static/js/types/api-generated.ts | 10 +++++----- airflow/www/templates/airflow/dag.html | 2 +- clients/python/README.md | 4 ++-- .../endpoints/test_dag_run_endpoint.py | 8 ++++---- .../schemas/test_dataset_schema.py | 4 ++-- 11 files changed, 36 insertions(+), 38 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 8c02e4e6f0d47..c027085691e8e 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -149,7 +149,7 @@ def get_asset_events( query = apply_sorting(query, order_by, {}, allowed_attrs) events = session.scalars(query.offset(offset).limit(limit)).all() return asset_event_collection_schema.dump( - AssetEventCollection(dataset_events=events, total_entries=total_entries) + AssetEventCollection(asset_events=events, total_entries=total_entries) ) diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py b/airflow/api_connexion/endpoints/dag_run_endpoint.py index 02d4663837f4e..44891c0ef2c84 100644 --- a/airflow/api_connexion/endpoints/dag_run_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py @@ -114,10 +114,8 @@ def get_dag_run( @security.requires_access_dag("GET", DagAccessEntity.RUN) @security.requires_access_asset("GET") @provide_session -def get_upstream_dataset_events( - *, dag_id: str, dag_run_id: str, session: Session = NEW_SESSION -) -> APIResponse: - """If dag run is dataset-triggered, return the asset events that triggered it.""" +def get_upstream_asset_events(*, dag_id: str, dag_run_id: str, session: Session = NEW_SESSION) -> APIResponse: + """If dag run is asset-triggered, return the asset events that triggered it.""" dag_run: DagRun | None = session.scalar( select(DagRun).where( DagRun.dag_id == dag_id, @@ -131,7 +129,7 @@ def get_upstream_dataset_events( ) events = dag_run.consumed_dataset_events return asset_event_collection_schema.dump( - AssetEventCollection(dataset_events=events, total_entries=len(events)) + AssetEventCollection(asset_events=events, total_entries=len(events)) ) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 4a822db5bc876..bf60eaa027ffe 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1181,26 +1181,26 @@ paths: "404": $ref: "#/components/responses/NotFound" - /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents: + /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamAssetEvents: parameters: - $ref: "#/components/parameters/DAGID" - $ref: "#/components/parameters/DAGRunID" get: - summary: Get dataset events for a DAG run + summary: Get asset events for a DAG run description: | - Get datasets for a dag run. + Get asset for a dag run. *New in version 2.4.0* x-openapi-router-controller: airflow.api_connexion.endpoints.dag_run_endpoint - operationId: get_upstream_dataset_events - tags: [DAGRun, Dataset] + operationId: get_upstream_asset_events + tags: [DAGRun, Asset] responses: "200": description: Success. content: application/json: schema: - $ref: "#/components/schemas/DatasetEventCollection" + $ref: "#/components/schemas/AssetEventCollection" "401": $ref: "#/components/responses/Unauthenticated" "403": @@ -2600,7 +2600,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DatasetEventCollection" + $ref: "#/components/schemas/AssetEventCollection" "401": $ref: "#/components/responses/Unauthenticated" "403": @@ -4737,9 +4737,9 @@ components: state: $ref: "#/components/schemas/DagState" - DatasetEventCollection: + AssetEventCollection: description: | - A collection of dataset events. + A collection of asset events. *New in version 2.4.0* type: object diff --git a/airflow/api_connexion/schemas/asset_schema.py b/airflow/api_connexion/schemas/asset_schema.py index 282c7818c56ca..ad7c1b7f5e3ba 100644 --- a/airflow/api_connexion/schemas/asset_schema.py +++ b/airflow/api_connexion/schemas/asset_schema.py @@ -150,14 +150,14 @@ class Meta: class AssetEventCollection(NamedTuple): """List of Asset events with meta.""" - dataset_events: list[AssetEvent] + asset_events: list[AssetEvent] total_entries: int class AssetEventCollectionSchema(Schema): """Asset Event Collection Schema.""" - dataset_events = fields.List(fields.Nested(AssetEventSchema)) + asset_events = fields.List(fields.Nested(AssetEventSchema)) total_entries = fields.Int() diff --git a/airflow/www/static/js/api/useUpstreamAssetEvents.ts b/airflow/www/static/js/api/useUpstreamAssetEvents.ts index 20086af0c3bff..c34b89dee4dcc 100644 --- a/airflow/www/static/js/api/useUpstreamAssetEvents.ts +++ b/airflow/www/static/js/api/useUpstreamAssetEvents.ts @@ -23,21 +23,21 @@ import { useQuery, UseQueryOptions } from "react-query"; import { getMetaValue } from "src/utils"; import type { DatasetEventCollection, - GetUpstreamDatasetEventsVariables, + GetUpstreamAssetEventsVariables, } from "src/types/api-generated"; -interface Props extends GetUpstreamDatasetEventsVariables { +interface Props extends GetUpstreamAssetEventsVariables { options?: UseQueryOptions; } const useUpstreamAssetEvents = ({ dagId, dagRunId, options }: Props) => { const upstreamEventsUrl = ( getMetaValue("upstream_dataset_events_api") || - `api/v1/dags/${dagId}/dagRuns/_DAG_RUN_ID_/upstreamDatasetEvents` + `api/v1/dags/${dagId}/dagRuns/_DAG_RUN_ID_/upstreamAssetEvents` ).replace("_DAG_RUN_ID_", encodeURIComponent(dagRunId)); const query = useQuery( - ["upstreamDatasetEvents", dagRunId], + ["upstreamAssetEvents", dagRunId], () => axios.get(upstreamEventsUrl), options ); diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index c3e9575ff5a4b..39936e95df807 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -164,7 +164,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { ); const { - data: { datasetEvents: upstreamDatasetEvents = [] }, + data: { datasetEvents: upstreamAssetEvents = [] }, } = useUpstreamAssetEvents({ dagId, dagRunId: selected.runId || "", @@ -265,7 +265,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { hoveredTaskState, isZoomedOut, datasetEvents: selected.runId - ? [...upstreamDatasetEvents, ...downstreamDatasetEvents] + ? [...upstreamAssetEvents, ...downstreamDatasetEvents] : [], }), [ @@ -277,7 +277,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { groups, hoveredTaskState, isZoomedOut, - upstreamDatasetEvents, + upstreamAssetEvents, downstreamDatasetEvents, ] ); diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index a0d121316e34a..5f2c592f95939 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -262,13 +262,13 @@ export interface paths { }; }; }; - "/dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents": { + "/dags/{dag_id}/dagRuns/{dag_run_id}/upstreamAssetEvents": { /** * Get datasets for a dag run. * * *New in version 2.4.0* */ - get: operations["get_upstream_dataset_events"]; + get: operations["get_upstream_asset_events"]; parameters: { path: { /** The DAG ID. */ @@ -3595,7 +3595,7 @@ export interface operations { * * *New in version 2.4.0* */ - get_upstream_dataset_events: { + get_upstream_asset_events: { parameters: { path: { /** The DAG ID. */ @@ -5633,8 +5633,8 @@ export type ClearDagRunVariables = CamelCasedPropertiesDeep< operations["clear_dag_run"]["parameters"]["path"] & operations["clear_dag_run"]["requestBody"]["content"]["application/json"] >; -export type GetUpstreamDatasetEventsVariables = CamelCasedPropertiesDeep< - operations["get_upstream_dataset_events"]["parameters"]["path"] +export type GetUpstreamAssetEventsVariables = CamelCasedPropertiesDeep< + operations["get_upstream_asset_events"]["parameters"]["path"] >; export type SetDagRunNoteVariables = CamelCasedPropertiesDeep< operations["set_dag_run_note"]["parameters"]["path"] & diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 0d10fd09d72fe..01b20b2ac7719 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -73,7 +73,7 @@ - + diff --git a/clients/python/README.md b/clients/python/README.md index 050054b383c9f..f6f18327318ae 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -343,7 +343,7 @@ Class | Method | HTTP request | Description *DAGRunApi* | [**get_dag_run**](docs/DAGRunApi.md#get_dag_run) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id} | Get a DAG run *DAGRunApi* | [**get_dag_runs**](docs/DAGRunApi.md#get_dag_runs) | **GET** /dags/{dag_id}/dagRuns | List DAG runs *DAGRunApi* | [**get_dag_runs_batch**](docs/DAGRunApi.md#get_dag_runs_batch) | **POST** /dags/~/dagRuns/list | List DAG runs (batch) -*DAGRunApi* | [**get_upstream_dataset_events**](docs/DAGRunApi.md#get_upstream_dataset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents | Get dataset events for a DAG run +*DAGRunApi* | [**get_upstream_asset_events**](docs/DAGRunApi.md#get_upstream_asset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamAssetEvents | Get asset events for a DAG run *DAGRunApi* | [**post_dag_run**](docs/DAGRunApi.md#post_dag_run) | **POST** /dags/{dag_id}/dagRuns | Trigger a new DAG run *DAGRunApi* | [**set_dag_run_note**](docs/DAGRunApi.md#set_dag_run_note) | **PATCH** /dags/{dag_id}/dagRuns/{dag_run_id}/setNote | Update the DagRun note. *DAGRunApi* | [**update_dag_run_state**](docs/DAGRunApi.md#update_dag_run_state) | **PATCH** /dags/{dag_id}/dagRuns/{dag_run_id} | Modify a DAG run @@ -351,7 +351,7 @@ Class | Method | HTTP request | Description *AssetApi* | [**get_asset**](docs/DatasetApi.md#get_asset) | **GET** /datasets/{uri} | Get an asset *AssetApi* | [**get_asset_events**](docs/DatasetApi.md#get_asset_events) | **GET** /datasets/events | Get asset events *DatasetApi* | [**get_assets**](docs/DatasetApi.md#get_assets) | **GET** /datasets | List assets -*DatasetApi* | [**get_upstream_dataset_events**](docs/DatasetApi.md#get_upstream_dataset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents | Get dataset events for a DAG run +*DatasetApi* | [**get_upstream_asset_events**](docs/DatasetApi.md#get_upstream_asset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamAssetEvents | Get dataset events for a DAG run *EventLogApi* | [**get_event_log**](docs/EventLogApi.md#get_event_log) | **GET** /eventLogs/{event_log_id} | Get a log entry *EventLogApi* | [**get_event_logs**](docs/EventLogApi.md#get_event_logs) | **GET** /eventLogs | List log entries *ImportErrorApi* | [**get_import_error**](docs/ImportErrorApi.md#get_import_error) | **GET** /importErrors/{import_error_id} | Get an import error diff --git a/tests/api_connexion/endpoints/test_dag_run_endpoint.py b/tests/api_connexion/endpoints/test_dag_run_endpoint.py index 73c75b98a43b1..b2e797358ef28 100644 --- a/tests/api_connexion/endpoints/test_dag_run_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_run_endpoint.py @@ -1799,12 +1799,12 @@ def test_should_respond_200(self, dag_maker, session): assert event.timestamp response = self.client.get( - "api/v1/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamDatasetEvents", + "api/v1/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamAssetEvents", environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 200 expected_response = { - "dataset_events": [ + "asset_events": [ { "timestamp": event.timestamp.isoformat(), "dataset_id": asset1_id, @@ -1835,7 +1835,7 @@ def test_should_respond_200(self, dag_maker, session): def test_should_respond_404(self): response = self.client.get( - "api/v1/dags/invalid-id/dagRuns/invalid-id/upstreamDatasetEvents", + "api/v1/dags/invalid-id/dagRuns/invalid-id/upstreamAssetEvents", environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 404 @@ -1859,7 +1859,7 @@ def test_should_raises_401_unauthenticated(self, session): session.add(dagrun_model) session.commit() - response = self.client.get("api/v1/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamDatasetEvents") + response = self.client.get("api/v1/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamAssetEvents") assert_401(response) diff --git a/tests/api_connexion/schemas/test_dataset_schema.py b/tests/api_connexion/schemas/test_dataset_schema.py index a9a5ce9e9673b..07d041b1a1d60 100644 --- a/tests/api_connexion/schemas/test_dataset_schema.py +++ b/tests/api_connexion/schemas/test_dataset_schema.py @@ -219,10 +219,10 @@ def test_serialize(self, session): session.add_all(events) session.flush() serialized_data = asset_event_collection_schema.dump( - AssetEventCollection(dataset_events=events, total_entries=2) + AssetEventCollection(asset_events=events, total_entries=2) ) assert serialized_data == { - "dataset_events": [ + "asset_events": [ {"id": 1, "timestamp": self.timestamp, **common}, {"id": 2, "timestamp": self.timestamp, **common}, ], From 287e1b23bf72562a2da37f5000c8aa7872b13ab5 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 18:14:45 +0900 Subject: [PATCH 27/43] feat(api_connexion/openapi/v1): rename DatasetURI as AssetURI --- airflow/api_connexion/openapi/v1.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index bf60eaa027ffe..2220608230a83 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1248,7 +1248,7 @@ paths: /dags/{dag_id}/datasets/queuedEvent/{uri}: parameters: - $ref: "#/components/parameters/DAGID" - - $ref: "#/components/parameters/DatasetURI" + - $ref: "#/components/parameters/AssetURI" get: summary: Get a queued Dataset event for a DAG @@ -1373,7 +1373,7 @@ paths: /datasets/queuedEvent/{uri}: parameters: - - $ref: "#/components/parameters/DatasetURI" + - $ref: "#/components/parameters/AssetURI" get: summary: Get queued Asset events for an Asset. @@ -2557,7 +2557,7 @@ paths: /datasets/{uri}: parameters: - - $ref: "#/components/parameters/DatasetURI" + - $ref: "#/components/parameters/AssetURI" get: summary: Get a dataset description: Get a dataset by uri. @@ -5545,14 +5545,14 @@ components: required: true description: The import error ID. - DatasetURI: + AssetURI: in: path name: uri schema: type: string format: path required: true - description: The encoded Dataset URI + description: The encoded Asset URI PoolName: in: path From b28753cc2828942a3a9f1aaa91aa3788eafd6b42 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 18:25:22 +0900 Subject: [PATCH 28/43] feat(api_connexion/openapi/v1): rename DatasetCollection as AssetCollection --- airflow/api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 6 +++--- airflow/api_connexion/schemas/asset_schema.py | 4 ++-- airflow/www/static/js/api/useAssets.ts | 2 +- airflow/www/static/js/types/api-generated.ts | 8 ++++---- clients/python/README.md | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index c027085691e8e..9fc501633800c 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -109,7 +109,7 @@ def get_assets( .offset(offset) .limit(limit) ).all() - return asset_collection_schema.dump(AssetCollection(datasets=assets, total_entries=total_entries)) + return asset_collection_schema.dump(AssetCollection(assets=assets, total_entries=total_entries)) @security.requires_access_asset("GET") diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 2220608230a83..868a4876677d3 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2549,7 +2549,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DatasetCollection" + $ref: "#/components/schemas/AssetCollection" "401": $ref: "#/components/responses/Unauthenticated" "403": @@ -4586,9 +4586,9 @@ components: description: The dataset reference update time nullable: false - DatasetCollection: + AssetCollection: description: | - A collection of datasets. + A collection of assets. *New in version 2.4.0* type: object diff --git a/airflow/api_connexion/schemas/asset_schema.py b/airflow/api_connexion/schemas/asset_schema.py index ad7c1b7f5e3ba..662f73a50d8b9 100644 --- a/airflow/api_connexion/schemas/asset_schema.py +++ b/airflow/api_connexion/schemas/asset_schema.py @@ -93,14 +93,14 @@ class Meta: class AssetCollection(NamedTuple): """List of Assets with meta.""" - datasets: list[AssetModel] + assets: list[AssetModel] total_entries: int class AssetCollectionSchema(Schema): """Asset Collection Schema.""" - datasets = fields.List(fields.Nested(AssetSchema)) + assets = fields.List(fields.Nested(AssetSchema)) total_entries = fields.Int() diff --git a/airflow/www/static/js/api/useAssets.ts b/airflow/www/static/js/api/useAssets.ts index b9af13ff65cb2..3654c583c12ef 100644 --- a/airflow/www/static/js/api/useAssets.ts +++ b/airflow/www/static/js/api/useAssets.ts @@ -36,7 +36,7 @@ export default function useAssets({ dagIds, enabled = true }: Props) { const dagIdsParam = dagIds && dagIds.length ? { dag_ids: dagIds.join(",") } : {}; - return axios.get(datasetsUrl, { + return axios.get(datasetsUrl, { params: { ...dagIdsParam, }, diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 5f2c592f95939..1e24a939e9eab 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1898,7 +1898,7 @@ export interface components { * * *New in version 2.4.0* */ - DatasetCollection: { + AssetCollection: { datasets?: components["schemas"]["Dataset"][]; } & components["schemas"]["CollectionInfo"]; /** @@ -5073,7 +5073,7 @@ export interface operations { /** Success. */ 200: { content: { - "application/json": components["schemas"]["DatasetCollection"]; + "application/json": components["schemas"]["AssetCollection"]; }; }; 401: components["responses"]["Unauthenticated"]; @@ -5431,8 +5431,8 @@ export type TaskOutletDatasetReference = CamelCasedPropertiesDeep< export type DagScheduleDatasetReference = CamelCasedPropertiesDeep< components["schemas"]["DagScheduleDatasetReference"] >; -export type DatasetCollection = CamelCasedPropertiesDeep< - components["schemas"]["DatasetCollection"] +export type AssetCollection = CamelCasedPropertiesDeep< + components["schemas"]["AssetCollection"] >; export type DatasetEvent = CamelCasedPropertiesDeep< components["schemas"]["DatasetEvent"] diff --git a/clients/python/README.md b/clients/python/README.md index f6f18327318ae..b67b9a6405a6f 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -432,8 +432,8 @@ Class | Method | HTTP request | Description - [DagWarningCollection](docs/DagWarningCollection.md) - [DagWarningCollectionAllOf](docs/DagWarningCollectionAllOf.md) - [Dataset](docs/Dataset.md) - - [DatasetCollection](docs/DatasetCollection.md) - - [DatasetCollectionAllOf](docs/DatasetCollectionAllOf.md) + - [AssetCollection](docs/AssetCollection.md) + - [AssetCollectionAllOf](docs/AssetCollectionAllOf.md) - [DatasetEvent](docs/DatasetEvent.md) - [DatasetEventCollection](docs/DatasetEventCollection.md) - [DatasetEventCollectionAllOf](docs/DatasetEventCollectionAllOf.md) From 97991a04f1ef53f3a77726f99b13cb5364c1722f Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 18:28:20 +0900 Subject: [PATCH 29/43] feat(api_connexion/openapi/v1): rename DagScheduleDatasetReference as DagScheduleAssetReference --- airflow/api_connexion/openapi/v1.yaml | 12 ++++++------ airflow/www/static/js/types/api-generated.ts | 8 ++++---- clients/python/README.md | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 868a4876677d3..eda4af0e1ae46 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -4536,7 +4536,7 @@ components: consuming_dags: type: array items: - $ref: "#/components/schemas/DagScheduleDatasetReference" + $ref: "#/components/schemas/DagScheduleAssetReference" producing_tasks: type: array items: @@ -4566,24 +4566,24 @@ components: description: The dataset update time nullable: false - DagScheduleDatasetReference: + DagScheduleAssetReference: description: | - A datasets reference to a downstream DAG. + An asset reference to a downstream DAG. *New in version 2.4.0* type: object properties: dag_id: type: string - description: The DAG ID that depends on the dataset. + description: The DAG ID that depends on the asset. nullable: true created_at: type: string - description: The dataset reference creation time + description: The asset reference creation time nullable: false updated_at: type: string - description: The dataset reference update time + description: The asset reference update time nullable: false AssetCollection: diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 1e24a939e9eab..eb68fb4df377e 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1862,7 +1862,7 @@ export interface components { created_at?: string; /** @description The dataset update time */ updated_at?: string; - consuming_dags?: components["schemas"]["DagScheduleDatasetReference"][]; + consuming_dags?: components["schemas"]["DagScheduleAssetReference"][]; producing_tasks?: components["schemas"]["TaskOutletDatasetReference"][]; }; /** @@ -1885,7 +1885,7 @@ export interface components { * * *New in version 2.4.0* */ - DagScheduleDatasetReference: { + DagScheduleAssetReference: { /** @description The DAG ID that depends on the dataset. */ dag_id?: string | null; /** @description The dataset reference creation time */ @@ -5428,8 +5428,8 @@ export type Dataset = CamelCasedPropertiesDeep< export type TaskOutletDatasetReference = CamelCasedPropertiesDeep< components["schemas"]["TaskOutletDatasetReference"] >; -export type DagScheduleDatasetReference = CamelCasedPropertiesDeep< - components["schemas"]["DagScheduleDatasetReference"] +export type DagScheduleAssetReference = CamelCasedPropertiesDeep< + components["schemas"]["DagScheduleAssetReference"] >; export type AssetCollection = CamelCasedPropertiesDeep< components["schemas"]["AssetCollection"] diff --git a/clients/python/README.md b/clients/python/README.md index b67b9a6405a6f..d204feff20556 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -426,7 +426,7 @@ Class | Method | HTTP request | Description - [DAGRun](docs/DAGRun.md) - [DAGRunCollection](docs/DAGRunCollection.md) - [DAGRunCollectionAllOf](docs/DAGRunCollectionAllOf.md) - - [DagScheduleDatasetReference](docs/DagScheduleDatasetReference.md) + - [DagScheduleAssetReference](docs/DagScheduleAssetReference.md) - [DagState](docs/DagState.md) - [DagWarning](docs/DagWarning.md) - [DagWarningCollection](docs/DagWarningCollection.md) From 749b681afe11f2f9eb429b59ef80402f02952224 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 18:34:14 +0900 Subject: [PATCH 30/43] feat(api_connexion/openapi/v1): rename TaskOutletDatasetReference as TaskOutletAssetReference --- airflow/api_connexion/openapi/v1.yaml | 12 ++++++------ airflow/www/static/js/types/api-generated.ts | 8 ++++---- clients/python/README.md | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index eda4af0e1ae46..700f16ecc2b59 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -4540,22 +4540,22 @@ components: producing_tasks: type: array items: - $ref: "#/components/schemas/TaskOutletDatasetReference" + $ref: "#/components/schemas/TaskOutletAssetReference" - TaskOutletDatasetReference: + TaskOutletAssetReference: description: | - A datasets reference to an upstream task. + An asset reference to an upstream task. *New in version 2.4.0* type: object properties: dag_id: type: string - description: The DAG ID that updates the dataset. + description: The DAG ID that updates the asset. nullable: true task_id: type: string - description: The task ID that updates the dataset. + description: The task ID that updates the asset. nullable: true created_at: type: string @@ -4563,7 +4563,7 @@ components: nullable: false updated_at: type: string - description: The dataset update time + description: The asset update time nullable: false DagScheduleAssetReference: diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index eb68fb4df377e..dc2cc47ded1ea 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1863,14 +1863,14 @@ export interface components { /** @description The dataset update time */ updated_at?: string; consuming_dags?: components["schemas"]["DagScheduleAssetReference"][]; - producing_tasks?: components["schemas"]["TaskOutletDatasetReference"][]; + producing_tasks?: components["schemas"]["TaskOutletAssetReference"][]; }; /** * @description A datasets reference to an upstream task. * * *New in version 2.4.0* */ - TaskOutletDatasetReference: { + TaskOutletAssetReference: { /** @description The DAG ID that updates the dataset. */ dag_id?: string | null; /** @description The task ID that updates the dataset. */ @@ -5425,8 +5425,8 @@ export type ActionResource = CamelCasedPropertiesDeep< export type Dataset = CamelCasedPropertiesDeep< components["schemas"]["Dataset"] >; -export type TaskOutletDatasetReference = CamelCasedPropertiesDeep< - components["schemas"]["TaskOutletDatasetReference"] +export type TaskOutletAssetReference = CamelCasedPropertiesDeep< + components["schemas"]["TaskOutletAssetReference"] >; export type DagScheduleAssetReference = CamelCasedPropertiesDeep< components["schemas"]["DagScheduleAssetReference"] diff --git a/clients/python/README.md b/clients/python/README.md index d204feff20556..faa698a5db0a4 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -481,7 +481,7 @@ Class | Method | HTTP request | Description - [TaskInstanceCollectionAllOf](docs/TaskInstanceCollectionAllOf.md) - [TaskInstanceReference](docs/TaskInstanceReference.md) - [TaskInstanceReferenceCollection](docs/TaskInstanceReferenceCollection.md) - - [TaskOutletDatasetReference](docs/TaskOutletDatasetReference.md) + - [TaskOutletAssetReference](docs/TaskOutletAssetReference.md) - [TaskState](docs/TaskState.md) - [TimeDelta](docs/TimeDelta.md) - [Trigger](docs/Trigger.md) From 4203a41e87a90caa4be8febce99e0c5b8992ac8e Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 21:10:09 +0900 Subject: [PATCH 31/43] feat(js/api): rename DatasetEventCollection as AssetEventCollection --- airflow/www/static/js/api/useAssetEvents.ts | 6 +- .../static/js/api/useUpstreamAssetEvents.ts | 6 +- airflow/www/static/js/types/api-generated.ts | 78 +++++++++---------- clients/python/README.md | 4 +- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/airflow/www/static/js/api/useAssetEvents.ts b/airflow/www/static/js/api/useAssetEvents.ts index ff8425e5092a7..25fc815eb1662 100644 --- a/airflow/www/static/js/api/useAssetEvents.ts +++ b/airflow/www/static/js/api/useAssetEvents.ts @@ -23,12 +23,12 @@ import { useQuery, UseQueryOptions } from "react-query"; import { getMetaValue } from "src/utils"; import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; import type { - DatasetEventCollection, + AssetEventCollection, GetAssetEventsVariables, } from "src/types/api-generated"; interface Props extends GetAssetEventsVariables { - options?: UseQueryOptions; + options?: UseQueryOptions; } const useAssetEvents = ({ @@ -42,7 +42,7 @@ const useAssetEvents = ({ orderBy, options, }: Props) => { - const query = useQuery( + const query = useQuery( [ "datasets-events", assetId, diff --git a/airflow/www/static/js/api/useUpstreamAssetEvents.ts b/airflow/www/static/js/api/useUpstreamAssetEvents.ts index c34b89dee4dcc..d35349ddb4ae0 100644 --- a/airflow/www/static/js/api/useUpstreamAssetEvents.ts +++ b/airflow/www/static/js/api/useUpstreamAssetEvents.ts @@ -22,12 +22,12 @@ import { useQuery, UseQueryOptions } from "react-query"; import { getMetaValue } from "src/utils"; import type { - DatasetEventCollection, + AssetEventCollection, GetUpstreamAssetEventsVariables, } from "src/types/api-generated"; interface Props extends GetUpstreamAssetEventsVariables { - options?: UseQueryOptions; + options?: UseQueryOptions; } const useUpstreamAssetEvents = ({ dagId, dagRunId, options }: Props) => { @@ -36,7 +36,7 @@ const useUpstreamAssetEvents = ({ dagId, dagRunId, options }: Props) => { `api/v1/dags/${dagId}/dagRuns/_DAG_RUN_ID_/upstreamAssetEvents` ).replace("_DAG_RUN_ID_", encodeURIComponent(dagRunId)); - const query = useQuery( + const query = useQuery( ["upstreamAssetEvents", dagRunId], () => axios.get(upstreamEventsUrl), options diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index dc2cc47ded1ea..65002937abb30 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -264,7 +264,7 @@ export interface paths { }; "/dags/{dag_id}/dagRuns/{dag_run_id}/upstreamAssetEvents": { /** - * Get datasets for a dag run. + * Get asset for a dag run. * * *New in version 2.4.0* */ @@ -296,7 +296,7 @@ export interface paths { }; "/dags/{dag_id}/datasets/queuedEvent/{uri}": { /** - * Get a queued Dataset event for a DAG. + * Get a queued asset event for a DAG. * * *New in version 2.9.0* */ @@ -311,8 +311,8 @@ export interface paths { path: { /** The DAG ID. */ dag_id: components["parameters"]["DAGID"]; - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; }; }; @@ -365,8 +365,8 @@ export interface paths { delete: operations["delete_asset_queued_events"]; parameters: { path: { - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; }; }; @@ -786,12 +786,12 @@ export interface paths { get: operations["get_assets"]; }; "/datasets/{uri}": { - /** Get a dataset by uri. */ + /** Get an asset by uri. */ get: operations["get_asset"]; parameters: { path: { - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; }; }; @@ -1866,35 +1866,35 @@ export interface components { producing_tasks?: components["schemas"]["TaskOutletAssetReference"][]; }; /** - * @description A datasets reference to an upstream task. + * @description An asset reference to an upstream task. * * *New in version 2.4.0* */ TaskOutletAssetReference: { - /** @description The DAG ID that updates the dataset. */ + /** @description The DAG ID that updates the asset. */ dag_id?: string | null; - /** @description The task ID that updates the dataset. */ + /** @description The task ID that updates the asset. */ task_id?: string | null; /** @description The dataset creation time */ created_at?: string; - /** @description The dataset update time */ + /** @description The asset update time */ updated_at?: string; }; /** - * @description A datasets reference to a downstream DAG. + * @description An asset reference to a downstream DAG. * * *New in version 2.4.0* */ DagScheduleAssetReference: { - /** @description The DAG ID that depends on the dataset. */ + /** @description The DAG ID that depends on the asset. */ dag_id?: string | null; - /** @description The dataset reference creation time */ + /** @description The asset reference creation time */ created_at?: string; - /** @description The dataset reference update time */ + /** @description The asset reference update time */ updated_at?: string; }; /** - * @description A collection of datasets. + * @description A collection of assets. * * *New in version 2.4.0* */ @@ -1983,11 +1983,11 @@ export interface components { state?: components["schemas"]["DagState"]; }; /** - * @description A collection of dataset events. + * @description A collection of asset events. * * *New in version 2.4.0* */ - DatasetEventCollection: { + AssetEventCollection: { dataset_events?: components["schemas"]["DatasetEvent"][]; } & components["schemas"]["CollectionInfo"]; /** @description The option of configuration. */ @@ -2541,8 +2541,8 @@ export interface components { EventLogID: number; /** @description The import error ID. */ ImportErrorID: number; - /** @description The encoded Dataset URI */ - DatasetURI: string; + /** @description The encoded Asset URI */ + AssetURI: string; /** @description The pool name. */ PoolName: string; /** @description The variable Key. */ @@ -3591,7 +3591,7 @@ export interface operations { }; }; /** - * Get datasets for a dag run. + * Get asset for a dag run. * * *New in version 2.4.0* */ @@ -3608,7 +3608,7 @@ export interface operations { /** Success. */ 200: { content: { - "application/json": components["schemas"]["DatasetEventCollection"]; + "application/json": components["schemas"]["AssetEventCollection"]; }; }; 401: components["responses"]["Unauthenticated"]; @@ -3650,7 +3650,7 @@ export interface operations { }; }; /** - * Get a queued Dataset event for a DAG. + * Get a queued asset event for a DAG. * * *New in version 2.9.0* */ @@ -3659,8 +3659,8 @@ export interface operations { path: { /** The DAG ID. */ dag_id: components["parameters"]["DAGID"]; - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; query: { /** Timestamp to select event logs occurring before. */ @@ -3689,8 +3689,8 @@ export interface operations { path: { /** The DAG ID. */ dag_id: components["parameters"]["DAGID"]; - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; query: { /** Timestamp to select event logs occurring before. */ @@ -3787,8 +3787,8 @@ export interface operations { get_asset_queued_events: { parameters: { path: { - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; query: { /** Timestamp to select event logs occurring before. */ @@ -3815,8 +3815,8 @@ export interface operations { delete_asset_queued_events: { parameters: { path: { - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; query: { /** Timestamp to select event logs occurring before. */ @@ -5080,12 +5080,12 @@ export interface operations { 403: components["responses"]["PermissionDenied"]; }; }; - /** Get a dataset by uri. */ + /** Get an asset by uri. */ get_asset: { parameters: { path: { - /** The encoded Dataset URI */ - uri: components["parameters"]["DatasetURI"]; + /** The encoded Asset URI */ + uri: components["parameters"]["AssetURI"]; }; }; responses: { @@ -5131,7 +5131,7 @@ export interface operations { /** Success. */ 200: { content: { - "application/json": components["schemas"]["DatasetEventCollection"]; + "application/json": components["schemas"]["AssetEventCollection"]; }; }; 401: components["responses"]["Unauthenticated"]; @@ -5449,8 +5449,8 @@ export type QueuedEventCollection = CamelCasedPropertiesDeep< export type BasicDAGRun = CamelCasedPropertiesDeep< components["schemas"]["BasicDAGRun"] >; -export type DatasetEventCollection = CamelCasedPropertiesDeep< - components["schemas"]["DatasetEventCollection"] +export type AssetEventCollection = CamelCasedPropertiesDeep< + components["schemas"]["AssetEventCollection"] >; export type ConfigOption = CamelCasedPropertiesDeep< components["schemas"]["ConfigOption"] diff --git a/clients/python/README.md b/clients/python/README.md index faa698a5db0a4..de9dfbbbe4afe 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -435,8 +435,8 @@ Class | Method | HTTP request | Description - [AssetCollection](docs/AssetCollection.md) - [AssetCollectionAllOf](docs/AssetCollectionAllOf.md) - [DatasetEvent](docs/DatasetEvent.md) - - [DatasetEventCollection](docs/DatasetEventCollection.md) - - [DatasetEventCollectionAllOf](docs/DatasetEventCollectionAllOf.md) + - [AssetEventCollection](docs/AssetEventCollection.md) + - [AssetEventCollectionAllOf](docs/AssetEventCollectionAllOf.md) - [Error](docs/Error.md) - [EventLog](docs/EventLog.md) - [EventLogCollection](docs/EventLogCollection.md) From a50b4c57f0806f94acfe2b90d78318273fde7b50 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 21:32:52 +0900 Subject: [PATCH 32/43] feat(api_connexion/openapi/v1): rename DatasetEvent as AssetEvent --- airflow/api_connexion/openapi/v1.yaml | 24 ++++++++--------- .../www/static/js/api/useCreateAssetEvent.ts | 2 +- .../static/js/components/DatasetEventCard.tsx | 4 +-- .../js/components/SourceTaskInstance.tsx | 4 +-- .../details/dagRun/DatasetTriggerEvents.tsx | 4 +-- .../www/static/js/dag/details/graph/Node.tsx | 4 +-- .../www/static/js/dag/details/graph/utils.ts | 4 +-- .../taskInstance/DatasetUpdateEvents.tsx | 4 +-- .../www/static/js/datasets/AssetEvents.tsx | 4 +-- airflow/www/static/js/types/api-generated.ts | 26 +++++++++---------- clients/python/README.md | 10 +++---- 11 files changed, 45 insertions(+), 45 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 700f16ecc2b59..fbe7af34616ee 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2625,7 +2625,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/DatasetEvent' + $ref: '#/components/schemas/AssetEvent' "400": $ref: "#/components/responses/BadRequest" '401': @@ -4601,39 +4601,39 @@ components: $ref: "#/components/schemas/Dataset" - $ref: "#/components/schemas/CollectionInfo" - DatasetEvent: + AssetEvent: description: | - A dataset event. + An asset event. *New in version 2.4.0* type: object properties: dataset_id: type: integer - description: The dataset id + description: The asset id dataset_uri: type: string - description: The URI of the dataset + description: The URI of the asset nullable: false extra: type: object - description: The dataset event extra + description: The asset event extra nullable: true source_dag_id: type: string - description: The DAG ID that updated the dataset. + description: The DAG ID that updated the asset. nullable: true source_task_id: type: string - description: The task ID that updated the dataset. + description: The task ID that updated the asset. nullable: true source_run_id: type: string - description: The DAG run ID that updated the dataset. + description: The DAG run ID that updated the asset. nullable: true source_map_index: type: integer - description: The task map index that updated the dataset. + description: The task map index that updated the asset. nullable: true created_dagruns: type: array @@ -4641,7 +4641,7 @@ components: $ref: "#/components/schemas/BasicDAGRun" timestamp: type: string - description: The dataset event creation time + description: The asset event creation time nullable: false CreateAssetEvent: @@ -4749,7 +4749,7 @@ components: dataset_events: type: array items: - $ref: "#/components/schemas/DatasetEvent" + $ref: "#/components/schemas/AssetEvent" - $ref: "#/components/schemas/CollectionInfo" # Configuration diff --git a/airflow/www/static/js/api/useCreateAssetEvent.ts b/airflow/www/static/js/api/useCreateAssetEvent.ts index e0fea7a2cf679..7d2322c33ce9d 100644 --- a/airflow/www/static/js/api/useCreateAssetEvent.ts +++ b/airflow/www/static/js/api/useCreateAssetEvent.ts @@ -37,7 +37,7 @@ export default function useCreateAssetEvent({ datasetId, uri }: Props) { return useMutation( ["createAssetEvent", uri], - (extra?: API.DatasetEvent["extra"]) => + (extra?: API.AssetEvent["extra"]) => axios.post(createAssetUrl, { asset_uri: uri, extra: extra || {}, diff --git a/airflow/www/static/js/components/DatasetEventCard.tsx b/airflow/www/static/js/components/DatasetEventCard.tsx index 2367c8efa9b4a..1f321933c4b5b 100644 --- a/airflow/www/static/js/components/DatasetEventCard.tsx +++ b/airflow/www/static/js/components/DatasetEventCard.tsx @@ -21,7 +21,7 @@ import React from "react"; import { isEmpty } from "lodash"; import { TbApi } from "react-icons/tb"; -import type { DatasetEvent } from "src/types/api-generated"; +import type { AssetEvent } from "src/types/api-generated"; import { Box, Flex, @@ -43,7 +43,7 @@ import SourceTaskInstance from "./SourceTaskInstance"; import TriggeredDagRuns from "./TriggeredDagRuns"; type CardProps = { - datasetEvent: DatasetEvent; + datasetEvent: AssetEvent; showSource?: boolean; showTriggeredDagRuns?: boolean; }; diff --git a/airflow/www/static/js/components/SourceTaskInstance.tsx b/airflow/www/static/js/components/SourceTaskInstance.tsx index 4c63198c5f40c..f52e9b4c75b85 100644 --- a/airflow/www/static/js/components/SourceTaskInstance.tsx +++ b/airflow/www/static/js/components/SourceTaskInstance.tsx @@ -22,7 +22,7 @@ import { Box, Link, Tooltip, Flex } from "@chakra-ui/react"; import { FiLink } from "react-icons/fi"; import { useTaskInstance } from "src/api"; -import type { DatasetEvent } from "src/types/api-generated"; +import type { AssetEvent } from "src/types/api-generated"; import { useContainerRef } from "src/context/containerRef"; import { SimpleStatus } from "src/dag/StatusBox"; import InstanceTooltip from "src/components/InstanceTooltip"; @@ -30,7 +30,7 @@ import type { TaskInstance } from "src/types"; import { getMetaValue } from "src/utils"; type SourceTIProps = { - datasetEvent: DatasetEvent; + datasetEvent: AssetEvent; showLink?: boolean; }; diff --git a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx index b3e94991ee75e..086614956be3b 100644 --- a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx +++ b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx @@ -22,7 +22,7 @@ import { Box, Text } from "@chakra-ui/react"; import { useUpstreamAssetEvents } from "src/api"; import type { DagRun as DagRunType } from "src/types"; import { CardDef, CardList } from "src/components/Table"; -import type { DatasetEvent } from "src/types/api-generated"; +import type { AssetEvent } from "src/types/api-generated"; import DatasetEventCard from "src/components/DatasetEventCard"; import { getMetaValue } from "src/utils"; @@ -32,7 +32,7 @@ interface Props { const dagId = getMetaValue("dag_id"); -const cardDef: CardDef = { +const cardDef: CardDef = { card: ({ row }) => ( ), diff --git a/airflow/www/static/js/dag/details/graph/Node.tsx b/airflow/www/static/js/dag/details/graph/Node.tsx index daedfb8524e0b..89eafe79f9c00 100644 --- a/airflow/www/static/js/dag/details/graph/Node.tsx +++ b/airflow/www/static/js/dag/details/graph/Node.tsx @@ -23,7 +23,7 @@ import { Handle, NodeProps, Position } from "reactflow"; import { TbLogicAnd, TbLogicOr } from "react-icons/tb"; import type { DepNode, DagRun, Task, TaskInstance } from "src/types"; -import type { DatasetEvent } from "src/types/api-generated"; +import type { AssetEvent } from "src/types/api-generated"; import Tooltip from "src/components/Tooltip"; import { useContainerRef } from "src/context/containerRef"; @@ -49,7 +49,7 @@ export interface CustomNodeProps { style?: string; isZoomedOut: boolean; class: DepNode["value"]["class"]; - datasetEvent?: DatasetEvent; + datasetEvent?: AssetEvent; } const Node = (props: NodeProps) => { diff --git a/airflow/www/static/js/dag/details/graph/utils.ts b/airflow/www/static/js/dag/details/graph/utils.ts index 2fb7351525e71..dba77cc511c04 100644 --- a/airflow/www/static/js/dag/details/graph/utils.ts +++ b/airflow/www/static/js/dag/details/graph/utils.ts @@ -24,7 +24,7 @@ import type { ElkExtendedEdge } from "elkjs"; import type { SelectionProps } from "src/dag/useSelection"; import { getTask } from "src/utils"; import type { Task, TaskInstance, NodeType } from "src/types"; -import type { DatasetEvent } from "src/types/api-generated"; +import type { AssetEvent } from "src/types/api-generated"; import type { CustomNodeProps } from "./Node"; @@ -38,7 +38,7 @@ interface FlattenNodesProps { onToggleGroups: (groupIds: string[]) => void; hoveredTaskState?: string | null; isZoomedOut: boolean; - datasetEvents?: DatasetEvent[]; + datasetEvents?: AssetEvent[]; } // Generate a flattened list of nodes for react-flow to render diff --git a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx index 3eb16380101a5..a0f3ff704b2fd 100644 --- a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx @@ -23,7 +23,7 @@ import { useAssetEvents } from "src/api"; import type { DagRun as DagRunType } from "src/types"; import { getMetaValue } from "src/utils"; import { CardDef, CardList } from "src/components/Table"; -import type { DatasetEvent } from "src/types/api-generated"; +import type { AssetEvent } from "src/types/api-generated"; import DatasetEventCard from "src/components/DatasetEventCard"; interface Props { @@ -31,7 +31,7 @@ interface Props { taskId: string; } -const cardDef: CardDef = { +const cardDef: CardDef = { card: ({ row }) => , }; diff --git a/airflow/www/static/js/datasets/AssetEvents.tsx b/airflow/www/static/js/datasets/AssetEvents.tsx index a651a9184843d..06998c633f2d1 100644 --- a/airflow/www/static/js/datasets/AssetEvents.tsx +++ b/airflow/www/static/js/datasets/AssetEvents.tsx @@ -25,7 +25,7 @@ import { Box, Flex, Heading, Select } from "@chakra-ui/react"; import { useAssetEvents } from "src/api"; import { CardList, type CardDef } from "src/components/Table"; -import type { DatasetEvent } from "src/types/api-generated"; +import type { AssetEvent } from "src/types/api-generated"; import DatasetEventCard from "src/components/DatasetEventCard"; type Props = { @@ -33,7 +33,7 @@ type Props = { showLabel?: boolean; }; -const cardDef: CardDef = { +const cardDef: CardDef = { card: ({ row }) => , }; diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 65002937abb30..9d2df16fd852d 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1902,24 +1902,24 @@ export interface components { datasets?: components["schemas"]["Dataset"][]; } & components["schemas"]["CollectionInfo"]; /** - * @description A dataset event. + * @description An asset event. * * *New in version 2.4.0* */ - DatasetEvent: { - /** @description The dataset id */ + AssetEvent: { + /** @description The asset id */ dataset_id?: number; - /** @description The URI of the dataset */ + /** @description The URI of the asset */ dataset_uri?: string; - /** @description The dataset event extra */ + /** @description The asset event extra */ extra?: { [key: string]: unknown } | null; - /** @description The DAG ID that updated the dataset. */ + /** @description The DAG ID that updated the asset. */ source_dag_id?: string | null; - /** @description The task ID that updated the dataset. */ + /** @description The task ID that updated the asset. */ source_task_id?: string | null; - /** @description The DAG run ID that updated the dataset. */ + /** @description The DAG run ID that updated the asset. */ source_run_id?: string | null; - /** @description The task map index that updated the dataset. */ + /** @description The task map index that updated the asset. */ source_map_index?: number | null; created_dagruns?: components["schemas"]["BasicDAGRun"][]; /** @description The dataset event creation time */ @@ -1988,7 +1988,7 @@ export interface components { * *New in version 2.4.0* */ AssetEventCollection: { - dataset_events?: components["schemas"]["DatasetEvent"][]; + dataset_events?: components["schemas"]["AssetEvent"][]; } & components["schemas"]["CollectionInfo"]; /** @description The option of configuration. */ ConfigOption: { @@ -5145,7 +5145,7 @@ export interface operations { /** Success. */ 200: { content: { - "application/json": components["schemas"]["DatasetEvent"]; + "application/json": components["schemas"]["AssetEvent"]; }; }; 400: components["responses"]["BadRequest"]; @@ -5434,8 +5434,8 @@ export type DagScheduleAssetReference = CamelCasedPropertiesDeep< export type AssetCollection = CamelCasedPropertiesDeep< components["schemas"]["AssetCollection"] >; -export type DatasetEvent = CamelCasedPropertiesDeep< - components["schemas"]["DatasetEvent"] +export type AssetEvent = CamelCasedPropertiesDeep< + components["schemas"]["AssetEvent"] >; export type CreateAssetEvent = CamelCasedPropertiesDeep< components["schemas"]["CreateAssetEvent"] diff --git a/clients/python/README.md b/clients/python/README.md index de9dfbbbe4afe..12986158141b7 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -402,6 +402,11 @@ Class | Method | HTTP request | Description - [ActionCollection](docs/ActionCollection.md) - [ActionCollectionAllOf](docs/ActionCollectionAllOf.md) - [ActionResource](docs/ActionResource.md) + - [AssetCollection](docs/AssetCollection.md) + - [AssetCollectionAllOf](docs/AssetCollectionAllOf.md) + - [AssetEvent](docs/AssetEvent.md) + - [AssetEventCollection](docs/AssetEventCollection.md) + - [AssetEventCollectionAllOf](docs/AssetEventCollectionAllOf.md) - [BasicDAGRun](docs/BasicDAGRun.md) - [ClassReference](docs/ClassReference.md) - [ClearDagRun](docs/ClearDagRun.md) @@ -432,11 +437,6 @@ Class | Method | HTTP request | Description - [DagWarningCollection](docs/DagWarningCollection.md) - [DagWarningCollectionAllOf](docs/DagWarningCollectionAllOf.md) - [Dataset](docs/Dataset.md) - - [AssetCollection](docs/AssetCollection.md) - - [AssetCollectionAllOf](docs/AssetCollectionAllOf.md) - - [DatasetEvent](docs/DatasetEvent.md) - - [AssetEventCollection](docs/AssetEventCollection.md) - - [AssetEventCollectionAllOf](docs/AssetEventCollectionAllOf.md) - [Error](docs/Error.md) - [EventLog](docs/EventLog.md) - [EventLogCollection](docs/EventLogCollection.md) From 70e14f80b7e5589cb47cc3d114172aee12a5bcc9 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 21:40:52 +0900 Subject: [PATCH 33/43] feat(api_connexion/openapi/v1): rename Dataset as Asset --- airflow/api_connexion/openapi/v1.yaml | 20 ++++----- airflow/www/static/js/api/useAsset.ts | 2 +- airflow/www/static/js/datasets/AssetsList.tsx | 2 +- airflow/www/static/js/types/api-generated.ts | 42 +++++++++---------- airflow/www/static/js/types/index.ts | 2 +- 5 files changed, 33 insertions(+), 35 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index fbe7af34616ee..a3dc07e8f9b84 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2570,7 +2570,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Dataset" + $ref: "#/components/schemas/Asset" "401": $ref: "#/components/responses/Unauthenticated" "403": @@ -4507,31 +4507,31 @@ components: $ref: "#/components/schemas/Resource" description: The permission resource - Dataset: + Asset: description: | - A dataset item. + An asset item. *New in version 2.4.0* type: object properties: id: type: integer - description: The dataset id + description: The asset id uri: type: string - description: The dataset uri + description: The asset uri nullable: false extra: type: object - description: The dataset extra + description: The asset extra nullable: true created_at: type: string - description: The dataset creation time + description: The asset creation time nullable: false updated_at: type: string - description: The dataset update time + description: The asset update time nullable: false consuming_dags: type: array @@ -4598,7 +4598,7 @@ components: datasets: type: array items: - $ref: "#/components/schemas/Dataset" + $ref: "#/components/schemas/Asset" - $ref: "#/components/schemas/CollectionInfo" AssetEvent: @@ -6024,12 +6024,12 @@ components: security: [] tags: + - name: Asset - name: Config - name: Connection - name: DAG - name: DAGRun - name: DagWarning - - name: Dataset - name: EventLog - name: ImportError - name: Monitoring diff --git a/airflow/www/static/js/api/useAsset.ts b/airflow/www/static/js/api/useAsset.ts index 106be25aee3df..ea9b30f413671 100644 --- a/airflow/www/static/js/api/useAsset.ts +++ b/airflow/www/static/js/api/useAsset.ts @@ -33,6 +33,6 @@ export default function useAsset({ uri }: Props) { "__URI__", encodeURIComponent(uri) ); - return axios.get(datasetUrl); + return axios.get(datasetUrl); }); } diff --git a/airflow/www/static/js/datasets/AssetsList.tsx b/airflow/www/static/js/datasets/AssetsList.tsx index ff1867e44cc57..c1f3e4b8b4cff 100644 --- a/airflow/www/static/js/datasets/AssetsList.tsx +++ b/airflow/www/static/js/datasets/AssetsList.tsx @@ -99,7 +99,7 @@ const AssetsList = ({ onSelect }: Props) => { const data = useMemo(() => datasets, [datasets]); const memoSort = useMemo(() => sortBy, [sortBy]); - const onDatasetSelect = (row: Row) => { + const onDatasetSelect = (row: Row) => { if (row.original.uri) onSelect({ uri: row.original.uri }); }; diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 9d2df16fd852d..aeb9ef88e4a08 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1847,20 +1847,20 @@ export interface components { resource?: components["schemas"]["Resource"]; }; /** - * @description A dataset item. + * @description An asset item. * * *New in version 2.4.0* */ - Dataset: { - /** @description The dataset id */ + Asset: { + /** @description The asset id */ id?: number; - /** @description The dataset uri */ + /** @description The asset uri */ uri?: string; - /** @description The dataset extra */ + /** @description The asset extra */ extra?: { [key: string]: unknown } | null; - /** @description The dataset creation time */ + /** @description The asset creation time */ created_at?: string; - /** @description The dataset update time */ + /** @description The asset update time */ updated_at?: string; consuming_dags?: components["schemas"]["DagScheduleAssetReference"][]; producing_tasks?: components["schemas"]["TaskOutletAssetReference"][]; @@ -1899,7 +1899,7 @@ export interface components { * *New in version 2.4.0* */ AssetCollection: { - datasets?: components["schemas"]["Dataset"][]; + datasets?: components["schemas"]["Asset"][]; } & components["schemas"]["CollectionInfo"]; /** * @description An asset event. @@ -1922,7 +1922,7 @@ export interface components { /** @description The task map index that updated the asset. */ source_map_index?: number | null; created_dagruns?: components["schemas"]["BasicDAGRun"][]; - /** @description The dataset event creation time */ + /** @description The asset event creation time */ timestamp?: string; }; CreateAssetEvent: { @@ -1943,7 +1943,7 @@ export interface components { created_at?: string; }; /** - * @description A collection of Dataset Dag Run Queues. + * @description A collection of asset Dag Run Queues. * * *New in version 2.9.0* */ @@ -2623,13 +2623,13 @@ export interface components { FilterTags: string[]; /** @description The Asset ID that updated the asset. */ FilterAssetID: number; - /** @description The DAG ID that updated the dataset. */ + /** @description The DAG ID that updated the asset. */ FilterSourceDAGID: string; - /** @description The task ID that updated the dataset. */ + /** @description The task ID that updated the asset. */ FilterSourceTaskID: string; - /** @description The DAG run ID that updated the dataset. */ + /** @description The DAG run ID that updated the asset. */ FilterSourceRunID: string; - /** @description The map index that updated the dataset. */ + /** @description The map index that updated the asset. */ FilterSourceMapIndex: number; /** @description Filter on map index for mapped task. */ FilterMapIndex: number; @@ -5092,7 +5092,7 @@ export interface operations { /** Success. */ 200: { content: { - "application/json": components["schemas"]["Dataset"]; + "application/json": components["schemas"]["Asset"]; }; }; 401: components["responses"]["Unauthenticated"]; @@ -5117,13 +5117,13 @@ export interface operations { order_by?: components["parameters"]["OrderBy"]; /** The Asset ID that updated the asset. */ asset_id?: components["parameters"]["FilterAssetID"]; - /** The DAG ID that updated the dataset. */ + /** The DAG ID that updated the asset. */ source_dag_id?: components["parameters"]["FilterSourceDAGID"]; - /** The task ID that updated the dataset. */ + /** The task ID that updated the asset. */ source_task_id?: components["parameters"]["FilterSourceTaskID"]; - /** The DAG run ID that updated the dataset. */ + /** The DAG run ID that updated the asset. */ source_run_id?: components["parameters"]["FilterSourceRunID"]; - /** The map index that updated the dataset. */ + /** The map index that updated the asset. */ source_map_index?: components["parameters"]["FilterSourceMapIndex"]; }; }; @@ -5422,9 +5422,7 @@ export type Resource = CamelCasedPropertiesDeep< export type ActionResource = CamelCasedPropertiesDeep< components["schemas"]["ActionResource"] >; -export type Dataset = CamelCasedPropertiesDeep< - components["schemas"]["Dataset"] ->; +export type Asset = CamelCasedPropertiesDeep; export type TaskOutletAssetReference = CamelCasedPropertiesDeep< components["schemas"]["TaskOutletAssetReference"] >; diff --git a/airflow/www/static/js/types/index.ts b/airflow/www/static/js/types/index.ts index 1ce07bb350795..5ae6868647994 100644 --- a/airflow/www/static/js/types/index.ts +++ b/airflow/www/static/js/types/index.ts @@ -193,7 +193,7 @@ export interface WebserverEdge { isSourceDataset?: boolean; } -interface DatasetListItem extends API.Dataset { +interface DatasetListItem extends API.Asset { lastDatasetUpdate: string | null; totalUpdates: number; } From 1d1c80c1e7b3e94448e383de43ff5d904a596117 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 18:11:58 +0900 Subject: [PATCH 34/43] docs(api_connexion/openapi/v1): update dataset to asset in v1.yaml --- airflow/api_connexion/openapi/v1.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index a3dc07e8f9b84..bb5e9e75b2a35 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1251,9 +1251,9 @@ paths: - $ref: "#/components/parameters/AssetURI" get: - summary: Get a queued Dataset event for a DAG + summary: Get a queued asset event for a DAG description: | - Get a queued Dataset event for a DAG. + Get a queued asset event for a DAG. *New in version 2.9.0* x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint @@ -2559,8 +2559,8 @@ paths: parameters: - $ref: "#/components/parameters/AssetURI" get: - summary: Get a dataset - description: Get a dataset by uri. + summary: Get an asset + description: Get an asset by uri. x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint operationId: get_asset tags: [Asset] @@ -4133,7 +4133,7 @@ components: nullable: true dataset_expression: type: object - description: Nested dataset any/all conditions + description: Nested asset any/all conditions nullable: true doc_md: type: string @@ -4559,7 +4559,7 @@ components: nullable: true created_at: type: string - description: The dataset creation time + description: The asset creation time nullable: false updated_at: type: string @@ -4674,7 +4674,7 @@ components: QueuedEventCollection: description: | - A collection of Dataset Dag Run Queues. + A collection of asset Dag Run Queues. *New in version 2.9.0* type: object @@ -5745,28 +5745,28 @@ components: name: source_dag_id schema: type: string - description: The DAG ID that updated the dataset. + description: The DAG ID that updated the asset. FilterSourceTaskID: in: query name: source_task_id schema: type: string - description: The task ID that updated the dataset. + description: The task ID that updated the asset. FilterSourceRunID: in: query name: source_run_id schema: type: string - description: The DAG run ID that updated the dataset. + description: The DAG run ID that updated the asset. FilterSourceMapIndex: in: query name: source_map_index schema: type: integer - description: The map index that updated the dataset. + description: The map index that updated the asset. FilterMapIndex: in: query From 9982a6d1599c9ca6a13058a70e63d0c92c6b5fa6 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 21:46:43 +0900 Subject: [PATCH 35/43] feat(api_connexion): rename endpoint datasets as assets --- .../api_connexion/endpoints/asset_endpoint.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 12 +- clients/python/README.md | 6 +- .../auth-manager/access-control.rst | 6 +- .../endpoints/test_asset_endpoint.py | 276 +++++++++++++++--- 5 files changed, 242 insertions(+), 60 deletions(-) diff --git a/airflow/api_connexion/endpoints/asset_endpoint.py b/airflow/api_connexion/endpoints/asset_endpoint.py index 9fc501633800c..cbbe542ea7987 100644 --- a/airflow/api_connexion/endpoints/asset_endpoint.py +++ b/airflow/api_connexion/endpoints/asset_endpoint.py @@ -57,7 +57,7 @@ from airflow.api_connexion.types import APIResponse -RESOURCE_EVENT_PREFIX = "dataset" +RESOURCE_EVENT_PREFIX = "asset" @security.requires_access_asset("GET") diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index bb5e9e75b2a35..75b0df529004c 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -1245,7 +1245,7 @@ paths: "404": $ref: "#/components/responses/NotFound" - /dags/{dag_id}/datasets/queuedEvent/{uri}: + /dags/{dag_id}/assets/queuedEvent/{uri}: parameters: - $ref: "#/components/parameters/DAGID" - $ref: "#/components/parameters/AssetURI" @@ -1298,7 +1298,7 @@ paths: "404": $ref: "#/components/responses/NotFound" - /dags/{dag_id}/datasets/queuedEvent: + /dags/{dag_id}/assets/queuedEvent: parameters: - $ref: "#/components/parameters/DAGID" @@ -1371,7 +1371,7 @@ paths: "404": $ref: "#/components/responses/NotFound" - /datasets/queuedEvent/{uri}: + /assets/queuedEvent/{uri}: parameters: - $ref: "#/components/parameters/AssetURI" @@ -2517,7 +2517,7 @@ paths: "403": $ref: "#/components/responses/PermissionDenied" - /datasets: + /assets: get: summary: List assets x-openapi-router-controller: airflow.api_connexion.endpoints.asset_endpoint @@ -2555,7 +2555,7 @@ paths: "403": $ref: "#/components/responses/PermissionDenied" - /datasets/{uri}: + /assets/{uri}: parameters: - $ref: "#/components/parameters/AssetURI" get: @@ -2578,7 +2578,7 @@ paths: "404": $ref: "#/components/responses/NotFound" - /datasets/events: + /assets/events: get: summary: Get asset events description: Get asset events diff --git a/clients/python/README.md b/clients/python/README.md index 12986158141b7..e1427fce92d46 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -348,9 +348,9 @@ Class | Method | HTTP request | Description *DAGRunApi* | [**set_dag_run_note**](docs/DAGRunApi.md#set_dag_run_note) | **PATCH** /dags/{dag_id}/dagRuns/{dag_run_id}/setNote | Update the DagRun note. *DAGRunApi* | [**update_dag_run_state**](docs/DAGRunApi.md#update_dag_run_state) | **PATCH** /dags/{dag_id}/dagRuns/{dag_run_id} | Modify a DAG run *DagWarningApi* | [**get_dag_warnings**](docs/DagWarningApi.md#get_dag_warnings) | **GET** /dagWarnings | List dag warnings -*AssetApi* | [**get_asset**](docs/DatasetApi.md#get_asset) | **GET** /datasets/{uri} | Get an asset -*AssetApi* | [**get_asset_events**](docs/DatasetApi.md#get_asset_events) | **GET** /datasets/events | Get asset events -*DatasetApi* | [**get_assets**](docs/DatasetApi.md#get_assets) | **GET** /datasets | List assets +*AssetApi* | [**get_asset**](docs/DatasetApi.md#get_asset) | **GET** /assets/{uri} | Get an asset +*AssetApi* | [**get_asset_events**](docs/DatasetApi.md#get_asset_events) | **GET** /assets/events | Get asset events +*DatasetApi* | [**get_assets**](docs/DatasetApi.md#get_assets) | **GET** /assets | List assets *DatasetApi* | [**get_upstream_asset_events**](docs/DatasetApi.md#get_upstream_asset_events) | **GET** /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamAssetEvents | Get dataset events for a DAG run *EventLogApi* | [**get_event_log**](docs/EventLogApi.md#get_event_log) | **GET** /eventLogs/{event_log_id} | Get a log entry *EventLogApi* | [**get_event_logs**](docs/EventLogApi.md#get_event_logs) | **GET** /eventLogs | List log entries diff --git a/docs/apache-airflow-providers-fab/auth-manager/access-control.rst b/docs/apache-airflow-providers-fab/auth-manager/access-control.rst index 0363e9c7eb8b1..5292c3f1f6296 100644 --- a/docs/apache-airflow-providers-fab/auth-manager/access-control.rst +++ b/docs/apache-airflow-providers-fab/auth-manager/access-control.rst @@ -166,9 +166,9 @@ Endpoint /dags/{dag_id}/dagRuns/{dag_run_id} DELETE DAGs.can_edit, DAG Runs.can_delete User /dags/{dag_id}/dagRuns/{dag_run_id} GET DAGs.can_read, DAG Runs.can_read Viewer /dags/~/dagRuns/list POST DAGs.can_edit, DAG Runs.can_read User -/datasets GET Assets.can_read Viewer -/datasets/{uri} GET Assets.can_read Viewer -/datasets/events GET Assets.can_read Viewer +/assets GET Assets.can_read Viewer +/assets/{uri} GET Assets.can_read Viewer +/assets/events GET Assets.can_read Viewer /eventLogs GET Audit Logs.can_read Viewer /eventLogs/{event_log_id} GET Audit Logs.can_read Viewer /importErrors GET ImportError.can_read Viewer diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index 0b8d608018361..c3014fd25e331 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -95,7 +95,7 @@ def test_should_respond_200(self, session): with assert_queries_count(6): response = self.client.get( - f"/api/v1/datasets/{urllib.parse.quote('s3://bucket/key', safe='')}", + f"/api/v1/assets/{urllib.parse.quote('s3://bucket/key', safe='')}", environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 200 @@ -112,7 +112,7 @@ def test_should_respond_200(self, session): def test_should_respond_404(self): response = self.client.get( - f"/api/v1/datasets/{urllib.parse.quote('s3://bucket/key', safe='')}", + f"/api/v1/assets/{urllib.parse.quote('s3://bucket/key', safe='')}", environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 404 @@ -125,7 +125,7 @@ def test_should_respond_404(self): def test_should_raises_401_unauthenticated(self, session): self._create_dataset(session) - response = self.client.get(f"/api/v1/datasets/{urllib.parse.quote('s3://bucket/key', safe='')}") + response = self.client.get(f"/api/v1/assets/{urllib.parse.quote('s3://bucket/key', safe='')}") assert_401(response) @@ -146,7 +146,7 @@ def test_should_respond_200(self, session): assert session.query(AssetModel).count() == 2 with assert_queries_count(10): - response = self.client.get("/api/v1/datasets", environ_overrides={"REMOTE_USER": "test"}) + response = self.client.get("/api/v1/assets", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 response_data = response.json @@ -191,7 +191,7 @@ def test_order_by_raises_400_for_invalid_attr(self, session): assert session.query(AssetModel).count() == 2 response = self.client.get( - "/api/v1/datasets?order_by=fake", environ_overrides={"REMOTE_USER": "test"} + "/api/v1/assets?order_by=fake", environ_overrides={"REMOTE_USER": "test"} ) # missing attr assert response.status_code == 400 @@ -212,21 +212,21 @@ def test_should_raises_401_unauthenticated(self, session): session.commit() assert session.query(AssetModel).count() == 2 - response = self.client.get("/api/v1/datasets") + response = self.client.get("/api/v1/assets") assert_401(response) @pytest.mark.parametrize( "url, expected_datasets", [ - ("api/v1/datasets?uri_pattern=s3", {"s3://folder/key"}), - ("api/v1/datasets?uri_pattern=bucket", {"gcp://bucket/key", "wasb://some_dataset_bucket_/key"}), + ("api/v1/assets?uri_pattern=s3", {"s3://folder/key"}), + ("api/v1/assets?uri_pattern=bucket", {"gcp://bucket/key", "wasb://some_dataset_bucket_/key"}), ( - "api/v1/datasets?uri_pattern=dataset", + "api/v1/assets?uri_pattern=dataset", {"somescheme://dataset/key", "wasb://some_dataset_bucket_/key"}, ), ( - "api/v1/datasets?uri_pattern=", + "api/v1/assets?uri_pattern=", { "gcp://bucket/key", "s3://folder/key", @@ -266,7 +266,7 @@ def test_filter_datasets_by_dag_ids_works(self, dag_ids, expected_num, session): session.add_all([dataset1, dataset2, dataset3, dag1, dag2, dag3, dag_ref1, dag_ref2, task_ref1]) session.commit() response = self.client.get( - f"/api/v1/datasets?dag_ids={dag_ids}", environ_overrides={"REMOTE_USER": "test"} + f"/api/v1/assets?dag_ids={dag_ids}", environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 200 response_data = response.json @@ -293,7 +293,7 @@ def test_filter_datasets_by_dag_ids_and_uri_pattern_works( session.add_all([asset1, asset2, asset3, dag1, dag2, dag3, dag_ref1, dag_ref2, task_ref1]) session.commit() response = self.client.get( - f"/api/v1/datasets?dag_ids={dag_ids}&uri_pattern={uri_pattern}", + f"/api/v1/assets?dag_ids={dag_ids}&uri_pattern={uri_pattern}", environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 200 @@ -306,13 +306,13 @@ class TestGetDatasetsEndpointPagination(TestDatasetEndpoint): "url, expected_dataset_uris", [ # Limit test data - ("/api/v1/datasets?limit=1", ["s3://bucket/key/1"]), - ("/api/v1/datasets?limit=100", [f"s3://bucket/key/{i}" for i in range(1, 101)]), + ("/api/v1/assets?limit=1", ["s3://bucket/key/1"]), + ("/api/v1/assets?limit=100", [f"s3://bucket/key/{i}" for i in range(1, 101)]), # Offset test data - ("/api/v1/datasets?offset=1", [f"s3://bucket/key/{i}" for i in range(2, 102)]), - ("/api/v1/datasets?offset=3", [f"s3://bucket/key/{i}" for i in range(4, 104)]), + ("/api/v1/assets?offset=1", [f"s3://bucket/key/{i}" for i in range(2, 102)]), + ("/api/v1/assets?offset=3", [f"s3://bucket/key/{i}" for i in range(4, 104)]), # Limit and offset test data - ("/api/v1/datasets?offset=3&limit=3", [f"s3://bucket/key/{i}" for i in [4, 5, 6]]), + ("/api/v1/assets?offset=3&limit=3", [f"s3://bucket/key/{i}" for i in [4, 5, 6]]), ], ) @provide_session @@ -348,7 +348,7 @@ def test_should_respect_page_size_limit_default(self, session): session.add_all(assets) session.commit() - response = self.client.get("/api/v1/datasets", environ_overrides={"REMOTE_USER": "test"}) + response = self.client.get("/api/v1/assets", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 assert len(response.json["datasets"]) == 100 @@ -367,7 +367,7 @@ def test_should_return_conf_max_if_req_max_above_conf(self, session): session.add_all(assets) session.commit() - response = self.client.get("/api/v1/datasets?limit=180", environ_overrides={"REMOTE_USER": "test"}) + response = self.client.get("/api/v1/assets?limit=180", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 assert len(response.json["datasets"]) == 150 @@ -391,7 +391,7 @@ def test_should_respond_200(self, session): session.commit() assert session.query(AssetEvent).count() == 2 - response = self.client.get("/api/v1/datasets/events", environ_overrides={"REMOTE_USER": "test"}) + response = self.client.get("/api/v1/assets/events", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 response_data = response.json @@ -454,7 +454,7 @@ def test_filtering(self, attr, value, session): assert session.query(AssetEvent).count() == 3 response = self.client.get( - f"/api/v1/datasets/events?{attr}={value}", environ_overrides={"REMOTE_USER": "test"} + f"/api/v1/assets/events?{attr}={value}", environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 200 @@ -496,7 +496,7 @@ def test_order_by_raises_400_for_invalid_attr(self, session): assert session.query(AssetEvent).count() == 2 response = self.client.get( - "/api/v1/datasets/events?order_by=fake", environ_overrides={"REMOTE_USER": "test"} + "/api/v1/assets/events?order_by=fake", environ_overrides={"REMOTE_USER": "test"} ) # missing attr assert response.status_code == 400 @@ -504,7 +504,7 @@ def test_order_by_raises_400_for_invalid_attr(self, session): assert response.json["detail"] == msg def test_should_raises_401_unauthenticated(self, session): - response = self.client.get("/api/v1/datasets/events") + response = self.client.get("/api/v1/assets/events") assert_401(response) def test_includes_created_dagrun(self, session): @@ -533,7 +533,7 @@ def test_includes_created_dagrun(self, session): event.created_dagruns.append(dagrun) session.commit() - response = self.client.get("/api/v1/datasets/events", environ_overrides={"REMOTE_USER": "test"}) + response = self.client.get("/api/v1/assets/events", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 response_data = response.json @@ -582,7 +582,7 @@ def test_should_respond_200(self, session): self._create_dataset(session) event_payload = {"asset_uri": "s3://bucket/key", "extra": {"foo": "bar"}} response = self.client.post( - "/api/v1/datasets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} + "/api/v1/assets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 200 @@ -612,7 +612,7 @@ def test_should_mask_sensitive_extra_logs(self, session): self._create_dataset(session) event_payload = {"asset_uri": "s3://bucket/key", "extra": {"password": "bar"}} response = self.client.post( - "/api/v1/datasets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} + "/api/v1/assets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 200 @@ -629,13 +629,13 @@ def test_order_by_raises_400_for_invalid_attr(self, session): self._create_dataset(session) event_invalid_payload = {"asset_uri": "TEST_ASSET_URI", "extra": {"foo": "bar"}, "fake": {}} response = self.client.post( - "/api/v1/datasets/events", json=event_invalid_payload, environ_overrides={"REMOTE_USER": "test"} + "/api/v1/assets/events", json=event_invalid_payload, environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 400 def test_should_raises_401_unauthenticated(self, session): self._create_dataset(session) - response = self.client.post("/api/v1/datasets/events", json={"asset_uri": "TEST_ASSET_URI"}) + response = self.client.post("/api/v1/assets/events", json={"asset_uri": "TEST_ASSET_URI"}) assert_401(response) @@ -644,23 +644,23 @@ class TestGetDatasetEventsEndpointPagination(TestDatasetEndpoint): "url, expected_event_runids", [ # Limit test data - ("/api/v1/datasets/events?limit=1&order_by=source_run_id", ["run1"]), + ("/api/v1/assets/events?limit=1&order_by=source_run_id", ["run1"]), ( - "/api/v1/datasets/events?limit=3&order_by=source_run_id", + "/api/v1/assets/events?limit=3&order_by=source_run_id", [f"run{i}" for i in range(1, 4)], ), # Offset test data ( - "/api/v1/datasets/events?offset=1&order_by=source_run_id", + "/api/v1/assets/events?offset=1&order_by=source_run_id", [f"run{i}" for i in range(2, 10)], ), ( - "/api/v1/datasets/events?offset=3&order_by=source_run_id", + "/api/v1/assets/events?offset=3&order_by=source_run_id", [f"run{i}" for i in range(4, 10)], ), # Limit and offset test data ( - "/api/v1/datasets/events?offset=3&limit=3&order_by=source_run_id", + "/api/v1/assets/events?offset=3&limit=3&order_by=source_run_id", [f"run{i}" for i in [4, 5, 6]], ), ], @@ -704,7 +704,7 @@ def test_should_respect_page_size_limit_default(self, session): session.add_all(events) session.commit() - response = self.client.get("/api/v1/datasets/events", environ_overrides={"REMOTE_USER": "test"}) + response = self.client.get("/api/v1/assets/events", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 assert len(response.json["asset_events"]) == 100 @@ -727,7 +727,7 @@ def test_should_return_conf_max_if_req_max_above_conf(self, session): session.commit() response = self.client.get( - "/api/v1/datasets/events?limit=180", environ_overrides={"REMOTE_USER": "test"} + "/api/v1/assets/events?limit=180", environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 200 @@ -752,11 +752,51 @@ def _create_dataset_dag_run_queues(self, dag_id, dataset_id, session): class TestGetDagDatasetQueuedEvent(TestQueuedEventEndpoint): +<<<<<<< HEAD +======= + @pytest.mark.usefixtures("time_freezer") + def test_should_respond_200(self, session, create_dummy_dag): + dag, _ = create_dummy_dag() + dag_id = dag.dag_id + dataset_id = self._create_dataset(session).id + self._create_dataset_dag_run_queues(dag_id, dataset_id, session) + dataset_uri = "s3://bucket/key" + + response = self.client.get( + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 200 + assert response.json == { + "created_at": self.default_time, + "uri": "s3://bucket/key", + "dag_id": "dag", + } + + def test_should_respond_404(self): + dag_id = "not_exists" + dataset_uri = "not_exists" + + response = self.client.get( + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 404 + assert { + "detail": "Queue event with dag_id: `not_exists` and asset uri: `not_exists` was not found", + "status": 404, + "title": "Queue event not found", + "type": EXCEPTIONS_LINK_MAP[404], + } == response.json + +>>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self, session): dag_id = "dummy" dataset_uri = "dummy" - response = self.client.get(f"/api/v1/dags/{dag_id}/datasets/queuedEvent/{dataset_uri}") + response = self.client.get(f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}") assert_401(response) @@ -765,7 +805,7 @@ def test_should_raise_403_forbidden(self, session): dataset_uri = "dummy" response = self.client.get( - f"/api/v1/dags/{dag_id}/datasets/queuedEvent/{dataset_uri}", + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) @@ -776,24 +816,67 @@ class TestDeleteDagDatasetQueuedEvent(TestDatasetEndpoint): def test_should_raises_401_unauthenticated(self, session): dag_id = "dummy" dataset_uri = "dummy" - response = self.client.delete(f"/api/v1/dags/{dag_id}/datasets/queuedEvent/{dataset_uri}") + response = self.client.delete(f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}") assert_401(response) def test_should_raise_403_forbidden(self, session): dag_id = "dummy" dataset_uri = "dummy" response = self.client.delete( - f"/api/v1/dags/{dag_id}/datasets/queuedEvent/{dataset_uri}", + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) assert response.status_code == 403 class TestGetDagDatasetQueuedEvents(TestQueuedEventEndpoint): +<<<<<<< HEAD +======= + @pytest.mark.usefixtures("time_freezer") + def test_should_respond_200(self, session, create_dummy_dag): + dag, _ = create_dummy_dag() + dag_id = dag.dag_id + dataset_id = self._create_dataset(session).id + self._create_dataset_dag_run_queues(dag_id, dataset_id, session) + + response = self.client.get( + f"/api/v1/dags/{dag_id}/assets/queuedEvent", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 200 + assert response.json == { + "queued_events": [ + { + "created_at": self.default_time, + "uri": "s3://bucket/key", + "dag_id": "dag", + } + ], + "total_entries": 1, + } + + def test_should_respond_404(self): + dag_id = "not_exists" + + response = self.client.get( + f"/api/v1/dags/{dag_id}/assets/queuedEvent", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 404 + assert { + "detail": "Queue event with dag_id: `not_exists` was not found", + "status": 404, + "title": "Queue event not found", + "type": EXCEPTIONS_LINK_MAP[404], + } == response.json + +>>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): dag_id = "dummy" - response = self.client.get(f"/api/v1/dags/{dag_id}/datasets/queuedEvent") + response = self.client.get(f"/api/v1/dags/{dag_id}/assets/queuedEvent") assert_401(response) @@ -801,7 +884,7 @@ def test_should_raise_403_forbidden(self): dag_id = "dummy" response = self.client.get( - f"/api/v1/dags/{dag_id}/datasets/queuedEvent", + f"/api/v1/dags/{dag_id}/assets/queuedEvent", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) @@ -809,10 +892,29 @@ def test_should_raise_403_forbidden(self): class TestDeleteDagDatasetQueuedEvents(TestDatasetEndpoint): +<<<<<<< HEAD +======= + def test_should_respond_404(self): + dag_id = "not_exists" + + response = self.client.delete( + f"/api/v1/dags/{dag_id}/assets/queuedEvent", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 404 + assert { + "detail": "Queue event with dag_id: `not_exists` was not found", + "status": 404, + "title": "Queue event not found", + "type": EXCEPTIONS_LINK_MAP[404], + } == response.json + +>>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): dag_id = "dummy" - response = self.client.delete(f"/api/v1/dags/{dag_id}/datasets/queuedEvent") + response = self.client.delete(f"/api/v1/dags/{dag_id}/assets/queuedEvent") assert_401(response) @@ -820,7 +922,7 @@ def test_should_raise_403_forbidden(self): dag_id = "dummy" response = self.client.delete( - f"/api/v1/dags/{dag_id}/datasets/queuedEvent", + f"/api/v1/dags/{dag_id}/assets/queuedEvent", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) @@ -828,10 +930,54 @@ def test_should_raise_403_forbidden(self): class TestGetDatasetQueuedEvents(TestQueuedEventEndpoint): +<<<<<<< HEAD +======= + @pytest.mark.usefixtures("time_freezer") + def test_should_respond_200(self, session, create_dummy_dag): + dag, _ = create_dummy_dag() + dag_id = dag.dag_id + dataset_id = self._create_dataset(session).id + self._create_dataset_dag_run_queues(dag_id, dataset_id, session) + dataset_uri = "s3://bucket/key" + + response = self.client.get( + f"/api/v1/assets/queuedEvent/{dataset_uri}", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 200 + assert response.json == { + "queued_events": [ + { + "created_at": self.default_time, + "uri": "s3://bucket/key", + "dag_id": "dag", + } + ], + "total_entries": 1, + } + + def test_should_respond_404(self): + dataset_uri = "not_exists" + + response = self.client.get( + f"/api/v1/assets/queuedEvent/{dataset_uri}", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 404 + assert { + "detail": "Queue event with asset uri: `not_exists` was not found", + "status": 404, + "title": "Queue event not found", + "type": EXCEPTIONS_LINK_MAP[404], + } == response.json + +>>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): dataset_uri = "not_exists" - response = self.client.get(f"/api/v1/datasets/queuedEvent/{dataset_uri}") + response = self.client.get(f"/api/v1/assets/queuedEvent/{dataset_uri}") assert_401(response) @@ -839,7 +985,7 @@ def test_should_raise_403_forbidden(self): dataset_uri = "not_exists" response = self.client.get( - f"/api/v1/datasets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{dataset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) @@ -847,10 +993,46 @@ def test_should_raise_403_forbidden(self): class TestDeleteAssetQueuedEvents(TestQueuedEventEndpoint): +<<<<<<< HEAD +======= + def test_delete_should_respond_204(self, session, create_dummy_dag): + dag, _ = create_dummy_dag() + dag_id = dag.dag_id + dataset_id = self._create_dataset(session).id + self._create_dataset_dag_run_queues(dag_id, dataset_id, session) + dataset_uri = "s3://bucket/key" + + response = self.client.delete( + f"/api/v1/assets/queuedEvent/{dataset_uri}", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 204 + conn = session.query(AssetDagRunQueue).all() + assert len(conn) == 0 + _check_last_log(session, dag_id=None, event="api.delete_asset_queued_events", execution_date=None) + + def test_should_respond_404(self): + dataset_uri = "not_exists" + + response = self.client.delete( + f"/api/v1/assets/queuedEvent/{dataset_uri}", + environ_overrides={"REMOTE_USER": "test_queued_event"}, + ) + + assert response.status_code == 404 + assert { + "detail": "Queue event with asset uri: `not_exists` was not found", + "status": 404, + "title": "Queue event not found", + "type": EXCEPTIONS_LINK_MAP[404], + } == response.json + +>>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): dataset_uri = "not_exists" - response = self.client.delete(f"/api/v1/datasets/queuedEvent/{dataset_uri}") + response = self.client.delete(f"/api/v1/assets/queuedEvent/{dataset_uri}") assert_401(response) @@ -858,7 +1040,7 @@ def test_should_raise_403_forbidden(self): dataset_uri = "not_exists" response = self.client.delete( - f"/api/v1/datasets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{dataset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) From 8c82e8af5c081d421e1bd5cc39afc9e7dd82a257 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 21:54:33 +0900 Subject: [PATCH 36/43] test(api_connexion): rename dataset as asset --- airflow/api_connexion/openapi/v1.yaml | 2 +- .../www/static/js/dag/details/graph/index.tsx | 14 +- airflow/www/static/js/types/api-generated.ts | 22 +- .../endpoints/test_asset_endpoint.py | 196 ++++++++++-------- .../schemas/test_dataset_schema.py | 12 +- 5 files changed, 131 insertions(+), 115 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 75b0df529004c..b0d022e50b382 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -4595,7 +4595,7 @@ components: allOf: - type: object properties: - datasets: + assets: type: array items: $ref: "#/components/schemas/Asset" diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index 39936e95df807..f5d3fe9bfc066 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -185,11 +185,11 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { if (showDatasets) { datasetNodes = [...upstreamDatasetNodes]; datasetEdges = [...upstreamDatasetEdges]; - datasetsCollection?.datasets?.forEach((dataset) => { - const producingTask = dataset?.producingTasks?.find( + datasetsCollection?.assets?.forEach((asset) => { + const producingTask = asset?.producingTasks?.find( (t) => t.dagId === dagId ); - if (dataset.uri) { + if (asset.uri) { // check that the task is in the graph if ( producingTask?.taskId && @@ -197,13 +197,13 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { ) { datasetEdges.push({ sourceId: producingTask.taskId, - targetId: dataset.uri, + targetId: asset.uri, }); datasetNodes.push({ - id: dataset.uri, + id: asset.uri, value: { class: "asset", - label: dataset.uri, + label: asset.uri, }, }); } @@ -335,7 +335,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { > - {!!datasetsCollection?.datasets?.length && ( + {!!datasetsCollection?.assets?.length && ( Show datasets: diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index aeb9ef88e4a08..6e9d219918b57 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -294,7 +294,7 @@ export interface paths { }; }; }; - "/dags/{dag_id}/datasets/queuedEvent/{uri}": { + "/dags/{dag_id}/assets/queuedEvent/{uri}": { /** * Get a queued asset event for a DAG. * @@ -316,7 +316,7 @@ export interface paths { }; }; }; - "/dags/{dag_id}/datasets/queuedEvent": { + "/dags/{dag_id}/assets/queuedEvent": { /** * Get queued Asset events for a DAG. * @@ -350,7 +350,7 @@ export interface paths { }; }; }; - "/datasets/queuedEvent/{uri}": { + "/assets/queuedEvent/{uri}": { /** * Get queued Asset events for an Asset * @@ -782,10 +782,10 @@ export interface paths { "/dagWarnings": { get: operations["get_dag_warnings"]; }; - "/datasets": { + "/assets": { get: operations["get_assets"]; }; - "/datasets/{uri}": { + "/assets/{uri}": { /** Get an asset by uri. */ get: operations["get_asset"]; parameters: { @@ -795,7 +795,7 @@ export interface paths { }; }; }; - "/datasets/events": { + "/assets/events": { /** Get asset events */ get: operations["get_asset_events"]; /** Create asset event */ @@ -1647,7 +1647,7 @@ export interface components { */ start_date?: string | null; dag_run_timeout?: components["schemas"]["TimeDelta"] | null; - /** @description Nested dataset any/all conditions */ + /** @description Nested asset any/all conditions */ dataset_expression?: { [key: string]: unknown } | null; doc_md?: string | null; default_view?: string | null; @@ -1875,7 +1875,7 @@ export interface components { dag_id?: string | null; /** @description The task ID that updates the asset. */ task_id?: string | null; - /** @description The dataset creation time */ + /** @description The asset creation time */ created_at?: string; /** @description The asset update time */ updated_at?: string; @@ -1899,7 +1899,7 @@ export interface components { * *New in version 2.4.0* */ AssetCollection: { - datasets?: components["schemas"]["Asset"][]; + assets?: components["schemas"]["Asset"][]; } & components["schemas"]["CollectionInfo"]; /** * @description An asset event. @@ -1932,7 +1932,7 @@ export interface components { extra?: { [key: string]: unknown } | null; }; QueuedEvent: { - /** @description The datata uri. */ + /** @description The asset uri. */ uri?: string; /** @description The DAG ID. */ dag_id?: string; @@ -1948,7 +1948,7 @@ export interface components { * *New in version 2.9.0* */ QueuedEventCollection: { - datasets?: components["schemas"]["QueuedEvent"][]; + queued_events?: components["schemas"]["QueuedEvent"][]; } & components["schemas"]["CollectionInfo"]; BasicDAGRun: { /** @description Run ID. */ diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index c3014fd25e331..bae498f5b7ce2 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -61,7 +61,7 @@ def configured_app(minimal_app_for_api): delete_user(app, username="test_no_permissions") -class TestDatasetEndpoint: +class TestAssetEndpoint: default_time = "2020-06-11T18:00:00+00:00" @pytest.fixture(autouse=True) @@ -75,7 +75,7 @@ def teardown_method(self) -> None: clear_db_assets() clear_db_runs() - def _create_dataset(self, session): + def _create_asset(self, session): asset_model = AssetModel( id=1, uri="s3://bucket/key", @@ -88,9 +88,9 @@ def _create_dataset(self, session): return asset_model -class TestGetDatasetEndpoint(TestDatasetEndpoint): +class TestGetAssetEndpoint(TestAssetEndpoint): def test_should_respond_200(self, session): - self._create_dataset(session) + self._create_asset(session) assert session.query(AssetModel).count() == 1 with assert_queries_count(6): @@ -124,12 +124,12 @@ def test_should_respond_404(self): } == response.json def test_should_raises_401_unauthenticated(self, session): - self._create_dataset(session) + self._create_asset(session) response = self.client.get(f"/api/v1/assets/{urllib.parse.quote('s3://bucket/key', safe='')}") assert_401(response) -class TestGetDatasets(TestDatasetEndpoint): +class TestGetAssets(TestAssetEndpoint): def test_should_respond_200(self, session): assets = [ AssetModel( @@ -151,7 +151,7 @@ def test_should_respond_200(self, session): assert response.status_code == 200 response_data = response.json assert response_data == { - "datasets": [ + "assets": [ { "id": 1, "uri": "s3://bucket/key/1", @@ -217,66 +217,66 @@ def test_should_raises_401_unauthenticated(self, session): assert_401(response) @pytest.mark.parametrize( - "url, expected_datasets", + "url, expected_assets", [ ("api/v1/assets?uri_pattern=s3", {"s3://folder/key"}), - ("api/v1/assets?uri_pattern=bucket", {"gcp://bucket/key", "wasb://some_dataset_bucket_/key"}), + ("api/v1/assets?uri_pattern=bucket", {"gcp://bucket/key", "wasb://some_asset_bucket_/key"}), ( - "api/v1/assets?uri_pattern=dataset", - {"somescheme://dataset/key", "wasb://some_dataset_bucket_/key"}, + "api/v1/assets?uri_pattern=asset", + {"somescheme://asset/key", "wasb://some_asset_bucket_/key"}, ), ( "api/v1/assets?uri_pattern=", { "gcp://bucket/key", "s3://folder/key", - "somescheme://dataset/key", - "wasb://some_dataset_bucket_/key", + "somescheme://asset/key", + "wasb://some_asset_bucket_/key", }, ), ], ) @provide_session - def test_filter_datasets_by_uri_pattern_works(self, url, expected_datasets, session): + def test_filter_assets_by_uri_pattern_works(self, url, expected_assets, session): asset1 = AssetModel("s3://folder/key") asset2 = AssetModel("gcp://bucket/key") - asset3 = AssetModel("somescheme://dataset/key") - asset4 = AssetModel("wasb://some_dataset_bucket_/key") + asset3 = AssetModel("somescheme://asset/key") + asset4 = AssetModel("wasb://some_asset_bucket_/key") session.add_all([asset1, asset2, asset3, asset4]) session.commit() response = self.client.get(url, environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 - dataset_urls = {dataset["uri"] for dataset in response.json["datasets"]} - assert expected_datasets == dataset_urls + asset_urls = {asset["uri"] for asset in response.json["assets"]} + assert expected_assets == asset_urls @pytest.mark.parametrize("dag_ids, expected_num", [("dag1,dag2", 2), ("dag3", 1), ("dag2,dag3", 2)]) @provide_session - def test_filter_datasets_by_dag_ids_works(self, dag_ids, expected_num, session): + def test_filter_assets_by_dag_ids_works(self, dag_ids, expected_num, session): session.query(DagModel).delete() session.commit() dag1 = DagModel(dag_id="dag1") dag2 = DagModel(dag_id="dag2") dag3 = DagModel(dag_id="dag3") - dataset1 = AssetModel("s3://folder/key") - dataset2 = AssetModel("gcp://bucket/key") - dataset3 = AssetModel("somescheme://dataset/key") - dag_ref1 = DagScheduleAssetReference(dag_id="dag1", dataset=dataset1) - dag_ref2 = DagScheduleAssetReference(dag_id="dag2", dataset=dataset2) - task_ref1 = TaskOutletAssetReference(dag_id="dag3", task_id="task1", dataset=dataset3) - session.add_all([dataset1, dataset2, dataset3, dag1, dag2, dag3, dag_ref1, dag_ref2, task_ref1]) + asset1 = AssetModel("s3://folder/key") + asset2 = AssetModel("gcp://bucket/key") + asset3 = AssetModel("somescheme://asset/key") + dag_ref1 = DagScheduleAssetReference(dag_id="dag1", dataset=asset1) + dag_ref2 = DagScheduleAssetReference(dag_id="dag2", dataset=asset2) + task_ref1 = TaskOutletAssetReference(dag_id="dag3", task_id="task1", dataset=asset3) + session.add_all([asset1, asset2, asset3, dag1, dag2, dag3, dag_ref1, dag_ref2, task_ref1]) session.commit() response = self.client.get( f"/api/v1/assets?dag_ids={dag_ids}", environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 200 response_data = response.json - assert len(response_data["datasets"]) == expected_num + assert len(response_data["assets"]) == expected_num @pytest.mark.parametrize( "dag_ids, uri_pattern,expected_num", [("dag1,dag2", "folder", 1), ("dag3", "nothing", 0), ("dag2,dag3", "key", 2)], ) - def test_filter_datasets_by_dag_ids_and_uri_pattern_works( + def test_filter_assets_by_dag_ids_and_uri_pattern_works( self, dag_ids, uri_pattern, expected_num, session ): session.query(DagModel).delete() @@ -286,7 +286,7 @@ def test_filter_datasets_by_dag_ids_and_uri_pattern_works( dag3 = DagModel(dag_id="dag3") asset1 = AssetModel("s3://folder/key") asset2 = AssetModel("gcp://bucket/key") - asset3 = AssetModel("somescheme://dataset/key") + asset3 = AssetModel("somescheme://asset/key") dag_ref1 = DagScheduleAssetReference(dag_id="dag1", dataset=asset1) dag_ref2 = DagScheduleAssetReference(dag_id="dag2", dataset=asset2) task_ref1 = TaskOutletAssetReference(dag_id="dag3", task_id="task1", dataset=asset3) @@ -298,12 +298,12 @@ def test_filter_datasets_by_dag_ids_and_uri_pattern_works( ) assert response.status_code == 200 response_data = response.json - assert len(response_data["datasets"]) == expected_num + assert len(response_data["assets"]) == expected_num -class TestGetDatasetsEndpointPagination(TestDatasetEndpoint): +class TestGetAssetsEndpointPagination(TestAssetEndpoint): @pytest.mark.parametrize( - "url, expected_dataset_uris", + "url, expected_asset_uris", [ # Limit test data ("/api/v1/assets?limit=1", ["s3://bucket/key/1"]), @@ -316,7 +316,7 @@ class TestGetDatasetsEndpointPagination(TestDatasetEndpoint): ], ) @provide_session - def test_limit_and_offset(self, url, expected_dataset_uris, session): + def test_limit_and_offset(self, url, expected_asset_uris, session): assets = [ AssetModel( uri=f"s3://bucket/key/{i}", @@ -332,8 +332,8 @@ def test_limit_and_offset(self, url, expected_dataset_uris, session): response = self.client.get(url, environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 - dataset_uris = [dataset["uri"] for dataset in response.json["datasets"]] - assert dataset_uris == expected_dataset_uris + asset_uris = [asset["uri"] for asset in response.json["assets"]] + assert asset_uris == expected_asset_uris def test_should_respect_page_size_limit_default(self, session): assets = [ @@ -351,7 +351,7 @@ def test_should_respect_page_size_limit_default(self, session): response = self.client.get("/api/v1/assets", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 - assert len(response.json["datasets"]) == 100 + assert len(response.json["assets"]) == 100 @conf_vars({("api", "maximum_page_limit"): "150"}) def test_should_return_conf_max_if_req_max_above_conf(self, session): @@ -370,12 +370,12 @@ def test_should_return_conf_max_if_req_max_above_conf(self, session): response = self.client.get("/api/v1/assets?limit=180", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 200 - assert len(response.json["datasets"]) == 150 + assert len(response.json["assets"]) == 150 -class TestGetDatasetEvents(TestDatasetEndpoint): +class TestGetAssetEvents(TestAssetEndpoint): def test_should_respond_200(self, session): - d = self._create_dataset(session) + d = self._create_asset(session) common = { "dataset_id": 1, "extra": {"foo": "bar"}, @@ -478,7 +478,7 @@ def test_filtering(self, attr, value, session): } def test_order_by_raises_400_for_invalid_attr(self, session): - self._create_dataset(session) + self._create_asset(session) events = [ AssetEvent( dataset_id=1, @@ -508,7 +508,7 @@ def test_should_raises_401_unauthenticated(self, session): assert_401(response) def test_includes_created_dagrun(self, session): - self._create_dataset(session) + self._create_asset(session) event = AssetEvent( id=1, dataset_id=1, @@ -567,7 +567,7 @@ def test_includes_created_dagrun(self, session): } -class TestPostDatasetEvents(TestDatasetEndpoint): +class TestPostAssetEvents(TestAssetEndpoint): @pytest.fixture def time_freezer(self) -> Generator: freezer = time_machine.travel(self.default_time, tick=False) @@ -579,7 +579,7 @@ def time_freezer(self) -> Generator: @pytest.mark.usefixtures("time_freezer") def test_should_respond_200(self, session): - self._create_dataset(session) + self._create_asset(session) event_payload = {"asset_uri": "s3://bucket/key", "extra": {"foo": "bar"}} response = self.client.post( "/api/v1/assets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} @@ -609,7 +609,7 @@ def test_should_respond_200(self, session): @pytest.mark.enable_redact def test_should_mask_sensitive_extra_logs(self, session): - self._create_dataset(session) + self._create_asset(session) event_payload = {"asset_uri": "s3://bucket/key", "extra": {"password": "bar"}} response = self.client.post( "/api/v1/assets/events", json=event_payload, environ_overrides={"REMOTE_USER": "test"} @@ -626,7 +626,7 @@ def test_should_mask_sensitive_extra_logs(self, session): ) def test_order_by_raises_400_for_invalid_attr(self, session): - self._create_dataset(session) + self._create_asset(session) event_invalid_payload = {"asset_uri": "TEST_ASSET_URI", "extra": {"foo": "bar"}, "fake": {}} response = self.client.post( "/api/v1/assets/events", json=event_invalid_payload, environ_overrides={"REMOTE_USER": "test"} @@ -634,12 +634,12 @@ def test_order_by_raises_400_for_invalid_attr(self, session): assert response.status_code == 400 def test_should_raises_401_unauthenticated(self, session): - self._create_dataset(session) + self._create_asset(session) response = self.client.post("/api/v1/assets/events", json={"asset_uri": "TEST_ASSET_URI"}) assert_401(response) -class TestGetDatasetEventsEndpointPagination(TestDatasetEndpoint): +class TestGetAssetEventsEndpointPagination(TestAssetEndpoint): @pytest.mark.parametrize( "url, expected_event_runids", [ @@ -667,7 +667,7 @@ class TestGetDatasetEventsEndpointPagination(TestDatasetEndpoint): ) @provide_session def test_limit_and_offset(self, url, expected_event_runids, session): - self._create_dataset(session) + self._create_asset(session) events = [ AssetEvent( dataset_id=1, @@ -689,7 +689,7 @@ def test_limit_and_offset(self, url, expected_event_runids, session): assert event_runids == expected_event_runids def test_should_respect_page_size_limit_default(self, session): - self._create_dataset(session) + self._create_asset(session) events = [ AssetEvent( dataset_id=1, @@ -711,7 +711,7 @@ def test_should_respect_page_size_limit_default(self, session): @conf_vars({("api", "maximum_page_limit"): "150"}) def test_should_return_conf_max_if_req_max_above_conf(self, session): - self._create_dataset(session) + self._create_asset(session) events = [ AssetEvent( dataset_id=1, @@ -734,7 +734,7 @@ def test_should_return_conf_max_if_req_max_above_conf(self, session): assert len(response.json["asset_events"]) == 150 -class TestQueuedEventEndpoint(TestDatasetEndpoint): +class TestQueuedEventEndpoint(TestAssetEndpoint): @pytest.fixture def time_freezer(self) -> Generator: freezer = time_machine.travel(self.default_time, tick=False) @@ -744,26 +744,30 @@ def time_freezer(self) -> Generator: freezer.stop() - def _create_dataset_dag_run_queues(self, dag_id, dataset_id, session): - adrq = AssetDagRunQueue(target_dag_id=dag_id, dataset_id=dataset_id) + def _create_asset_dag_run_queues(self, dag_id, asset_id, session): + adrq = AssetDagRunQueue(target_dag_id=dag_id, dataset_id=asset_id) session.add(adrq) session.commit() return adrq +<<<<<<< HEAD class TestGetDagDatasetQueuedEvent(TestQueuedEventEndpoint): <<<<<<< HEAD ======= +======= +class TestGetDagAssetQueuedEvent(TestQueuedEventEndpoint): +>>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) @pytest.mark.usefixtures("time_freezer") def test_should_respond_200(self, session, create_dummy_dag): dag, _ = create_dummy_dag() dag_id = dag.dag_id - dataset_id = self._create_dataset(session).id - self._create_dataset_dag_run_queues(dag_id, dataset_id, session) - dataset_uri = "s3://bucket/key" + asset_id = self._create_asset(session).id + self._create_asset_dag_run_queues(dag_id, asset_id, session) + asset_uri = "s3://bucket/key" response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_queued_event"}, ) @@ -776,10 +780,10 @@ def test_should_respond_200(self, session, create_dummy_dag): def test_should_respond_404(self): dag_id = "not_exists" - dataset_uri = "not_exists" + asset_uri = "not_exists" response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_queued_event"}, ) @@ -794,50 +798,54 @@ def test_should_respond_404(self): >>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self, session): dag_id = "dummy" - dataset_uri = "dummy" + asset_uri = "dummy" - response = self.client.get(f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}") + response = self.client.get(f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}") assert_401(response) def test_should_raise_403_forbidden(self, session): dag_id = "dummy" - dataset_uri = "dummy" + asset_uri = "dummy" response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) assert response.status_code == 403 -class TestDeleteDagDatasetQueuedEvent(TestDatasetEndpoint): +class TestDeleteDagAssetQueuedEvent(TestAssetEndpoint): def test_should_raises_401_unauthenticated(self, session): dag_id = "dummy" - dataset_uri = "dummy" - response = self.client.delete(f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}") + asset_uri = "dummy" + response = self.client.delete(f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}") assert_401(response) def test_should_raise_403_forbidden(self, session): dag_id = "dummy" - dataset_uri = "dummy" + asset_uri = "dummy" response = self.client.delete( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{dataset_uri}", + f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) assert response.status_code == 403 +<<<<<<< HEAD class TestGetDagDatasetQueuedEvents(TestQueuedEventEndpoint): <<<<<<< HEAD ======= +======= +class TestGetDagAssetQueuedEvents(TestQueuedEventEndpoint): +>>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) @pytest.mark.usefixtures("time_freezer") def test_should_respond_200(self, session, create_dummy_dag): dag, _ = create_dummy_dag() dag_id = dag.dag_id - dataset_id = self._create_dataset(session).id - self._create_dataset_dag_run_queues(dag_id, dataset_id, session) + asset_id = self._create_asset(session).id + self._create_asset_dag_run_queues(dag_id, asset_id, session) response = self.client.get( f"/api/v1/dags/{dag_id}/assets/queuedEvent", @@ -891,9 +899,13 @@ def test_should_raise_403_forbidden(self): assert response.status_code == 403 +<<<<<<< HEAD class TestDeleteDagDatasetQueuedEvents(TestDatasetEndpoint): <<<<<<< HEAD ======= +======= +class TestDeleteDagAssetQueuedEvents(TestAssetEndpoint): +>>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) def test_should_respond_404(self): dag_id = "not_exists" @@ -929,19 +941,23 @@ def test_should_raise_403_forbidden(self): assert response.status_code == 403 +<<<<<<< HEAD class TestGetDatasetQueuedEvents(TestQueuedEventEndpoint): <<<<<<< HEAD ======= +======= +class TestGetAssetQueuedEvents(TestQueuedEventEndpoint): +>>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) @pytest.mark.usefixtures("time_freezer") def test_should_respond_200(self, session, create_dummy_dag): dag, _ = create_dummy_dag() dag_id = dag.dag_id - dataset_id = self._create_dataset(session).id - self._create_dataset_dag_run_queues(dag_id, dataset_id, session) - dataset_uri = "s3://bucket/key" + asset_id = self._create_asset(session).id + self._create_asset_dag_run_queues(dag_id, asset_id, session) + asset_uri = "s3://bucket/key" response = self.client.get( - f"/api/v1/assets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_queued_event"}, ) @@ -958,10 +974,10 @@ def test_should_respond_200(self, session, create_dummy_dag): } def test_should_respond_404(self): - dataset_uri = "not_exists" + asset_uri = "not_exists" response = self.client.get( - f"/api/v1/assets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_queued_event"}, ) @@ -975,17 +991,17 @@ def test_should_respond_404(self): >>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): - dataset_uri = "not_exists" + asset_uri = "not_exists" - response = self.client.get(f"/api/v1/assets/queuedEvent/{dataset_uri}") + response = self.client.get(f"/api/v1/assets/queuedEvent/{asset_uri}") assert_401(response) def test_should_raise_403_forbidden(self): - dataset_uri = "not_exists" + asset_uri = "not_exists" response = self.client.get( - f"/api/v1/assets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) @@ -998,12 +1014,12 @@ class TestDeleteAssetQueuedEvents(TestQueuedEventEndpoint): def test_delete_should_respond_204(self, session, create_dummy_dag): dag, _ = create_dummy_dag() dag_id = dag.dag_id - dataset_id = self._create_dataset(session).id - self._create_dataset_dag_run_queues(dag_id, dataset_id, session) - dataset_uri = "s3://bucket/key" + asset_id = self._create_asset(session).id + self._create_asset_dag_run_queues(dag_id, asset_id, session) + asset_uri = "s3://bucket/key" response = self.client.delete( - f"/api/v1/assets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_queued_event"}, ) @@ -1013,10 +1029,10 @@ def test_delete_should_respond_204(self, session, create_dummy_dag): _check_last_log(session, dag_id=None, event="api.delete_asset_queued_events", execution_date=None) def test_should_respond_404(self): - dataset_uri = "not_exists" + asset_uri = "not_exists" response = self.client.delete( - f"/api/v1/assets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_queued_event"}, ) @@ -1030,17 +1046,17 @@ def test_should_respond_404(self): >>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): - dataset_uri = "not_exists" + asset_uri = "not_exists" - response = self.client.delete(f"/api/v1/assets/queuedEvent/{dataset_uri}") + response = self.client.delete(f"/api/v1/assets/queuedEvent/{asset_uri}") assert_401(response) def test_should_raise_403_forbidden(self): - dataset_uri = "not_exists" + asset_uri = "not_exists" response = self.client.delete( - f"/api/v1/assets/queuedEvent/{dataset_uri}", + f"/api/v1/assets/queuedEvent/{asset_uri}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) diff --git a/tests/api_connexion/schemas/test_dataset_schema.py b/tests/api_connexion/schemas/test_dataset_schema.py index 07d041b1a1d60..4a6fdf6e6d513 100644 --- a/tests/api_connexion/schemas/test_dataset_schema.py +++ b/tests/api_connexion/schemas/test_dataset_schema.py @@ -106,13 +106,13 @@ def test_serialize(self, session): session.add_all(assets) session.add_all(asset_aliases) session.flush() - serialized_data = asset_collection_schema.dump(AssetCollection(datasets=assets, total_entries=2)) - serialized_data["datasets"][0]["id"] = 1 - serialized_data["datasets"][1]["id"] = 2 - serialized_data["datasets"][0]["aliases"][0]["id"] = 1 - serialized_data["datasets"][0]["aliases"][1]["id"] = 2 + serialized_data = asset_collection_schema.dump(AssetCollection(assets=assets, total_entries=2)) + serialized_data["assets"][0]["id"] = 1 + serialized_data["assets"][1]["id"] = 2 + serialized_data["assets"][0]["aliases"][0]["id"] = 1 + serialized_data["assets"][0]["aliases"][1]["id"] = 2 assert serialized_data == { - "datasets": [ + "assets": [ { "id": 1, "uri": "s3://bucket/key/1", From 7c2ba6a99dca509980208b8a70417782834ab270 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 22:06:46 +0900 Subject: [PATCH 37/43] fix(api_connexion/openapi/v1): fix queued_events property name error --- airflow/api_connexion/openapi/v1.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index b0d022e50b382..eecebebcb7e9a 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -4663,7 +4663,7 @@ components: properties: uri: type: string - description: The datata uri. + description: The asset uri. dag_id: type: string description: The DAG ID. @@ -4681,7 +4681,7 @@ components: allOf: - type: object properties: - datasets: + queued_events: type: array items: $ref: "#/components/schemas/QueuedEvent" From 729314e42e942ddede296447e95a7358a94b2d8c Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 1 Oct 2024 22:40:08 +0900 Subject: [PATCH 38/43] feat(api_fastapi): rename next_run_datasets as next_run_assets --- airflow/api_fastapi/openapi/v1-generated.yaml | 2 +- airflow/api_fastapi/views/ui/assets.py | 2 +- airflow/ui/openapi-gen/requests/services.gen.ts | 2 +- airflow/ui/openapi-gen/requests/types.gen.ts | 2 +- tests/api_fastapi/views/ui/test_assets.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index a54e0e4ca57dd..37fa4c3293fab 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -7,7 +7,7 @@ info: Users should not rely on those but use the public ones instead. version: 0.1.0 paths: - /ui/next_run_datasets/{dag_id}: + /ui/next_run_assets/{dag_id}: get: tags: - Asset diff --git a/airflow/api_fastapi/views/ui/assets.py b/airflow/api_fastapi/views/ui/assets.py index 01cc9fd1cfbff..4a4ad1d0df9b4 100644 --- a/airflow/api_fastapi/views/ui/assets.py +++ b/airflow/api_fastapi/views/ui/assets.py @@ -30,7 +30,7 @@ assets_router = AirflowRouter(tags=["Asset"]) -@assets_router.get("/next_run_datasets/{dag_id}", include_in_schema=False) +@assets_router.get("/next_run_assets/{dag_id}", include_in_schema=False) async def next_run_assets( dag_id: str, request: Request, diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 24c960d2b7d5f..3859523fa8001 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -28,7 +28,7 @@ export class AssetService { ): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/ui/next_run_datasets/{dag_id}", + url: "/ui/next_run_assets/{dag_id}", path: { dag_id: data.dagId, }, diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index b38d5c00a69f3..da109af81e574 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -141,7 +141,7 @@ export type DeleteConnectionData = { export type DeleteConnectionResponse = void; export type $OpenApiTs = { - "/ui/next_run_datasets/{dag_id}": { + "/ui/next_run_assets/{dag_id}": { get: { req: NextRunAssetsData; res: { diff --git a/tests/api_fastapi/views/ui/test_assets.py b/tests/api_fastapi/views/ui/test_assets.py index 7aff14249b4db..99cd9bc1e9960 100644 --- a/tests/api_fastapi/views/ui/test_assets.py +++ b/tests/api_fastapi/views/ui/test_assets.py @@ -34,14 +34,14 @@ def cleanup(): initial_db_init() -def test_next_run_datasets(test_client, dag_maker): +def test_next_run_assets(test_client, dag_maker): with dag_maker(dag_id="upstream", schedule=[Asset(uri="s3://bucket/key/1")], serialized=True): EmptyOperator(task_id="task1") dag_maker.create_dagrun() dag_maker.dagbag.sync_to_db() - response = test_client.get("/ui/next_run_datasets/upstream") + response = test_client.get("/ui/next_run_assets/upstream") assert response.status_code == 200 assert response.json() == { From d0c08df6ca7fcf904e9fe55d07afeb8b49bac8b3 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 2 Oct 2024 08:38:32 +0900 Subject: [PATCH 39/43] test: resolve test conflict --- .../endpoints/test_asset_endpoint.py | 198 ------------------ 1 file changed, 198 deletions(-) diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index bae498f5b7ce2..40ad59067f594 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -751,51 +751,7 @@ def _create_asset_dag_run_queues(self, dag_id, asset_id, session): return adrq -<<<<<<< HEAD -class TestGetDagDatasetQueuedEvent(TestQueuedEventEndpoint): -<<<<<<< HEAD -======= -======= class TestGetDagAssetQueuedEvent(TestQueuedEventEndpoint): ->>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) - @pytest.mark.usefixtures("time_freezer") - def test_should_respond_200(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - asset_uri = "s3://bucket/key" - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 200 - assert response.json == { - "created_at": self.default_time, - "uri": "s3://bucket/key", - "dag_id": "dag", - } - - def test_should_respond_404(self): - dag_id = "not_exists" - asset_uri = "not_exists" - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with dag_id: `not_exists` and asset uri: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - ->>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self, session): dag_id = "dummy" asset_uri = "dummy" @@ -833,54 +789,7 @@ def test_should_raise_403_forbidden(self, session): assert response.status_code == 403 -<<<<<<< HEAD -class TestGetDagDatasetQueuedEvents(TestQueuedEventEndpoint): -<<<<<<< HEAD -======= -======= class TestGetDagAssetQueuedEvents(TestQueuedEventEndpoint): ->>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) - @pytest.mark.usefixtures("time_freezer") - def test_should_respond_200(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 200 - assert response.json == { - "queued_events": [ - { - "created_at": self.default_time, - "uri": "s3://bucket/key", - "dag_id": "dag", - } - ], - "total_entries": 1, - } - - def test_should_respond_404(self): - dag_id = "not_exists" - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with dag_id: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - ->>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): dag_id = "dummy" @@ -899,30 +808,7 @@ def test_should_raise_403_forbidden(self): assert response.status_code == 403 -<<<<<<< HEAD -class TestDeleteDagDatasetQueuedEvents(TestDatasetEndpoint): -<<<<<<< HEAD -======= -======= class TestDeleteDagAssetQueuedEvents(TestAssetEndpoint): ->>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) - def test_should_respond_404(self): - dag_id = "not_exists" - - response = self.client.delete( - f"/api/v1/dags/{dag_id}/assets/queuedEvent", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with dag_id: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - ->>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): dag_id = "dummy" @@ -941,55 +827,7 @@ def test_should_raise_403_forbidden(self): assert response.status_code == 403 -<<<<<<< HEAD -class TestGetDatasetQueuedEvents(TestQueuedEventEndpoint): -<<<<<<< HEAD -======= -======= class TestGetAssetQueuedEvents(TestQueuedEventEndpoint): ->>>>>>> 9b7df72feb (test(api_connexion): rename dataset as asset) - @pytest.mark.usefixtures("time_freezer") - def test_should_respond_200(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - asset_uri = "s3://bucket/key" - - response = self.client.get( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 200 - assert response.json == { - "queued_events": [ - { - "created_at": self.default_time, - "uri": "s3://bucket/key", - "dag_id": "dag", - } - ], - "total_entries": 1, - } - - def test_should_respond_404(self): - asset_uri = "not_exists" - - response = self.client.get( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with asset uri: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - ->>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): asset_uri = "not_exists" @@ -1009,42 +847,6 @@ def test_should_raise_403_forbidden(self): class TestDeleteAssetQueuedEvents(TestQueuedEventEndpoint): -<<<<<<< HEAD -======= - def test_delete_should_respond_204(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - asset_uri = "s3://bucket/key" - - response = self.client.delete( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 204 - conn = session.query(AssetDagRunQueue).all() - assert len(conn) == 0 - _check_last_log(session, dag_id=None, event="api.delete_asset_queued_events", execution_date=None) - - def test_should_respond_404(self): - asset_uri = "not_exists" - - response = self.client.delete( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with asset uri: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - ->>>>>>> f2fa713cd9 (feat(api_connexion): rename endpoint datasets as assets) def test_should_raises_401_unauthenticated(self): asset_uri = "not_exists" From bd3ddfa6ca1f46641f7706c3a9bbd87b0ddd7af5 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 2 Oct 2024 21:54:36 +0900 Subject: [PATCH 40/43] docs(newsfragments): add newsfragments for dataset to asset endpoint rename --- newsfragments/42579.significant.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 newsfragments/42579.significant.rst diff --git a/newsfragments/42579.significant.rst b/newsfragments/42579.significant.rst new file mode 100644 index 0000000000000..8a43583e25766 --- /dev/null +++ b/newsfragments/42579.significant.rst @@ -0,0 +1,20 @@ +**Breaking Change** + +* Rename dataset endpoints as asset endpoints + + * Rename ``/datasets`` as ``/assets`` + * Rename ``/datasets/{uri}`` as ``/assets/{uri}`` + * Rename ``/datasets/events`` as ``/assets/events`` + * Rename ``/datasets/queuedEvent/{uri}`` as ``/ui/next_run_assets/upstream`` + * Rename ``/dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents`` as ``/ui/next_run_assets/upstream`` + * Rename ``/dags/{dag_id}/datasets/queuedEvent/{uri}`` as ``/ui/next_run_assets/upstream`` + * Rename ``/dags/{dag_id}/datasets/queuedEvent`` as ``/ui/next_run_assets/upstream`` + * Rename ``/ui/next_run_datasets/upstream`` as ``/ui/next_run_assets/upstream`` + + +* Rename dataset schema as asset endpoints + + * Rename ``AssetCollection.datasets`` as ``AssetCollection.assets`` + * Rename ``AssetEventCollection.dataset_events`` as ``AssetEventCollection.asset_events`` + * Rename ``AssetEventCollectionSchema.dataset_events`` as ``AssetEventCollectionSchema.asset_events`` + * Rename ``CreateAssetEventSchema.dataset_uri`` as ``CreateAssetEventSchema.asset_uri`` From 6eb5d29ee2640f472bbc5d7b7bc105097aea5750 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 2 Oct 2024 22:29:37 +0900 Subject: [PATCH 41/43] feat(js/api): rename datasetEvents as assetEvents --- airflow/api_connexion/openapi/v1.yaml | 2 +- airflow/www/static/js/api/useAssetEvents.ts | 2 +- airflow/www/static/js/api/useUpstreamAssetEvents.ts | 4 ++-- .../js/dag/details/dagRun/DatasetTriggerEvents.tsx | 4 ++-- airflow/www/static/js/dag/details/graph/index.tsx | 12 ++++++------ airflow/www/static/js/dag/details/graph/utils.ts | 6 +++--- .../dag/details/taskInstance/DatasetUpdateEvents.tsx | 4 ++-- airflow/www/static/js/datasetUtils.js | 8 ++++---- airflow/www/static/js/datasets/AssetEvents.tsx | 4 ++-- airflow/www/static/js/datasets/Main.tsx | 4 ++-- airflow/www/static/js/types/api-generated.ts | 2 +- 11 files changed, 26 insertions(+), 26 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index eecebebcb7e9a..828a3af25e879 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -4746,7 +4746,7 @@ components: allOf: - type: object properties: - dataset_events: + asset_events: type: array items: $ref: "#/components/schemas/AssetEvent" diff --git a/airflow/www/static/js/api/useAssetEvents.ts b/airflow/www/static/js/api/useAssetEvents.ts index 25fc815eb1662..8901715d1526c 100644 --- a/airflow/www/static/js/api/useAssetEvents.ts +++ b/airflow/www/static/js/api/useAssetEvents.ts @@ -80,7 +80,7 @@ const useAssetEvents = ({ ); return { ...query, - data: query.data ?? { datasetEvents: [], totalEntries: 0 }, + data: query.data ?? { assetEvents: [], totalEntries: 0 }, }; }; diff --git a/airflow/www/static/js/api/useUpstreamAssetEvents.ts b/airflow/www/static/js/api/useUpstreamAssetEvents.ts index d35349ddb4ae0..437205501d6c5 100644 --- a/airflow/www/static/js/api/useUpstreamAssetEvents.ts +++ b/airflow/www/static/js/api/useUpstreamAssetEvents.ts @@ -32,7 +32,7 @@ interface Props extends GetUpstreamAssetEventsVariables { const useUpstreamAssetEvents = ({ dagId, dagRunId, options }: Props) => { const upstreamEventsUrl = ( - getMetaValue("upstream_dataset_events_api") || + getMetaValue("upstream_asset_events_api") || `api/v1/dags/${dagId}/dagRuns/_DAG_RUN_ID_/upstreamAssetEvents` ).replace("_DAG_RUN_ID_", encodeURIComponent(dagRunId)); @@ -44,7 +44,7 @@ const useUpstreamAssetEvents = ({ dagId, dagRunId, options }: Props) => { return { ...query, - data: query.data ?? { datasetEvents: [], totalEntries: 0 }, + data: query.data ?? { assetEvents: [], totalEntries: 0 }, }; }; diff --git a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx index 086614956be3b..6946150b1d7db 100644 --- a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx +++ b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx @@ -40,7 +40,7 @@ const cardDef: CardDef = { const DatasetTriggerEvents = ({ runId }: Props) => { const { - data: { datasetEvents = [] }, + data: { assetEvents = [] }, isLoading, } = useUpstreamAssetEvents({ dagRunId: runId, dagId }); @@ -66,7 +66,7 @@ const DatasetTriggerEvents = ({ runId }: Props) => { [] ); - const data = useMemo(() => datasetEvents, [datasetEvents]); + const data = useMemo(() => assetEvents, [assetEvents]); return ( diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index f5d3fe9bfc066..b9511b6664181 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -164,7 +164,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { ); const { - data: { datasetEvents: upstreamAssetEvents = [] }, + data: { assetEvents: upstreamAssetEvents = [] }, } = useUpstreamAssetEvents({ dagId, dagRunId: selected.runId || "", @@ -175,7 +175,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { }); const { - data: { datasetEvents: downstreamDatasetEvents = [] }, + data: { assetEvents: downstreamAssetEvents = [] }, } = useAssetEvents({ sourceDagId: dagId, sourceRunId: selected.runId || undefined, @@ -211,7 +211,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { }); // Check if there is a dataset event even though we did not find a dataset - downstreamDatasetEvents.forEach((de) => { + downstreamAssetEvents.forEach((de) => { const hasNode = datasetNodes.find((node) => node.id === de.datasetUri); if (!hasNode && de.sourceTaskId && de.datasetUri) { datasetEdges.push({ @@ -264,8 +264,8 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { groups, hoveredTaskState, isZoomedOut, - datasetEvents: selected.runId - ? [...upstreamAssetEvents, ...downstreamDatasetEvents] + assetEvents: selected.runId + ? [...upstreamAssetEvents, ...downstreamAssetEvents] : [], }), [ @@ -278,7 +278,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { hoveredTaskState, isZoomedOut, upstreamAssetEvents, - downstreamDatasetEvents, + downstreamAssetEvents, ] ); diff --git a/airflow/www/static/js/dag/details/graph/utils.ts b/airflow/www/static/js/dag/details/graph/utils.ts index dba77cc511c04..4888cfd2a3c7f 100644 --- a/airflow/www/static/js/dag/details/graph/utils.ts +++ b/airflow/www/static/js/dag/details/graph/utils.ts @@ -38,7 +38,7 @@ interface FlattenNodesProps { onToggleGroups: (groupIds: string[]) => void; hoveredTaskState?: string | null; isZoomedOut: boolean; - datasetEvents?: AssetEvent[]; + assetEvents?: AssetEvent[]; } // Generate a flattened list of nodes for react-flow to render @@ -52,7 +52,7 @@ export const flattenNodes = ({ parent, hoveredTaskState, isZoomedOut, - datasetEvents, + assetEvents, }: FlattenNodesProps) => { let nodes: ReactFlowNode[] = []; let edges: ElkExtendedEdge[] = []; @@ -93,7 +93,7 @@ export const flattenNodes = ({ }, datasetEvent: node.value.class === "asset" - ? datasetEvents?.find((de) => de.datasetUri === node.value.label) + ? assetEvents?.find((de) => de.datasetUri === node.value.label) : undefined, ...node.value, }, diff --git a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx index a0f3ff704b2fd..c3a77fa32e794 100644 --- a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx @@ -39,7 +39,7 @@ const dagId = getMetaValue("dag_id") || undefined; const DatasetUpdateEvents = ({ runId, taskId }: Props) => { const { - data: { datasetEvents = [] }, + data: { assetEvents = [] }, isLoading, } = useAssetEvents({ sourceDagId: dagId, @@ -69,7 +69,7 @@ const DatasetUpdateEvents = ({ runId, taskId }: Props) => { [] ); - const data = useMemo(() => datasetEvents, [datasetEvents]); + const data = useMemo(() => assetEvents, [assetEvents]); return ( diff --git a/airflow/www/static/js/datasetUtils.js b/airflow/www/static/js/datasetUtils.js index dabc0a91cb42c..5ba5e3fe584fa 100644 --- a/airflow/www/static/js/datasetUtils.js +++ b/airflow/www/static/js/datasetUtils.js @@ -22,7 +22,7 @@ import { getMetaValue } from "./utils"; export function openDatasetModal(dagId, summary, nextDatasets, error) { - const datasetEvents = nextDatasets.events || []; + const assetEvents = nextDatasets.events || []; const expression = nextDatasets.dataset_expression; const datasetsUrl = getMetaValue("datasets_url"); $("#dataset_expression").empty(); @@ -32,7 +32,7 @@ export function openDatasetModal(dagId, summary, nextDatasets, error) { $("#dataset_expression").text(JSON.stringify(expression, null, 2)); $("#datasetNextRunModal").modal({}); if (summary) $("#next_run_summary").text(summary); - datasetEvents.forEach((d) => { + assetEvents.forEach((d) => { const row = document.createElement("tr"); const uriCell = document.createElement("td"); @@ -63,11 +63,11 @@ export function getDatasetTooltipInfo(dagId, run, setNextDatasets) { } $.get(nextRunUrl) .done((nextDatasets) => { - const datasetEvents = nextDatasets.events; + const assetEvents = nextDatasets.events; let count = 0; let title = "Pending datasets:
"; setNextDatasets(nextDatasets); - datasetEvents.forEach((d) => { + assetEvents.forEach((d) => { if (!d.created_at) { if (count < 4) title += `${d.uri}
`; count += 1; diff --git a/airflow/www/static/js/datasets/AssetEvents.tsx b/airflow/www/static/js/datasets/AssetEvents.tsx index 06998c633f2d1..d4ca050e3e68a 100644 --- a/airflow/www/static/js/datasets/AssetEvents.tsx +++ b/airflow/www/static/js/datasets/AssetEvents.tsx @@ -48,7 +48,7 @@ const Events = ({ assetId, showLabel }: Props) => { const orderBy = sort ? `${sort.desc ? "-" : ""}${snakeCase(sort.id)}` : ""; const { - data: { datasetEvents = [], totalEntries = 0 }, + data: { assetEvents = [], totalEntries = 0 }, isLoading: isEventsLoading, } = useAssetEvents({ assetId, @@ -83,7 +83,7 @@ const Events = ({ assetId, showLabel }: Props) => { [] ); - const data = useMemo(() => datasetEvents, [datasetEvents]); + const data = useMemo(() => assetEvents, [assetEvents]); return ( diff --git a/airflow/www/static/js/datasets/Main.tsx b/airflow/www/static/js/datasets/Main.tsx index 9f662d66b98e9..7e56cc9b3af94 100644 --- a/airflow/www/static/js/datasets/Main.tsx +++ b/airflow/www/static/js/datasets/Main.tsx @@ -47,7 +47,7 @@ import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; import Tooltip from "src/components/Tooltip"; import { useContainerRef } from "src/context/containerRef"; -import DatasetEvents from "./AssetEvents"; +import AssetEvents from "./AssetEvents"; import AssetsList from "./AssetsList"; import AssetDetails from "./AssetDetails"; import type { OnSelectProps } from "./types"; @@ -240,7 +240,7 @@ const Datasets = () => { {!selectedUri && ( - + )} {!!selectedUri && ( diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 6e9d219918b57..b68cd377cbe45 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1988,7 +1988,7 @@ export interface components { * *New in version 2.4.0* */ AssetEventCollection: { - dataset_events?: components["schemas"]["AssetEvent"][]; + asset_events?: components["schemas"]["AssetEvent"][]; } & components["schemas"]["CollectionInfo"]; /** @description The option of configuration. */ ConfigOption: { From 2eaff9068a8e0e0ad913f0ca119452288c6fad3b Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 3 Oct 2024 06:50:54 +0900 Subject: [PATCH 42/43] feat(js/api): rename variable datasetEvent as assetEvent --- airflow/www/static/js/api/useAssetEvents.ts | 2 +- .../static/js/components/DatasetEventCard.tsx | 31 ++++++++++--------- .../js/components/SourceTaskInstance.tsx | 10 ++---- .../details/dagRun/DatasetTriggerEvents.tsx | 2 +- .../js/dag/details/graph/DatasetNode.tsx | 24 +++++++------- .../www/static/js/dag/details/graph/Node.tsx | 2 +- .../www/static/js/dag/details/graph/utils.ts | 2 +- .../taskInstance/DatasetUpdateEvents.tsx | 2 +- .../www/static/js/datasets/AssetEvents.tsx | 2 +- 9 files changed, 37 insertions(+), 40 deletions(-) diff --git a/airflow/www/static/js/api/useAssetEvents.ts b/airflow/www/static/js/api/useAssetEvents.ts index 8901715d1526c..068bb471ef64e 100644 --- a/airflow/www/static/js/api/useAssetEvents.ts +++ b/airflow/www/static/js/api/useAssetEvents.ts @@ -55,7 +55,7 @@ const useAssetEvents = ({ orderBy, ], () => { - const datasetsUrl = getMetaValue("dataset_events_api"); + const datasetsUrl = getMetaValue("asset_events_api"); const params = new URLSearchParamsWrapper(); diff --git a/airflow/www/static/js/components/DatasetEventCard.tsx b/airflow/www/static/js/components/DatasetEventCard.tsx index 1f321933c4b5b..9dd1ee91e3731 100644 --- a/airflow/www/static/js/components/DatasetEventCard.tsx +++ b/airflow/www/static/js/components/DatasetEventCard.tsx @@ -43,7 +43,7 @@ import SourceTaskInstance from "./SourceTaskInstance"; import TriggeredDagRuns from "./TriggeredDagRuns"; type CardProps = { - datasetEvent: AssetEvent; + assetEvent: AssetEvent; showSource?: boolean; showTriggeredDagRuns?: boolean; }; @@ -51,7 +51,7 @@ type CardProps = { const datasetsUrl = getMetaValue("datasets_url"); const DatasetEventCard = ({ - datasetEvent, + assetEvent, showSource = true, showTriggeredDagRuns = true, }: CardProps) => { @@ -60,14 +60,16 @@ const DatasetEventCard = ({ const selectedUri = decodeURIComponent(searchParams.get("uri") || ""); const containerRef = useContainerRef(); - const { from_rest_api: fromRestApi, ...extra } = - datasetEvent?.extra as Record; + const { from_rest_api: fromRestApi, ...extra } = assetEvent?.extra as Record< + string, + string + >; return ( - @@ -111,17 +112,17 @@ const DatasetEventCard = ({ )} - {!!datasetEvent.sourceTaskId && ( - + {!!assetEvent.sourceTaskId && ( + )} )} - {showTriggeredDagRuns && !!datasetEvent?.createdDagruns?.length && ( + {showTriggeredDagRuns && !!assetEvent?.createdDagruns?.length && ( <> Triggered Dag Runs: - + )} diff --git a/airflow/www/static/js/components/SourceTaskInstance.tsx b/airflow/www/static/js/components/SourceTaskInstance.tsx index f52e9b4c75b85..4343d3ce82443 100644 --- a/airflow/www/static/js/components/SourceTaskInstance.tsx +++ b/airflow/www/static/js/components/SourceTaskInstance.tsx @@ -30,20 +30,16 @@ import type { TaskInstance } from "src/types"; import { getMetaValue } from "src/utils"; type SourceTIProps = { - datasetEvent: AssetEvent; + assetEvent: AssetEvent; showLink?: boolean; }; const gridUrl = getMetaValue("grid_url"); const dagId = getMetaValue("dag_id") || "__DAG_ID__"; -const SourceTaskInstance = ({ - datasetEvent, - showLink = true, -}: SourceTIProps) => { +const SourceTaskInstance = ({ assetEvent, showLink = true }: SourceTIProps) => { const containerRef = useContainerRef(); - const { sourceDagId, sourceRunId, sourceTaskId, sourceMapIndex } = - datasetEvent; + const { sourceDagId, sourceRunId, sourceTaskId, sourceMapIndex } = assetEvent; const { data: taskInstance } = useTaskInstance({ dagId: sourceDagId || "", diff --git a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx index 6946150b1d7db..6deedb073e8d9 100644 --- a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx +++ b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx @@ -34,7 +34,7 @@ const dagId = getMetaValue("dag_id"); const cardDef: CardDef = { card: ({ row }) => ( - + ), }; diff --git a/airflow/www/static/js/dag/details/graph/DatasetNode.tsx b/airflow/www/static/js/dag/details/graph/DatasetNode.tsx index bfd288f072dc3..d80f399b032a3 100644 --- a/airflow/www/static/js/dag/details/graph/DatasetNode.tsx +++ b/airflow/www/static/js/dag/details/graph/DatasetNode.tsx @@ -47,11 +47,11 @@ import type { CustomNodeProps } from "./Node"; const datasetsUrl = getMetaValue("datasets_url"); const DatasetNode = ({ - data: { label, height, width, latestDagRunId, isZoomedOut, datasetEvent }, + data: { label, height, width, latestDagRunId, isZoomedOut, assetEvent }, }: NodeProps) => { const containerRef = useContainerRef(); - const { from_rest_api: fromRestApi } = (datasetEvent?.extra || {}) as Record< + const { from_rest_api: fromRestApi } = (assetEvent?.extra || {}) as Record< string, string >; @@ -61,8 +61,8 @@ const DatasetNode = ({ Dataset - {!!datasetEvent && ( + {!!assetEvent && ( {/* @ts-ignore */} - {moment(datasetEvent.timestamp).fromNow()} + {moment(assetEvent.timestamp).fromNow()} )} @@ -120,23 +120,23 @@ const DatasetNode = ({ {label} - {!!datasetEvent && ( + {!!assetEvent && ( -