From 837171a561a541a905b8aec447d8c9f01c211c21 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 25 Aug 2023 12:24:15 +0800 Subject: [PATCH 1/2] docs(providers/microsoft): add example dag for AzureBatchOperator --- .../example_azure_batch_operator.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py diff --git a/airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py b/airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py new file mode 100644 index 0000000000000..e23a97a1318e0 --- /dev/null +++ b/airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Airflow DAG that trigger a task in Azure Batch. + +This DAG relies on the following OS environment variables + +* POOL_ID - The Pool ID in Batch accounts. +""" + + +from __future__ import annotations + +import os +from datetime import datetime + +from airflow import DAG +from airflow.providers.microsoft.azure.operators.batch import AzureBatchOperator + +POOL_ID = os.environ.get("POOL_ID", "example-pool") + +with DAG( + dag_id="example_azure_batch", + start_date=datetime(2021, 1, 1), + catchup=False, + doc_md=__doc__, + tags=["example"], +) as dag: + azure_batch_operator = AzureBatchOperator( + task_id="azure_batch", + batch_pool_id=POOL_ID, + batch_pool_vm_size="standard_d2s_v3", + batch_job_id="example-job", + batch_task_command_line=("/bin/bash -c 'set -e; set -o pipefail; echo hello world!; wait'"), + batch_task_id="example-task", + vm_node_agent_sku_id="batch.node.ubuntu 22.04", + vm_publisher="Canonical", + vm_offer="0001-com-ubuntu-server-jammy", + vm_sku="22_04-lts-gen2", + target_dedicated_nodes=1, + ) From c2ca4ffd4f7ba6239af032062958a6a6ba36fda4 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 25 Aug 2023 12:33:49 +0800 Subject: [PATCH 2/2] docs(providers/microsoft): add AzureBatchOperator example dag to doc page --- .../example_azure_batch_operator.py | 4 +- .../providers/microsoft/azure/provider.yaml | 2 + .../operators/batch.rst | 41 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 docs/apache-airflow-providers-microsoft-azure/operators/batch.rst diff --git a/airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py b/airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py index e23a97a1318e0..7dc1c68978097 100644 --- a/airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py +++ b/airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py @@ -41,12 +41,13 @@ doc_md=__doc__, tags=["example"], ) as dag: + # [START howto_azure_batch_operator] azure_batch_operator = AzureBatchOperator( task_id="azure_batch", batch_pool_id=POOL_ID, batch_pool_vm_size="standard_d2s_v3", batch_job_id="example-job", - batch_task_command_line=("/bin/bash -c 'set -e; set -o pipefail; echo hello world!; wait'"), + batch_task_command_line="/bin/bash -c 'set -e; set -o pipefail; echo hello world!; wait'", batch_task_id="example-task", vm_node_agent_sku_id="batch.node.ubuntu 22.04", vm_publisher="Canonical", @@ -54,3 +55,4 @@ vm_sku="22_04-lts-gen2", target_dedicated_nodes=1, ) + # [END howto_azure_batch_operator] diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index a0ed5fbdf2599..ab713bdbe1d84 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -87,6 +87,8 @@ dependencies: integrations: - integration-name: Microsoft Azure Batch external-doc-url: https://azure.microsoft.com/en-us/services/batch/ + how-to-guide: + - /docs/apache-airflow-providers-microsoft-azure/operators/batch.rst logo: /integration-logos/azure/Microsoft-Azure-Batch.png tags: [azure] - integration-name: Microsoft Azure Blob Storage diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst b/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst new file mode 100644 index 0000000000000..561a3e9b75bb2 --- /dev/null +++ b/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst @@ -0,0 +1,41 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Azure Batch Operator +================================= + +AzureBatchOperator +---------------------------------- +Use the +:class:`~airflow.providers.microsoft.azure.operators.batch.AzureBatchOperator` to trigger a task on Azure Batch + +Below is an example of using this operator to trigger a task on Azure Batch + +.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dag/example_azure_batch_operator.py + :language: python + :dedent: 0 + :start-after: [START howto_azure_batch_operator] + :end-before: [END howto_azure_batch_operator] + + +Reference +--------- + +For further information, look at: + +* `Azure Batch Documentation `__