diff --git a/airflow/providers/apache/pig/example_dags/example_pig.py b/airflow/providers/apache/pig/example_dags/example_pig.py index e3ab899e2f5be..ed1b34ab0c8a4 100644 --- a/airflow/providers/apache/pig/example_dags/example_pig.py +++ b/airflow/providers/apache/pig/example_dags/example_pig.py @@ -30,9 +30,11 @@ tags=['example'], ) +# [START create_pig] run_this = PigOperator( task_id="run_example_pig_script", pig="ls /;", pig_opts="-x local", dag=dag, ) +# [END create_pig] diff --git a/airflow/providers/apache/pig/provider.yaml b/airflow/providers/apache/pig/provider.yaml index ae1cf0870dc5d..434e307b96ff1 100644 --- a/airflow/providers/apache/pig/provider.yaml +++ b/airflow/providers/apache/pig/provider.yaml @@ -19,7 +19,7 @@ package-name: apache-airflow-providers-apache-pig name: Apache Pig description: | - `Apache Pig `__ + `Apache Pig `__ versions: - 2.0.1 @@ -33,6 +33,8 @@ additional-dependencies: integrations: - integration-name: Apache Pig external-doc-url: https://pig.apache.org/ + how-to-guide: + - /docs/apache-airflow-providers-apache-pig/operators.rst logo: /integration-logos/apache/pig.png tags: [apache] @@ -46,7 +48,8 @@ hooks: python-modules: - airflow.providers.apache.pig.hooks.pig -hook-class-names: # deprecated - to be removed after providers add dependency on Airflow 2.2.0+ +hook-class-names: + # deprecated - to be removed after providers add dependency on Airflow 2.2.0+ - airflow.providers.apache.pig.hooks.pig.PigCliHook connection-types: diff --git a/docs/apache-airflow-providers-apache-pig/index.rst b/docs/apache-airflow-providers-apache-pig/index.rst index 490cfea2405da..4bd2b49184ce8 100644 --- a/docs/apache-airflow-providers-apache-pig/index.rst +++ b/docs/apache-airflow-providers-apache-pig/index.rst @@ -21,6 +21,12 @@ Content ------- +.. toctree:: + :maxdepth: 1 + :caption: Guides + + Operators + .. toctree:: :maxdepth: 1 :caption: References diff --git a/docs/apache-airflow-providers-apache-pig/operators.rst b/docs/apache-airflow-providers-apache-pig/operators.rst new file mode 100644 index 0000000000000..04e29e175b1e2 --- /dev/null +++ b/docs/apache-airflow-providers-apache-pig/operators.rst @@ -0,0 +1,32 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Apache Pig Operators +==================== + +Apache Pig is a platform for analyzing large data sets that consists of a high-level language +for expressing data analysis programs, coupled with infrastructure for evaluating these programs. +Pig programs are amenable to substantial parallelization, which in turns enables them to handle very large data sets. + +use the PigOperator to execute a pig script + +.. exampleinclude:: /../../airflow/providers/apache/pig/example_dags/example_pig.py + :language: python + :start-after: [START create_pig] + :end-before: [END create_pig]