From a2f042c9d8bec7825752e1a4df7c1fa5dd95b8da Mon Sep 17 00:00:00 2001 From: "Y.Yaswanth Raj Kumar" Date: Fri, 15 Mar 2019 12:52:18 +0530 Subject: [PATCH 1/4] [AIRFLOW-4095]Added some params as template fields Added below attributes as template_fields in S3CopyObjectOperator, now those params will accept Jinja template and values will changes in run time * source_bucket_key * dest_bucket_key * source_bucket_name * dest_bucket_name --- airflow/contrib/operators/s3_copy_object_operator.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/airflow/contrib/operators/s3_copy_object_operator.py b/airflow/contrib/operators/s3_copy_object_operator.py index 330138ed27594..406dfc94c0a9e 100644 --- a/airflow/contrib/operators/s3_copy_object_operator.py +++ b/airflow/contrib/operators/s3_copy_object_operator.py @@ -29,21 +29,21 @@ class S3CopyObjectOperator(BaseOperator): Note: the S3 connection used here needs to have access to both source and destination bucket/key. - :param source_bucket_key: The key of the source object. + :param source_bucket_key: The key of the source object. (templated) It can be either full s3:// style url or relative path from root level. When it's specified as a full s3:// url, please omit source_bucket_name. :type source_bucket_key: str - :param dest_bucket_key: The key of the object to copy to. + :param dest_bucket_key: The key of the object to copy to. (templated) The convention to specify `dest_bucket_key` is the same as `source_bucket_key`. :type dest_bucket_key: str - :param source_bucket_name: Name of the S3 bucket where the source object is in. + :param source_bucket_name: Name of the S3 bucket where the source object is in. (templated) It should be omitted when `source_bucket_key` is provided as a full s3:// url. :type source_bucket_name: str - :param dest_bucket_name: Name of the S3 bucket to where the object is copied. + :param dest_bucket_name: Name of the S3 bucket to where the object is copied. (templated) It should be omitted when `dest_bucket_key` is provided as a full s3:// url. :type dest_bucket_name: str @@ -65,6 +65,9 @@ class S3CopyObjectOperator(BaseOperator): :type verify: bool or str """ + template_fields = ('source_bucket_key', 'dest_bucket_key', + 'source_bucket_name', 'dest_bucket_name') + @apply_defaults def __init__( self, From 8ca0cdf6625827ecee4c318ca48dbe327231959f Mon Sep 17 00:00:00 2001 From: "Y.Yaswanth Raj Kumar" Date: Fri, 15 Mar 2019 13:02:45 +0530 Subject: [PATCH 2/4] [AIRFLOW-4095]Added some params as template fields --- airflow/contrib/operators/s3_delete_objects_operator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airflow/contrib/operators/s3_delete_objects_operator.py b/airflow/contrib/operators/s3_delete_objects_operator.py index 76caace90f7b9..0c4c5f98a42c0 100644 --- a/airflow/contrib/operators/s3_delete_objects_operator.py +++ b/airflow/contrib/operators/s3_delete_objects_operator.py @@ -30,9 +30,9 @@ class S3DeleteObjectsOperator(BaseOperator): Users may specify up to 1000 keys to delete. - :param bucket: Name of the bucket in which you are going to delete object(s) + :param bucket: Name of the bucket in which you are going to delete object(s). (templated) :type bucket: str - :param keys: The key(s) to delete from S3 bucket. + :param keys: The key(s) to delete from S3 bucket. (templated) When ``keys`` is a string, it's supposed to be the key name of the single object to delete. @@ -58,6 +58,8 @@ class S3DeleteObjectsOperator(BaseOperator): :type verify: bool or str """ + template_fields = ('keys', 'bucket') + @apply_defaults def __init__( self, From 84dbf2857362426a0f89feb895547e1b3311d285 Mon Sep 17 00:00:00 2001 From: "Y.Yaswanth Raj Kumar" Date: Fri, 15 Mar 2019 13:49:23 +0530 Subject: [PATCH 3/4] [AIRFLOW-4095]Removed extra space --- airflow/contrib/operators/s3_copy_object_operator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/contrib/operators/s3_copy_object_operator.py b/airflow/contrib/operators/s3_copy_object_operator.py index 406dfc94c0a9e..43651707f8925 100644 --- a/airflow/contrib/operators/s3_copy_object_operator.py +++ b/airflow/contrib/operators/s3_copy_object_operator.py @@ -67,7 +67,7 @@ class S3CopyObjectOperator(BaseOperator): template_fields = ('source_bucket_key', 'dest_bucket_key', 'source_bucket_name', 'dest_bucket_name') - + @apply_defaults def __init__( self, From 443b64842681b0d6cd026d002637721061c23895 Mon Sep 17 00:00:00 2001 From: "Y.Yaswanth Raj Kumar" Date: Fri, 15 Mar 2019 13:50:35 +0530 Subject: [PATCH 4/4] [AIRFLOW-4095] Removed extra space --- airflow/contrib/operators/s3_delete_objects_operator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/contrib/operators/s3_delete_objects_operator.py b/airflow/contrib/operators/s3_delete_objects_operator.py index 0c4c5f98a42c0..926667b497195 100644 --- a/airflow/contrib/operators/s3_delete_objects_operator.py +++ b/airflow/contrib/operators/s3_delete_objects_operator.py @@ -59,7 +59,7 @@ class S3DeleteObjectsOperator(BaseOperator): """ template_fields = ('keys', 'bucket') - + @apply_defaults def __init__( self,