From e0e8b88007e76aa75fd1d748cfc6a40305dbe193 Mon Sep 17 00:00:00 2001 From: kaxil Date: Mon, 23 Dec 2019 16:44:21 +0000 Subject: [PATCH 1/5] [AIRFLOW-4113] Unpin boto3 --- setup.py | 4 ++-- tests/hooks/test_s3_hook.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c8e5392b3b326..457074f3b1004 100644 --- a/setup.py +++ b/setup.py @@ -154,7 +154,7 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])): 'atlasclient>=0.1.2', ] aws = [ - 'boto3>=1.7.0, <1.8.0', + 'boto3~=1.10', ] azure = [ 'azure-cosmos>=3.0.1', @@ -380,7 +380,7 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])): 'ipdb', 'jira', 'mongomock', - 'moto==1.3.5', + 'moto>=1.3.14', 'parameterized', 'paramiko', 'pre-commit', diff --git a/tests/hooks/test_s3_hook.py b/tests/hooks/test_s3_hook.py index a5a25a29e6c3c..3d376b6f13a94 100644 --- a/tests/hooks/test_s3_hook.py +++ b/tests/hooks/test_s3_hook.py @@ -88,7 +88,9 @@ def test_create_bucket_us_standard_region(self): bucket = hook.get_bucket('new_bucket') self.assertIsNotNone(bucket) region = bucket.meta.client.get_bucket_location(Bucket=bucket.name).get('LocationConstraint', None) - self.assertEqual(region, 'us-east-1') + # https://github.com/spulec/moto/pull/1961 + # If location is "us-east-1", LocationConstraint should be None + self.assertIsNone(region) @mock_s3 def test_create_bucket_other_region(self): From edc2a560605b7d1efeba16546ee5a8fe55d586cd Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Mon, 23 Dec 2019 23:19:30 +0000 Subject: [PATCH 2/5] Fix test --- tests/contrib/hooks/test_aws_hook.py | 32 ++++++---------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/tests/contrib/hooks/test_aws_hook.py b/tests/contrib/hooks/test_aws_hook.py index 21183098edd19..7323410fcfdb3 100644 --- a/tests/contrib/hooks/test_aws_hook.py +++ b/tests/contrib/hooks/test_aws_hook.py @@ -192,33 +192,13 @@ def test_get_credentials_from_role_arn(self, mock_get_connection): mock_get_connection.return_value = mock_connection hook = AwsHook() credentials_from_hook = hook.get_credentials() - self.assertEqual(credentials_from_hook.access_key, 'AKIAIOSFODNN7EXAMPLE') - self.assertEqual(credentials_from_hook.secret_key, - 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY') - self.assertEqual(credentials_from_hook.token, - 'BQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh' - '3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4I' - 'gRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15' - 'fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE') + self.assertIn("ASIA", credentials_from_hook.access_key) - @unittest.skipIf(mock_sts is None, 'mock_sts package not present') - @mock.patch.object(AwsHook, 'get_connection') - @mock_sts - def test_get_credentials_from_role_arn_with_external_id(self, mock_get_connection): - mock_connection = Connection( - extra='{"role_arn":"arn:aws:iam::123456:role/role_arn",' - ' "external_id":"external_id"}') - mock_get_connection.return_value = mock_connection - hook = AwsHook() - credentials_from_hook = hook.get_credentials() - self.assertEqual(credentials_from_hook.access_key, 'AKIAIOSFODNN7EXAMPLE') - self.assertEqual(credentials_from_hook.secret_key, - 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY') - self.assertEqual(credentials_from_hook.token, - 'BQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh' - '3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4I' - 'gRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15' - 'fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE') + # We assert the length instead of actual values as the values are random: + # Details: https://github.com/spulec/moto/commit/ab0d23a0ba2506e6338ae20b3fde70da049f7b03 + self.assertEqual(20, len(credentials_from_hook.access_key)) + self.assertEqual(40, len(credentials_from_hook.secret_key)) + self.assertEqual(356, len(credentials_from_hook.token)) @unittest.skipIf(mock_iam is None, 'mock_iam package not present') @mock_iam From c5e391c7fca267ff7a0fb22a7b2cab9d88f98ae2 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Tue, 24 Dec 2019 00:08:50 +0000 Subject: [PATCH 3/5] Fix tests for DynamoDB --- tests/contrib/hooks/test_aws_dynamodb_hook.py | 2 +- tests/contrib/hooks/test_aws_hook.py | 4 ++-- tests/contrib/operators/test_hive_to_dynamodb_operator.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/contrib/hooks/test_aws_dynamodb_hook.py b/tests/contrib/hooks/test_aws_dynamodb_hook.py index 835c042b2dc62..5eab8feeb7814 100644 --- a/tests/contrib/hooks/test_aws_dynamodb_hook.py +++ b/tests/contrib/hooks/test_aws_dynamodb_hook.py @@ -55,7 +55,7 @@ def test_insert_batch_items_dynamodb_table(self): ], AttributeDefinitions=[ { - 'AttributeName': 'name', + 'AttributeName': 'id', 'AttributeType': 'S' } ], diff --git a/tests/contrib/hooks/test_aws_hook.py b/tests/contrib/hooks/test_aws_hook.py index 7323410fcfdb3..232769ad02323 100644 --- a/tests/contrib/hooks/test_aws_hook.py +++ b/tests/contrib/hooks/test_aws_hook.py @@ -64,7 +64,7 @@ def test_get_resource_type_returns_a_boto3_resource_of_the_requested_type(self): ], AttributeDefinitions=[ { - 'AttributeName': 'name', + 'AttributeName': 'id', 'AttributeType': 'S' } ], @@ -95,7 +95,7 @@ def test_get_session_returns_a_boto3_session(self): ], AttributeDefinitions=[ { - 'AttributeName': 'name', + 'AttributeName': 'id', 'AttributeType': 'S' } ], diff --git a/tests/contrib/operators/test_hive_to_dynamodb_operator.py b/tests/contrib/operators/test_hive_to_dynamodb_operator.py index 5948e83946cda..3732952b6de7e 100644 --- a/tests/contrib/operators/test_hive_to_dynamodb_operator.py +++ b/tests/contrib/operators/test_hive_to_dynamodb_operator.py @@ -75,7 +75,7 @@ def test_get_records_with_schema(self, mock_get_pandas_df): ], AttributeDefinitions=[ { - 'AttributeName': 'name', + 'AttributeName': 'id', 'AttributeType': 'S' } ], @@ -115,7 +115,7 @@ def test_pre_process_records_with_schema(self, mock_get_pandas_df): ], AttributeDefinitions=[ { - 'AttributeName': 'name', + 'AttributeName': 'id', 'AttributeType': 'S' } ], From 76376922d4ed0de64d9c3c8c66a286b5f3f350e8 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Tue, 24 Dec 2019 02:45:27 +0000 Subject: [PATCH 4/5] Skip test if create_task & delete_task method doesn't exist --- tests/providers/amazon/aws/hooks/test_datasync.py | 6 ++++++ tests/providers/amazon/aws/operators/test_datasync.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tests/providers/amazon/aws/hooks/test_datasync.py b/tests/providers/amazon/aws/hooks/test_datasync.py index 91a8e1e42eadd..89858d3951b44 100644 --- a/tests/providers/amazon/aws/hooks/test_datasync.py +++ b/tests/providers/amazon/aws/hooks/test_datasync.py @@ -32,7 +32,13 @@ def no_datasync(x): try: from moto import mock_datasync + from moto.datasync.models import DataSyncBackend + # ToDo: Remove after the moto>1.3.14 is released and contains following commit: + # https://github.com/spulec/moto/commit/5cfbe2bb3d24886f2b33bb4480c60b26961226fc + if "create_task" not in dir(DataSyncBackend) or "delete_task" not in dir(DataSyncBackend): + mock_datasync = no_datasync except ImportError: + # flake8: noqa: F811 mock_datasync = no_datasync diff --git a/tests/providers/amazon/aws/operators/test_datasync.py b/tests/providers/amazon/aws/operators/test_datasync.py index 046d7fa766336..af8c3a8c5946e 100644 --- a/tests/providers/amazon/aws/operators/test_datasync.py +++ b/tests/providers/amazon/aws/operators/test_datasync.py @@ -34,7 +34,13 @@ def no_datasync(x): try: from moto import mock_datasync + from moto.datasync.models import DataSyncBackend + # ToDo: Remove after the moto>1.3.14 is released and contains following commit: + # https://github.com/spulec/moto/commit/5cfbe2bb3d24886f2b33bb4480c60b26961226fc + if "create_task" not in dir(DataSyncBackend) or "delete_task" not in dir(DataSyncBackend): + mock_datasync = no_datasync except ImportError: + # flake8: noqa: F811 mock_datasync = no_datasync TEST_DAG_ID = "unit_tests" From 7eaec44820b35105d5d40d5a828cc79dcb2f0b5b Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Tue, 24 Dec 2019 14:00:05 +0000 Subject: [PATCH 5/5] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 457074f3b1004..f8b6f052b4831 100644 --- a/setup.py +++ b/setup.py @@ -380,7 +380,7 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])): 'ipdb', 'jira', 'mongomock', - 'moto>=1.3.14', + 'moto>=1.3.14,<2.0.0', 'parameterized', 'paramiko', 'pre-commit',