From 86d3a17334423f84d28081cbcc859aa2f88d11be Mon Sep 17 00:00:00 2001 From: elipapa Date: Thu, 15 Jun 2017 22:49:32 +0100 Subject: [PATCH 1/4] Client is now APIClient since docker-py>2.0 --- airflow/operators/docker_operator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py index bca604cba9710..7a8c2ccf00f61 100644 --- a/airflow/operators/docker_operator.py +++ b/airflow/operators/docker_operator.py @@ -18,7 +18,7 @@ from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults from airflow.utils.file import TemporaryDirectory -from docker import Client, tls +from docker import APIClient, tls import ast @@ -142,7 +142,7 @@ def execute(self, context): ) self.docker_url = self.docker_url.replace('tcp://', 'https://') - self.cli = Client(base_url=self.docker_url, version=self.api_version, tls=tls_config) + self.cli = APIClient(base_url=self.docker_url, version=self.api_version, tls=tls_config) if ':' not in self.image: image = self.image + ':latest' From f466af0ea6259c0d03e0aa7c8e3d9234c442193d Mon Sep 17 00:00:00 2001 From: elipapa Date: Thu, 15 Jun 2017 22:51:50 +0100 Subject: [PATCH 2/4] docker-py has been renamed to docker on v>2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 80668dac7aff0..f3d109e736d28 100644 --- a/setup.py +++ b/setup.py @@ -134,7 +134,7 @@ def check_previous(): 'sphinx-rtd-theme>=0.1.6', 'Sphinx-PyPI-upload>=0.2.1' ] -docker = ['docker-py>=1.6.0'] +docker = ['docker>=2.0.0'] druid = ['pydruid>=0.2.1'] emr = ['boto3>=1.0.0'] gcp_api = [ From 98a10ddc83729f6c92442518a791d891c1476dd1 Mon Sep 17 00:00:00 2001 From: Eliseo Papa Date: Tue, 1 May 2018 18:42:58 +0100 Subject: [PATCH 3/4] change Client to APIClient --- airflow/hooks/docker_hook.py | 8 +++---- tests/hooks/test_docker_hook.py | 8 +++---- tests/operators/docker_operator.py | 34 +++++++++++++++--------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/airflow/hooks/docker_hook.py b/airflow/hooks/docker_hook.py index 14a4a6ed1fec1..be9f8fa95efd9 100644 --- a/airflow/hooks/docker_hook.py +++ b/airflow/hooks/docker_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -from docker import Client +from docker import APIClient from docker.errors import APIError from airflow.exceptions import AirflowException @@ -60,7 +60,7 @@ def __init__(self, self.__reauth = False if extra_options.get('reauth') == 'no' else True def get_conn(self): - client = Client( + client = APIClient( base_url=self.__base_url, version=self.__version, tls=self.__tls diff --git a/tests/hooks/test_docker_hook.py b/tests/hooks/test_docker_hook.py index 5753aa5a22f17..2d9270ebf62a4 100644 --- a/tests/hooks/test_docker_hook.py +++ b/tests/hooks/test_docker_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,7 +26,7 @@ try: from airflow.hooks.docker_hook import DockerHook - from docker import Client + from docker import APIClient except ImportError: pass @@ -39,7 +39,7 @@ mock = None -@mock.patch('airflow.hooks.docker_hook.Client', autospec=True) +@mock.patch('airflow.hooks.docker_hook.APIClient', autospec=True) class DockerHookTest(unittest.TestCase): def setUp(self): configuration.load_test_config() diff --git a/tests/operators/docker_operator.py b/tests/operators/docker_operator.py index 1ebcbe92e4eff..0be47942cd2d6 100644 --- a/tests/operators/docker_operator.py +++ b/tests/operators/docker_operator.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -23,7 +23,7 @@ try: from airflow.operators.docker_operator import DockerOperator from airflow.hooks.docker_hook import DockerHook - from docker import Client + from docker import APIClient except ImportError: pass @@ -40,12 +40,12 @@ class DockerOperatorTestCase(unittest.TestCase): @mock.patch('airflow.utils.file.mkdtemp') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute(self, client_class_mock, mkdtemp_mock): host_config = mock.Mock() mkdtemp_mock.return_value = '/mkdtemp' - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = host_config client_mock.images.return_value = [] @@ -84,9 +84,9 @@ def test_execute(self, client_class_mock, mkdtemp_mock): client_mock.wait.assert_called_with('some_id') @mock.patch('airflow.operators.docker_operator.tls.TLSConfig') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_tls(self, client_class_mock, tls_class_mock): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = mock.Mock() client_mock.images.return_value = [] @@ -110,9 +110,9 @@ def test_execute_tls(self, client_class_mock, tls_class_mock): client_class_mock.assert_called_with(base_url='https://127.0.0.1:2376', tls=tls_mock, version=None) - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_unicode_logs(self, client_class_mock): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = mock.Mock() client_mock.images.return_value = [] @@ -132,9 +132,9 @@ def test_execute_unicode_logs(self, client_class_mock): logging.raiseExceptions = originalRaiseExceptions print_exception_mock.assert_not_called() - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_container_fails(self, client_class_mock): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = mock.Mock() client_mock.images.return_value = [] @@ -150,7 +150,7 @@ def test_execute_container_fails(self, client_class_mock): operator.execute(None) def test_on_kill(self): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) operator = DockerOperator(image='ubuntu', owner='unittest', task_id='unittest') operator.cli = client_mock @@ -160,10 +160,10 @@ def test_on_kill(self): client_mock.stop.assert_called_with('some_id') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_no_docker_conn_id_no_hook(self, operator_client_mock): # Mock out a Docker client, so operations don't raise errors - client_mock = mock.Mock(name='DockerOperator.Client mock', spec=Client) + client_mock = mock.Mock(name='DockerOperator.APIClient mock', spec=APIClient) client_mock.images.return_value = [] client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.logs.return_value = [] @@ -194,11 +194,11 @@ def test_execute_no_docker_conn_id_no_hook(self, operator_client_mock): ) @mock.patch('airflow.operators.docker_operator.DockerHook') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_with_docker_conn_id_use_hook(self, operator_client_mock, operator_docker_hook): # Mock out a Docker client, so operations don't raise errors - client_mock = mock.Mock(name='DockerOperator.Client mock', spec=Client) + client_mock = mock.Mock(name='DockerOperator.APIClient mock', spec=APIClient) client_mock.images.return_value = [] client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.logs.return_value = [] @@ -223,7 +223,7 @@ def test_execute_with_docker_conn_id_use_hook(self, operator_client_mock, self.assertEqual( operator_client_mock.call_count, 0, - 'Client was called on the operator instead of the hook' + 'APIClient was called on the operator instead of the hook' ) self.assertEqual( operator_docker_hook.call_count, 1, From 53e8518fceb5d0a56346f9431d84afbdea1c76d8 Mon Sep 17 00:00:00 2001 From: Eliseo Papa Date: Tue, 1 May 2018 18:49:32 +0100 Subject: [PATCH 4/4] remove unused import --- tests/hooks/test_docker_hook.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/hooks/test_docker_hook.py b/tests/hooks/test_docker_hook.py index 2d9270ebf62a4..446e9fb457bcf 100644 --- a/tests/hooks/test_docker_hook.py +++ b/tests/hooks/test_docker_hook.py @@ -26,7 +26,6 @@ try: from airflow.hooks.docker_hook import DockerHook - from docker import APIClient except ImportError: pass