From 634cdfa9430ca859aa6b298991df9fb715a47e89 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Mon, 29 Jul 2019 14:49:42 -0700 Subject: [PATCH 1/6] new dockerfile --- Dockerfile | 13 +++++++++++++ environment_setup/Dockerfile | 14 ++++++++++---- environment_setup/requirements.txt | 19 ++++++++++++++----- 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e65c7ca5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM conda/miniconda3 + +LABEL org.label-schema.vendor = "Microsoft" \ + org.label-schema.url = "dockerhub" \ + org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython" + + + +COPY environment_setup/requirements.txt /setup/ + +RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt + +CMD ["python"] \ No newline at end of file diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile index d199b243..e65c7ca5 100644 --- a/environment_setup/Dockerfile +++ b/environment_setup/Dockerfile @@ -1,7 +1,13 @@ FROM conda/miniconda3 -COPY . /setup +LABEL org.label-schema.vendor = "Microsoft" \ + org.label-schema.url = "dockerhub" \ + org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython" -RUN ["apt", "update"] -RUN apt install gcc -y && pip install -r /setup/requirements.txt -RUN pip install azure-cli==2.0.65 && pip install --upgrade azureml-sdk[cli] \ No newline at end of file + + +COPY environment_setup/requirements.txt /setup/ + +RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt + +CMD ["python"] \ No newline at end of file diff --git a/environment_setup/requirements.txt b/environment_setup/requirements.txt index b3c2a14c..25a4b815 100644 --- a/environment_setup/requirements.txt +++ b/environment_setup/requirements.txt @@ -1,5 +1,14 @@ -scipy==1.0.0 -scikit-learn==0.19.1 -numpy==1.14.5 -pandas==0.23.1 -pytest==4.3.0 \ No newline at end of file +#numpy==1.17.0 +#scipy==1.2.2 +#scikit-learn==0.19.1 +#pandas==0.23.1 +pytest==4.3.0 +flake8==3.7.8 +flake8_formatter_junit_xml==0.0.6 +requests>=2.22 +python-dotenv>=0.10.3 +azureml>=0.2 +azureml-core>=1.0 +azureml-pipeline>=1.0 +azure-cli==2.0.46 +azureml-sdk[cli] From e2290971083a6fb6de9264eb462bf9e71a4a60d0 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Tue, 30 Jul 2019 09:38:05 -0700 Subject: [PATCH 2/6] Pipeline building a docker image --- environment_setup/docker-image-pipeline.yml | 29 +++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/environment_setup/docker-image-pipeline.yml b/environment_setup/docker-image-pipeline.yml index 2ee943f1..600fc271 100644 --- a/environment_setup/docker-image-pipeline.yml +++ b/environment_setup/docker-image-pipeline.yml @@ -13,20 +13,17 @@ trigger: include: - environment_setup/* -steps: - -- task: Docker@2 - displayName: Login to Docker Hub - inputs: - command: login - containerRegistry: msftDockerHub +variables: + containerRegistry: $[coalesce(variables['acrServiceConnection'], 'acrconnection')] + imageName: $[coalesce(variables['imageName'], 'mlopspython')] -- task: Docker@2 - displayName: 'Build and push an image' - inputs: - command: buildAndPush - repository: dtzar/conda3mlops - buildContext: '$(Build.SourcesDirectory)/environment_setup' - tags: | - $(Build.BuildNumber) - latest \ No newline at end of file +steps: + - task: Docker@2 + displayName: Build and Push + inputs: + command: buildAndPush + containerRegistry: '$(containerRegistry)' + repository: '$(imageName)' + tags: 'latest' + buildContext: '$(Build.SourcesDirectory)' + dockerFile: '$(Build.SourcesDirectory)/environment_setup/Dockerfile' From 176319a31930500d855b0a1183f32de2c567ce07 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Tue, 30 Jul 2019 10:08:19 -0700 Subject: [PATCH 3/6] Update docker-image-pipeline.yml --- environment_setup/docker-image-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_setup/docker-image-pipeline.yml b/environment_setup/docker-image-pipeline.yml index 600fc271..37637ba6 100644 --- a/environment_setup/docker-image-pipeline.yml +++ b/environment_setup/docker-image-pipeline.yml @@ -15,7 +15,7 @@ trigger: variables: containerRegistry: $[coalesce(variables['acrServiceConnection'], 'acrconnection')] - imageName: $[coalesce(variables['imageName'], 'mlopspython')] + imageName: $[coalesce(variables['imageName'], 'public/mlops/mlopspython')] steps: - task: Docker@2 From 6d8704081ae92a8c26e4f83646fb95ef7a006bcd Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Tue, 30 Jul 2019 11:12:56 -0700 Subject: [PATCH 4/6] Pipeline building an image --- Dockerfile | 13 ------------- azure-pipelines.yml | 4 +++- environment_setup/docker-image-pipeline.yml | 2 +- environment_setup/requirements.txt | 7 ------- 4 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e65c7ca5..00000000 --- a/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM conda/miniconda3 - -LABEL org.label-schema.vendor = "Microsoft" \ - org.label-schema.url = "dockerhub" \ - org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython" - - - -COPY environment_setup/requirements.txt /setup/ - -RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt - -CMD ["python"] \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 787fa327..b1ae0a5d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,9 @@ pool: vmImage: 'Ubuntu 16.04' -container: dtzar/conda3mlops:latest +container: + image: mlopscr.azurecr.io/public/mlops/mlopspython:latest + endpoint: acrconnection variables: - group: AzureKeyVaultSecrets diff --git a/environment_setup/docker-image-pipeline.yml b/environment_setup/docker-image-pipeline.yml index 600fc271..c88884d8 100644 --- a/environment_setup/docker-image-pipeline.yml +++ b/environment_setup/docker-image-pipeline.yml @@ -15,7 +15,7 @@ trigger: variables: containerRegistry: $[coalesce(variables['acrServiceConnection'], 'acrconnection')] - imageName: $[coalesce(variables['imageName'], 'mlopspython')] + imageName: $[coalesce(variables['agentImageName'], 'public/mlops/mlopspython')] steps: - task: Docker@2 diff --git a/environment_setup/requirements.txt b/environment_setup/requirements.txt index 25a4b815..bbd03023 100644 --- a/environment_setup/requirements.txt +++ b/environment_setup/requirements.txt @@ -1,12 +1,5 @@ -#numpy==1.17.0 -#scipy==1.2.2 -#scikit-learn==0.19.1 -#pandas==0.23.1 pytest==4.3.0 -flake8==3.7.8 -flake8_formatter_junit_xml==0.0.6 requests>=2.22 -python-dotenv>=0.10.3 azureml>=0.2 azureml-core>=1.0 azureml-pipeline>=1.0 From 53c61276dc7bc3592341599e0b9b22f8466e39ad Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Tue, 30 Jul 2019 11:15:29 -0700 Subject: [PATCH 5/6] resolving conflicts --- environment_setup/docker-image-pipeline.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/environment_setup/docker-image-pipeline.yml b/environment_setup/docker-image-pipeline.yml index 9e2fdcd0..c88884d8 100644 --- a/environment_setup/docker-image-pipeline.yml +++ b/environment_setup/docker-image-pipeline.yml @@ -15,11 +15,7 @@ trigger: variables: containerRegistry: $[coalesce(variables['acrServiceConnection'], 'acrconnection')] -<<<<<<< HEAD imageName: $[coalesce(variables['agentImageName'], 'public/mlops/mlopspython')] -======= - imageName: $[coalesce(variables['imageName'], 'public/mlops/mlopspython')] ->>>>>>> 176319a31930500d855b0a1183f32de2c567ce07 steps: - task: Docker@2 From 791e1b54b9100a7766153edbd3ae63bd655970a5 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 31 Jul 2019 11:59:11 -0700 Subject: [PATCH 6/6] URL to the image on Dockerhub --- environment_setup/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile index e65c7ca5..b6b3be6a 100644 --- a/environment_setup/Dockerfile +++ b/environment_setup/Dockerfile @@ -1,7 +1,7 @@ FROM conda/miniconda3 LABEL org.label-schema.vendor = "Microsoft" \ - org.label-schema.url = "dockerhub" \ + org.label-schema.url = "https://hub.docker.com/r/microsoft/mlopspython" \ org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython"