From f2ea600ec4b0444117d99361492ea78db642b08b Mon Sep 17 00:00:00 2001 From: Shubha Rajan Date: Fri, 15 May 2020 14:14:26 -0700 Subject: [PATCH 1/7] copy dockerfile from testing-infra-docker --- .kokoro/docker/python-3.6/Dockerfile | 160 +++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 .kokoro/docker/python-3.6/Dockerfile diff --git a/.kokoro/docker/python-3.6/Dockerfile b/.kokoro/docker/python-3.6/Dockerfile new file mode 100644 index 00000000000..dba390228ca --- /dev/null +++ b/.kokoro/docker/python-3.6/Dockerfile @@ -0,0 +1,160 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:16.04 + +ENV DEBIAN_FRONTEND noninteractive + +# Ensure local Python is preferred over distribution Python. +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# At the moment, setting "LANG=C" on a Linux system fundamentally breaks +# Python 3. +ENV LANG C.UTF-8 + +# Install dependencies. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + git \ + graphviz \ + libbz2-dev \ + libdb5.3-dev \ + libexpat1-dev \ + libffi-dev \ + liblzma-dev \ + libreadline-dev \ + libsnappy-dev \ + libssl-dev \ + libsqlite3-dev \ + portaudio19-dev \ + redis-server \ + software-properties-common \ + ssh \ + sudo \ + tcl \ + tcl-dev \ + tk \ + tk-dev \ + uuid-dev \ + wget \ + zlib1g-dev \ + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + +# Install docker +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + +RUN add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + docker-ce \ + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + +# Install Microsoft ODBC 17 Driver and unixodbc for testing SQL Server samples +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ + && curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list \ + && apt-get update \ + && ACCEPT_EULA=Y apt-get install -y --no-install-recommends \ + msodbcsql17 \ + unixodbc-dev \ + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + +# Install the desired versions of Python. +RUN for PYTHON_VERSION in 2.7.17 3.4.10 3.5.9 3.6.9 3.7.5 3.8.0; do \ + set -ex \ + && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ + # 2.7.17 (Benjamin Peterson) + C01E1CAD5EA2C4F0B8E3571504C367C218ADD4FF \ + # 3.4.10, 3.5.9 (Larry Hastings) + 97FC712E4C024BBEA48A61ED3A5CA953F73C700D \ + # 3.6.9, 3.7.5 (Ned Deily) + 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D \ + # 3.8.0 (Ɓukasz Langa) + E3FF2839C048B25C084DEBE9B26995E310250568 \ + && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \ + && rm -r "$GNUPGHOME" python-${PYTHON_VERSION}.tar.xz.asc \ + && mkdir -p /usr/src/python-${PYTHON_VERSION} \ + && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \ + && rm python-${PYTHON_VERSION}.tar.xz \ + && cd /usr/src/python-${PYTHON_VERSION} \ + && ./configure \ + --enable-shared \ + # This works only on Python 2.7 and throws a warning on every other + # version, but seems otherwise harmless. + --enable-unicode=ucs4 \ + --with-system-ffi \ + --without-ensurepip \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + ; done \ + && rm -rf /usr/src/python* \ + && rm -rf ~/.cache/ + + +# Install pip on Python 3.6 only. +# If the environment variable is called "PIP_VERSION", pip explodes with +# "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 10.0.1 +RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3.6 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \ + # we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python + # ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.6/site-packages") + # https://github.com/docker-library/python/pull/143#issuecomment-241032683 + && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \ + # then we use "pip list" to ensure we don't have more than one pip version installed + # https://github.com/docker-library/python/pull/100 + && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ] + +# Ensure Pip for 3.8.0 +RUN python3.8 /tmp/get-pip.py +RUN rm /tmp/get-pip.py + +# Install "virtualenv", since the vast majority of users of this image +# will want it. +RUN pip install --no-cache-dir virtualenv + +# Setup Cloud SDK +ENV CLOUD_SDK_VERSION 272.0.0 +# Use system python for cloud sdk. +ENV CLOUDSDK_PYTHON python3.6 +RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN tar xzf google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz +RUN /google-cloud-sdk/install.sh +ENV PATH /google-cloud-sdk/bin:$PATH + +# Enable redis-server on boot. +RUN sudo systemctl enable redis-server.service + +CMD ["python3.6"] \ No newline at end of file From 0f2422bc5d841fde8c7730cba828f71a095bd7dc Mon Sep 17 00:00:00 2001 From: Shubha Rajan Date: Fri, 15 May 2020 14:15:05 -0700 Subject: [PATCH 2/7] remove python3.4 and python3.5 for faster builds --- .kokoro/docker/{python-3.6 => }/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .kokoro/docker/{python-3.6 => }/Dockerfile (98%) diff --git a/.kokoro/docker/python-3.6/Dockerfile b/.kokoro/docker/Dockerfile similarity index 98% rename from .kokoro/docker/python-3.6/Dockerfile rename to .kokoro/docker/Dockerfile index dba390228ca..5bb2975b854 100644 --- a/.kokoro/docker/python-3.6/Dockerfile +++ b/.kokoro/docker/Dockerfile @@ -88,7 +88,7 @@ RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ && rm -f /var/cache/apt/archives/*.deb # Install the desired versions of Python. -RUN for PYTHON_VERSION in 2.7.17 3.4.10 3.5.9 3.6.9 3.7.5 3.8.0; do \ +RUN for PYTHON_VERSION in 2.7.17 3.6.9 3.7.5 3.8.0; do \ set -ex \ && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ From b6bca43e0f69634f500a4b0ad65b23dec0749763 Mon Sep 17 00:00:00 2001 From: Shubha Rajan Date: Fri, 15 May 2020 16:13:25 -0700 Subject: [PATCH 3/7] add cloudbuild.yaml --- .kokoro/docker/cloudbuild.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .kokoro/docker/cloudbuild.yaml diff --git a/.kokoro/docker/cloudbuild.yaml b/.kokoro/docker/cloudbuild.yaml new file mode 100644 index 00000000000..6f8a66231de --- /dev/null +++ b/.kokoro/docker/cloudbuild.yaml @@ -0,0 +1,5 @@ +steps: + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/python-samples-testing-docker', '.'] +images: + - 'gcr.io/$PROJECT_ID/python-samples-testing-docker' \ No newline at end of file From 4487d51076213dc168439710efeb27ec1789db34 Mon Sep 17 00:00:00 2001 From: Shubha Rajan Date: Fri, 15 May 2020 16:41:41 -0700 Subject: [PATCH 4/7] update directory in cloudbuild.yaml --- .kokoro/docker/cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.kokoro/docker/cloudbuild.yaml b/.kokoro/docker/cloudbuild.yaml index 6f8a66231de..ac7900613b9 100644 --- a/.kokoro/docker/cloudbuild.yaml +++ b/.kokoro/docker/cloudbuild.yaml @@ -1,5 +1,6 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: ['build', '-t', 'gcr.io/$PROJECT_ID/python-samples-testing-docker', '.'] + dir: '.kokoro/docker' images: - 'gcr.io/$PROJECT_ID/python-samples-testing-docker' \ No newline at end of file From 53d891d9ac47016e788657cb190c99ba22b575ba Mon Sep 17 00:00:00 2001 From: Shubha Rajan Date: Fri, 15 May 2020 16:57:17 -0700 Subject: [PATCH 5/7] add license header and timeout to cloudbuild.yaml --- .kokoro/docker/cloudbuild.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.kokoro/docker/cloudbuild.yaml b/.kokoro/docker/cloudbuild.yaml index ac7900613b9..d6776f5967d 100644 --- a/.kokoro/docker/cloudbuild.yaml +++ b/.kokoro/docker/cloudbuild.yaml @@ -1,3 +1,19 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +timeout: 7200s # 2 hours + steps: - name: 'gcr.io/cloud-builders/docker' args: ['build', '-t', 'gcr.io/$PROJECT_ID/python-samples-testing-docker', '.'] From a46a22adb9d940badacb1ac06f3dffd5c5931a59 Mon Sep 17 00:00:00 2001 From: Shubha Rajan Date: Mon, 18 May 2020 12:45:39 -0700 Subject: [PATCH 6/7] add libmagickwand-dev to installed packages --- .kokoro/docker/Dockerfile | 3 ++- .kokoro/docker/cloudbuild.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile index 5bb2975b854..cb2e323b501 100644 --- a/.kokoro/docker/Dockerfile +++ b/.kokoro/docker/Dockerfile @@ -38,6 +38,7 @@ RUN apt-get update \ libexpat1-dev \ libffi-dev \ liblzma-dev \ + libmagickwand-dev \ libreadline-dev \ libsnappy-dev \ libssl-dev \ @@ -157,4 +158,4 @@ ENV PATH /google-cloud-sdk/bin:$PATH # Enable redis-server on boot. RUN sudo systemctl enable redis-server.service -CMD ["python3.6"] \ No newline at end of file +CMD ["python3.6"] diff --git a/.kokoro/docker/cloudbuild.yaml b/.kokoro/docker/cloudbuild.yaml index d6776f5967d..553d3cfa48d 100644 --- a/.kokoro/docker/cloudbuild.yaml +++ b/.kokoro/docker/cloudbuild.yaml @@ -19,4 +19,4 @@ steps: args: ['build', '-t', 'gcr.io/$PROJECT_ID/python-samples-testing-docker', '.'] dir: '.kokoro/docker' images: - - 'gcr.io/$PROJECT_ID/python-samples-testing-docker' \ No newline at end of file + - 'gcr.io/$PROJECT_ID/python-samples-testing-docker' From c27d4a1f0441d26900ad4da475589514dc9a34d5 Mon Sep 17 00:00:00 2001 From: Shubha Rajan Date: Tue, 19 May 2020 10:17:57 -0700 Subject: [PATCH 7/7] tag images with commit hash --- .kokoro/docker/cloudbuild.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.kokoro/docker/cloudbuild.yaml b/.kokoro/docker/cloudbuild.yaml index 553d3cfa48d..a9970bc6c6e 100644 --- a/.kokoro/docker/cloudbuild.yaml +++ b/.kokoro/docker/cloudbuild.yaml @@ -18,5 +18,7 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: ['build', '-t', 'gcr.io/$PROJECT_ID/python-samples-testing-docker', '.'] dir: '.kokoro/docker' + - name: gcr.io/cloud-builders/docker + args: ['tag', 'gcr.io/$PROJECT_ID/python-samples-testing-docker', 'gcr.io/$PROJECT_ID/python-samples-testing-docker:$SHORT_SHA'] images: - 'gcr.io/$PROJECT_ID/python-samples-testing-docker'