From 4d39bff88b213f9ccf5607a02dffb19cb6b732b3 Mon Sep 17 00:00:00 2001 From: driazati Date: Mon, 14 Feb 2022 15:15:28 -0800 Subject: [PATCH 1/2] Update CI to Python 3.7 and Ubuntu 18 This updates the docker image build to use Python 3.7 and Ubuntu 18 as discussed in #9703. The update is mostly straightforward except that the `apt` boost isn't built with 3.7 so we must now build it from source. --- docker/Dockerfile.ci_cpu | 5 +++- docker/Dockerfile.ci_i386 | 6 ++--- docker/install/ubuntu1804_install_python.sh | 4 +-- docker/install/ubuntu_install_boost.sh | 30 +++++++++++++++++++++ docker/install/ubuntu_install_caffe.sh | 28 +++++++++++-------- docker/install/ubuntu_install_llvm.sh | 9 ++++--- 6 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 docker/install/ubuntu_install_boost.sh diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu index 962d738a9fc2..cff6209920ac 100644 --- a/docker/Dockerfile.ci_cpu +++ b/docker/Dockerfile.ci_cpu @@ -16,7 +16,6 @@ # under the License. # CI docker CPU env -# tag: v0.62 FROM ubuntu:18.04 RUN apt-get update --fix-missing @@ -99,6 +98,10 @@ COPY install/ubuntu_download_arm_compute_lib_binaries.sh /install/ubuntu_downloa RUN bash /install/ubuntu_download_arm_compute_lib_binaries.sh # Caffe deps +COPY install/ubuntu_install_boost.sh /install/ubuntu_install_boost.sh +RUN bash /install/ubuntu_install_boost.sh + +# Caffe COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh RUN bash /install/ubuntu_install_caffe.sh diff --git a/docker/Dockerfile.ci_i386 b/docker/Dockerfile.ci_i386 index d4ce370c4205..e5e02b9755e4 100644 --- a/docker/Dockerfile.ci_i386 +++ b/docker/Dockerfile.ci_i386 @@ -18,7 +18,7 @@ # CI docker i386 env # tag: v0.53 -FROM ioft/i386-ubuntu:16.04 +FROM i386/ubuntu:18.04 RUN apt-get update --fix-missing && apt-get install -y ca-certificates @@ -28,8 +28,8 @@ RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh RUN bash /install/ubuntu_install_llvm.sh -COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh -RUN bash /install/ubuntu_install_python.sh +COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh +RUN bash /install/ubuntu1804_install_python.sh # Globally disable pip cache RUN pip config set global.no-cache-dir false diff --git a/docker/install/ubuntu1804_install_python.sh b/docker/install/ubuntu1804_install_python.sh index 5d39918f918c..f0af141133d6 100755 --- a/docker/install/ubuntu1804_install_python.sh +++ b/docker/install/ubuntu1804_install_python.sh @@ -23,10 +23,10 @@ set -o pipefail # install python and pip, don't modify this, modify install_python_package.sh apt-get update apt-get install -y software-properties-common -apt-get install -y python3-dev python3-setuptools +apt-get install -y python3.7 python3.7-dev python3-pip # Install pip -cd /tmp && wget -q https://bootstrap.pypa.io/pip/3.6/get-pip.py && python3 get-pip.py +update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 # Pin pip and setuptools versions pip3 install pip==19.3.1 setuptools==58.4.0 diff --git a/docker/install/ubuntu_install_boost.sh b/docker/install/ubuntu_install_boost.sh new file mode 100644 index 000000000000..ca56772a9b09 --- /dev/null +++ b/docker/install/ubuntu_install_boost.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.gz +tar -xf boost_1_67_0.tar.gz + +pushd boost_1_67_0 +./bootstrap.sh --with-python="$(which python3.7)" +./b2 install --with-python --with-system --with-filesystem --with-thread --with-regex +popd + +ln -s /usr/local/lib/libboost_python37.so.1.67.0 /usr/local/lib/libboost_python.so +ln -s /usr/local/lib/libboost_python37.a /usr/local/lib/libboost_python.a diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh index b26f339c3fa1..ab71eab54a2d 100644 --- a/docker/install/ubuntu_install_caffe.sh +++ b/docker/install/ubuntu_install_caffe.sh @@ -16,17 +16,15 @@ # specific language governing permissions and limitations # under the License. -set -e -set -u -set -o pipefail +set -euxo pipefail apt-get update --fix-missing -# Install dependencies -apt-get install -y --no-install-recommends libboost-filesystem-dev libboost-python-dev \ - libboost-system-dev libboost-thread-dev libboost-regex-dev protobuf-compiler \ +# # Install dependencies +apt-get install -y --no-install-recommends protobuf-compiler \ libprotobuf-dev libhdf5-serial-dev libopenblas-dev libgflags-dev libgoogle-glog-dev + # install python packages pip install "numpy" "protobuf" "scikit-image" "six" @@ -36,15 +34,23 @@ CAFFE_HOME="/opt/caffe" git clone --branch=ssd --depth 1 https://github.com/weiliu89/caffe /caffe_src cd /caffe_src + echo "Building Caffe" mkdir /caffe_src/build && cd /caffe_src/build -cmake .. -DCMAKE_INSTALL_PREFIX=${CAFFE_HOME} -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=1 \ - -Dpython_version=3 -DUSE_OPENCV=OFF -DUSE_LEVELDB=OFF -DUSE_LMDB=OFF -DBUILD_docs=OFF -DBLAS=open +cmake -DCMAKE_INSTALL_PREFIX=${CAFFE_HOME}\ + -DCMAKE_BUILD_TYPE=Release \ + -DCPU_ONLY=1 \ + -Dpython_version=3 \ + -DUSE_OPENCV=OFF \ + -DUSE_LEVELDB=OFF \ + -DUSE_LMDB=OFF \ + -DBUILD_docs=OFF \ + -DBLAS=open \ + .. + make all -j$(expr $(nproc) - 1) make pycaffe -j$(expr $(nproc) - 1) make test -j$(expr $(nproc) - 1) -make runtest -j$(expr $(nproc) - 1) -make pytest -j$(expr $(nproc) - 1) echo "Installing Caffe to /opt/caffe" make install @@ -54,4 +60,4 @@ cd / && rm -rf /caffe_src PYCAFFE_ROOT=${CAFFE_HOME}/python echo "${CAFFE_HOME}/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig -ln -s ${PYCAFFE_ROOT}/caffe /usr/local/lib/python3.6/dist-packages/caffe +ln -s ${PYCAFFE_ROOT}/caffe /usr/local/lib/python3.7/dist-packages/caffe diff --git a/docker/install/ubuntu_install_llvm.sh b/docker/install/ubuntu_install_llvm.sh index 1f4eec6ee6ef..6616bfc5eb2b 100755 --- a/docker/install/ubuntu_install_llvm.sh +++ b/docker/install/ubuntu_install_llvm.sh @@ -16,9 +16,10 @@ # specific language governing permissions and limitations # under the License. -set -e -set -u -set -o pipefail +set -euxo pipefail + +apt-get update +apt-get install -y gnupg echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\ >> /etc/apt/sources.list.d/llvm.list @@ -46,4 +47,4 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main\ >> /etc/apt/sources.list.d/llvm.list wget -q -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - -apt-get update && apt-get install -y llvm-4.0 llvm-9 llvm-8 llvm-7 clang-9 libclang-9-dev clang-8 libclang-8-dev clang-7 libclang-7-dev +apt-get install -y llvm-4.0 llvm-9 llvm-8 llvm-7 clang-9 libclang-9-dev clang-8 libclang-8-dev clang-7 libclang-7-dev From c7b5b4a6731af897e8e60c637a831d89bebf45c5 Mon Sep 17 00:00:00 2001 From: driazati Date: Tue, 15 Feb 2022 12:53:51 -0800 Subject: [PATCH 2/2] Address comments: pip hash, cleanup --- docker/install/ubuntu1804_install_python.sh | 22 +++++++++++++++++---- docker/install/ubuntu_install_boost.sh | 10 +++++++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docker/install/ubuntu1804_install_python.sh b/docker/install/ubuntu1804_install_python.sh index f0af141133d6..4676ffb8b20d 100755 --- a/docker/install/ubuntu1804_install_python.sh +++ b/docker/install/ubuntu1804_install_python.sh @@ -20,13 +20,27 @@ set -e set -u set -o pipefail -# install python and pip, don't modify this, modify install_python_package.sh + +cleanup() { + rm -rf base-requirements.txt +} + +trap cleanup 0 + + +# Install python and pip. Don't modify this to add Python package dependencies, +# instead modify install_python_package.sh apt-get update apt-get install -y software-properties-common apt-get install -y python3.7 python3.7-dev python3-pip - -# Install pip update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 # Pin pip and setuptools versions -pip3 install pip==19.3.1 setuptools==58.4.0 +# Hashes generated via: +# $ pip download == +# $ pip hash --algorithm sha512 .whl +cat < base-requirements.txt +pip==22.0.3 --hash=sha512:12ca75130a1ce9807060a66dd2341afc7c7e663357ca4b937868dbc733634e11bae49ffff96acb0f5f3fb16cb14f680b9b6d185155a711c6098eda5cfbf2f8f5 +setuptools==60.9.1 --hash=sha512:24c21006f0650209e6a934a5366614e32a98fbdf11bc0e941419731aa3c8cb6d217e486608b4834b8f89c14dc36b3bafa30e795c87b58705be2b156f78acc3c5 +EOF +pip3 install -r base-requirements.txt diff --git a/docker/install/ubuntu_install_boost.sh b/docker/install/ubuntu_install_boost.sh index ca56772a9b09..7af9bf46f496 100644 --- a/docker/install/ubuntu_install_boost.sh +++ b/docker/install/ubuntu_install_boost.sh @@ -18,7 +18,15 @@ set -euxo pipefail -curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.gz +cleanup() { + rm -rf /boost_1_67_0 /boost_1_67_0.tar.gz +} + +trap cleanup 0 + +curl -O https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.gz +BOOST_HASH=cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e +echo "$BOOST_HASH" boost_1_67_0.tar.gz | sha512sum -c tar -xf boost_1_67_0.tar.gz pushd boost_1_67_0