Skip to content

[build][test] upgrade pyyaml to v5.4.1 for n-1 framework version #834

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tejaschumbalkar wants to merge 1 commit into from
Closed

[build][test] upgrade pyyaml to v5.4.1 for n-1 framework version #834

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 9 additions & 37 deletions mxnet/buildspec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
Copy link
Copy Markdown
Author

@tejaschumbalkar tejaschumbalkar Feb 15, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Temp buildspec changes to build specific framework versions.

region: &REGION <set-$REGION-in-environment>
framework: &FRAMEWORK mxnet
version: &VERSION 1.8.0
version: &VERSION 1.7.0
cuda_version: &CUDA_VERSION cu101
os_version: &OS_VERSION ubuntu16.04

repository_info:
training_repository: &TRAINING_REPOSITORY
Expand Down Expand Up @@ -33,63 +35,33 @@
BuildMXNetCPUTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &MXNET_CPU_TRAINING_PY3 false
image_size_baseline: 1786
image_size_baseline: 4899
device_type: &DEVICE_TYPE cpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
os_version: &OS_VERSION ubuntu16.04
tag_python_version: &TAG_PYTHON_VERSION py36
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildMXNetGPUTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &MXNET_GPU_TRAINING_PY3 false
image_size_baseline: 6581
image_size_baseline: 4899
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu16.04
tag_python_version: &TAG_PYTHON_VERSION py36
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildMXNetCPUInferencePy3DockerImage:
<<: *INFERENCE_REPOSITORY
build: &MXNET_CPU_INFERENCE_PY3 false
image_size_baseline: 1687
device_type: &DEVICE_TYPE cpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
os_version: &OS_VERSION ubuntu16.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *INFERENCE_CONTEXT
BuildMXNetGPUInferencePy3DockerImage:
<< : *INFERENCE_REPOSITORY
build: &MXNET_GPU_INFERENCE_PY3 false
image_size_baseline: 5578
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu16.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *INFERENCE_CONTEXT
BuildMXNetExampleGPUTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &MXNET_GPU_TRAINING_PY3 false
image_size_baseline: 6742
image_size_baseline: 5000
base_image_name: BuildMXNetGPUTrainPy3DockerImage
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu16.04
tag_python_version: &TAG_PYTHON_VERSION py36
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION,
"-example" ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /example, /Dockerfile., *DEVICE_TYPE ]
Expand Down
4 changes: 2 additions & 2 deletions mxnet/training/docker/1.7.0/py3/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ RUN ${PIP} --no-cache-dir install --upgrade \

WORKDIR /

# install PyYAML==5.1.2 to avoid conflict with latest awscli
# install PyYAML==5.4.1 to avoid conflict with latest awscli
# python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
RUN ${PIP} install --no-cache --upgrade \
keras-mxnet==2.2.4.2 \
Expand All @@ -137,7 +137,7 @@ RUN ${PIP} install --no-cache --upgrade \
python-dateutil==2.8.0 \
tqdm==4.39.0 \
sagemaker-experiments==0.* \
PyYAML==5.3.1 \
PyYAML==5.4.1 \
mpi4py==3.0.2 \
"sagemaker-mxnet-training<5" \
${MX_URL} \
Expand Down
4 changes: 2 additions & 2 deletions mxnet/training/docker/1.7.0/py3/cu101/Dockerfile.gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ RUN ${PIP} --no-cache-dir install --upgrade \

WORKDIR /

# install PyYAML==5.1.2 to avoid conflict with latest awscli
# install PyYAML==5.4.1 to avoid conflict with latest awscli
# python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
RUN ${PIP} install --no-cache --upgrade \
keras-mxnet==2.2.4.2 \
Expand All @@ -179,7 +179,7 @@ RUN ${PIP} install --no-cache --upgrade \
python-dateutil==2.8.0 \
tqdm==4.39.0 \
sagemaker-experiments==0.* \
PyYAML==5.3.1 \
PyYAML==5.4.1 \
mpi4py==3.0.2 \
"sagemaker-mxnet-training<5" \
${MX_URL} \
Expand Down
50 changes: 4 additions & 46 deletions pytorch/buildspec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
region: &REGION <set-$REGION-in-environment>
framework: &FRAMEWORK pytorch
version: &VERSION 1.7.1
version: &VERSION 1.6.0
cuda_version: &CUDA_VERSION cu101
os_version: &OS_VERSION ubuntu16.04

repository_info:
training_repository: &TRAINING_REPOSITORY
Expand Down Expand Up @@ -35,68 +37,24 @@
target: config.properties

images:
BuildCPUPTTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &PYTORCH_CPU_TRAINING_PY3 false
image_size_baseline: 4899
device_type: &DEVICE_TYPE cpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py36
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildGPUPTTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &PYTORCH_GPU_TRAINING_PY3 false
image_size_baseline: 14000
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py36
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile.,
*DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildPyTorchExampleGPUTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &PYTORCH_GPU_TRAINING_PY3 false
image_size_baseline: 14500
base_image_name: BuildGPUPTTrainPy3DockerImage
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py36
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION,
"-example" ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /example, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildCPUPTInferencePy3DockerImage:
<<: *INFERENCE_REPOSITORY
build: &PYTORCH_CPU_INFERENCE_PY3 false
image_size_baseline: 4899
device_type: &DEVICE_TYPE cpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py36
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *INFERENCE_CONTEXT
BuildGPUPTInferencePy3DockerImage:
<<: *INFERENCE_REPOSITORY
build: &PYTORCH_GPU_INFERENCE_PY3 false
image_size_baseline: 14000
image_size_baseline: 9228
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py36
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
Expand Down
2 changes: 1 addition & 1 deletion pytorch/inference/docker/1.6.0/py3/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ RUN pip install --no-cache-dir "sagemaker-pytorch-inference>=2"

RUN curl https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.6.0/license.txt -o /license.txt

RUN conda install -y -c conda-forge pyyaml==5.3.1
RUN conda install -y -c conda-forge pyyaml==5.4.1
RUN pip install pillow==7.2.0 "awscli<2"

EXPOSE 8080 8081
Expand Down
2 changes: 1 addition & 1 deletion pytorch/inference/docker/1.6.0/py3/cu101/Dockerfile.gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ RUN pip install --no-cache-dir "sagemaker-pytorch-inference>=2"

RUN curl https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.6.0/license.txt -o /license.txt

RUN conda install -y -c conda-forge pyyaml==5.3.1
RUN conda install -y -c conda-forge pyyaml==5.4.1
RUN pip install pillow==7.2.0 "awscli<2"

EXPOSE 8080 8081
Expand Down
67 changes: 18 additions & 49 deletions tensorflow/buildspec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
region: &REGION <set-$REGION-in-environment>
framework: &FRAMEWORK tensorflow
version: &VERSION 2.4.1
short_version: &SHORT_VERSION 2.4
version: &VERSION 2.3.1
short_version: &SHORT_VERSION 2.3

repository_info:
training_repository: &TRAINING_REPOSITORY
Expand Down Expand Up @@ -33,74 +33,43 @@
target: dockerd-entrypoint.py

images:
BuildTensorflowCpuPy37TrainingDockerImage:
BuildTensorflowCPUTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &TENSORFLOW_CPU_TRAINING_PY3 false
image_size_baseline: &IMAGE_SIZE_BASELINE 4489
image_size_baseline: 4899
device_type: &DEVICE_TYPE cpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION
]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile.,
*DEVICE_TYPE ]
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildTensorflowGpuPy37Cu110TrainingDockerImage:
BuildTensorflowGPUTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &TENSORFLOW_GPU_TRAINING_PY3 false
image_size_baseline: &IMAGE_SIZE_BASELINE 7738
image_size_baseline: 11200
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
cuda_version: &CUDA_VERSION cu110
cuda_version: &CUDA_VERSION cu102
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION,
"-", *OS_VERSION ]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION,
/Dockerfile., *DEVICE_TYPE ]
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildTensorflowExampleGpuPy37Cu110TrainingDockerImage:
BuildTensorflowExampleGPUTrainPy3DockerImage:
<<: *TRAINING_REPOSITORY
build: &TENSORFLOW_GPU_TRAINING_PY3 false
image_size_baseline: &IMAGE_SIZE_BASELINE 7738
base_image_name: BuildTensorflowGpuPy37Cu110TrainingDockerImage
image_size_baseline: 11200
base_image_name: BuildTensorflowGPUTrainPy3DockerImage
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
cuda_version: &CUDA_VERSION cu110
cuda_version: &CUDA_VERSION cu102
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION,
"-", *OS_VERSION, "-example" ]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /example,
/Dockerfile., *DEVICE_TYPE ]
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION,
"-example" ]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /example, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *TRAINING_CONTEXT
BuildTensorflowCPUInferencePy3DockerImage:
<<: *INFERENCE_REPOSITORY
build: &TENSORFLOW_CPU_INFERENCE_PY3 false
image_size_baseline: 4899
device_type: &DEVICE_TYPE cpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *INFERENCE_CONTEXT
BuildTensorflowGPUInferencePy3DockerImage:
<<: *INFERENCE_REPOSITORY
build: &TENSORFLOW_GPU_INFERENCE_PY3 false
image_size_baseline: 7738
device_type: &DEVICE_TYPE gpu
python_version: &DOCKER_PYTHON_VERSION py3
tag_python_version: &TAG_PYTHON_VERSION py37
cuda_version: &CUDA_VERSION cu110
os_version: &OS_VERSION ubuntu18.04
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]
context:
<<: *INFERENCE_CONTEXT
4 changes: 2 additions & 2 deletions tensorflow/training/docker/2.3/py3/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ RUN ln -s /etc/ssl/certs/*.* /usr/local/ssl/certs/
RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
&& ln -s $(which ${PIP}) /usr/bin/pip

# install PyYAML==5.1.2 to avoid conflict with latest awscli
# install PyYAML==5.4.1 to avoid conflict with latest awscli
# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
RUN ${PIP} install --no-cache-dir -U \
numpy==1.19.1 \
Expand All @@ -143,7 +143,7 @@ RUN ${PIP} install --no-cache-dir -U \
pandas==1.1 \
Pillow==7.2.0 \
python-dateutil==2.8.1 \
pyYAML==5.3.1 \
pyYAML==5.4.1 \
requests==2.24.0 \
"awscli<2" \
mpi4py==3.0.3 \
Expand Down
4 changes: 2 additions & 2 deletions tensorflow/training/docker/2.3/py3/cu102/Dockerfile.gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ RUN ln -s /etc/ssl/certs/*.* /usr/local/ssl/certs/
RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
&& ln -s $(which ${PIP}) /usr/bin/pip

# install PyYAML==5.1.2 to avoid conflict with latest awscli
# install PyYAML==5.4.1 to avoid conflict with latest awscli
# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
RUN ${PIP} install --no-cache-dir -U \
cmake==3.18.2.post1 \
Expand All @@ -192,7 +192,7 @@ RUN ${PIP} install --no-cache-dir -U \
pandas==1.1 \
Pillow==7.2.0 \
python-dateutil==2.8.1 \
pyYAML==5.3.1 \
pyYAML==5.4.1 \
requests==2.24.0 \
"awscli<2" \
mpi4py==3.0.3 \
Expand Down
4 changes: 2 additions & 2 deletions tensorflow/training/docker/2.3/py3/cu110/Dockerfile.gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ RUN ln -s /etc/ssl/certs/*.* /usr/local/ssl/certs/
RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
&& ln -s $(which ${PIP}) /usr/bin/pip

# install PyYAML==5.1.2 to avoid conflict with latest awscli
# install PyYAML==5.4.1 to avoid conflict with latest awscli
# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
RUN ${PIP} install --no-cache-dir -U \
pybind11 \
Expand All @@ -199,7 +199,7 @@ RUN ${PIP} install --no-cache-dir -U \
pandas==1.1 \
Pillow==7.2.0 \
python-dateutil==2.8.1 \
pyYAML==5.3.1 \
pyYAML==5.4.1 \
requests==2.24.0 \
"awscli<2" \
mpi4py==3.0.3 \
Expand Down