From 25a1698d45377efb47ab9280e5e2ae8b7126c652 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 20 Nov 2023 20:04:16 -0800 Subject: [PATCH 1/3] [SPARK-46020][INFRA] Add `Python 3.12` to Infra docker image --- dev/infra/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 141c079f3938e..2a03291cf8050 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -127,3 +127,15 @@ RUN python3.11 -m pip install 'grpcio>=1.48,<1.57' 'grpcio-status>=1.48,<1.57' ' RUN python3.11 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.11 -m pip install torcheval RUN python3.11 -m pip install deepspeed + +# Install Python 3.12 at the last stage to avoid breaking the existing Python installations +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y \ + python3.12 python3.12-distutils \ + && rm -rf /var/lib/apt/lists/* +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 +RUN python3.12 -m pip install numpy 'pyarrow>=14.0.0' 'pandas<=2.1.3' scipy unittest-xml-reporting plotly>=4.8 'mlflow>=2.8.1' coverage matplotlib openpyxl 'memory-profiler==0.60.0' 'scikit-learn>=1.3.2' +RUN python3.12 -m pip install 'grpcio>=1.48,<1.57' 'grpcio-status>=1.48,<1.57' 'protobuf==4.25.1' 'googleapis-common-protos==1.56.4' +RUN python3.12 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu +RUN python3.12 -m pip install torcheval +RUN python3.12 -m pip install deepspeed From 843d4c52ab4b6889a28a320be07ab5d31734c810 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 20 Nov 2023 20:24:06 -0800 Subject: [PATCH 2/3] Remove memory-profiler --- dev/infra/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 2a03291cf8050..48df56b16c220 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -134,7 +134,7 @@ RUN apt-get update && apt-get install -y \ python3.12 python3.12-distutils \ && rm -rf /var/lib/apt/lists/* RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 -RUN python3.12 -m pip install numpy 'pyarrow>=14.0.0' 'pandas<=2.1.3' scipy unittest-xml-reporting plotly>=4.8 'mlflow>=2.8.1' coverage matplotlib openpyxl 'memory-profiler==0.60.0' 'scikit-learn>=1.3.2' +RUN python3.12 -m pip install numpy 'pyarrow>=14.0.0' 'pandas<=2.1.3' scipy unittest-xml-reporting plotly>=4.8 'mlflow>=2.8.1' coverage matplotlib openpyxl 'scikit-learn>=1.3.2' RUN python3.12 -m pip install 'grpcio>=1.48,<1.57' 'grpcio-status>=1.48,<1.57' 'protobuf==4.25.1' 'googleapis-common-protos==1.56.4' RUN python3.12 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.12 -m pip install torcheval From 4ed2789c45d1808f9d1b6e6a2f7dd112130a25ff Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 20 Nov 2023 20:33:01 -0800 Subject: [PATCH 3/3] Remove grpcio and torch and deepspeed --- dev/infra/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 48df56b16c220..5cf492ad86330 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -135,7 +135,4 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 RUN python3.12 -m pip install numpy 'pyarrow>=14.0.0' 'pandas<=2.1.3' scipy unittest-xml-reporting plotly>=4.8 'mlflow>=2.8.1' coverage matplotlib openpyxl 'scikit-learn>=1.3.2' -RUN python3.12 -m pip install 'grpcio>=1.48,<1.57' 'grpcio-status>=1.48,<1.57' 'protobuf==4.25.1' 'googleapis-common-protos==1.56.4' -RUN python3.12 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu -RUN python3.12 -m pip install torcheval -RUN python3.12 -m pip install deepspeed +RUN python3.12 -m pip install 'protobuf==4.25.1' 'googleapis-common-protos==1.56.4'