From 8b804bd77aad0f8a97d596c5b0b79e20fcf9e251 Mon Sep 17 00:00:00 2001 From: Danny Mccormick Date: Tue, 15 Jul 2025 10:49:47 -0400 Subject: [PATCH 1/2] Start pushing ml containers --- sdks/python/container/build.gradle | 1 + sdks/python/container/common.gradle | 2 + sdks/python/container/ml/build.gradle | 64 +++++++++ sdks/python/container/ml/common.gradle | 126 ++++++++++++++++++ .../py310/base_image_requirements.txt} | 0 sdks/python/container/ml/py310/build.gradle | 28 ++++ .../py311/base_image_requirements.txt} | 0 sdks/python/container/ml/py311/build.gradle | 28 ++++ .../py312/base_image_requirements.txt} | 0 sdks/python/container/ml/py312/build.gradle | 28 ++++ sdks/python/container/ml/py313/build.gradle | 28 ++++ .../py39/base_image_requirements.txt} | 0 sdks/python/container/ml/py39/build.gradle | 28 ++++ .../container/run_generate_requirements.sh | 9 +- 14 files changed, 340 insertions(+), 2 deletions(-) create mode 100644 sdks/python/container/ml/build.gradle create mode 100644 sdks/python/container/ml/common.gradle rename sdks/python/container/{py310/ml_image_requirements.txt => ml/py310/base_image_requirements.txt} (100%) create mode 100644 sdks/python/container/ml/py310/build.gradle rename sdks/python/container/{py311/ml_image_requirements.txt => ml/py311/base_image_requirements.txt} (100%) create mode 100644 sdks/python/container/ml/py311/build.gradle rename sdks/python/container/{py312/ml_image_requirements.txt => ml/py312/base_image_requirements.txt} (100%) create mode 100644 sdks/python/container/ml/py312/build.gradle create mode 100644 sdks/python/container/ml/py313/build.gradle rename sdks/python/container/{py39/ml_image_requirements.txt => ml/py39/base_image_requirements.txt} (100%) create mode 100644 sdks/python/container/ml/py39/build.gradle diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index f4de804b80b7..79cd0624b6d3 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -71,6 +71,7 @@ for(int i=min_python_version; i<=max_python_version; ++i) { tasks.register("pushAll") { dependsOn ':sdks:python:container:distroless:pushAll' + dependsOn ':sdks:python:container:ml:pushAll' for(int ver=min_python_version; ver<=max_python_version; ++ver) { if (!project.hasProperty("skip-python-3" + ver + "-images")) { dependsOn ':sdks:python:container:push3' + ver diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 6c3de38d56c1..6471971147fa 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -41,6 +41,7 @@ def generatePythonRequirements = tasks.register("generatePythonRequirements") { "${project.ext.pythonVersion} " + "${files(configurations.sdkSourceTarball.files).singleFile} " + "base_image_requirements.txt " + + "container" + "[gcp,dataframe,test] " + "${pipExtraOptions}" } @@ -51,6 +52,7 @@ def generatePythonRequirements = tasks.register("generatePythonRequirements") { "${project.ext.pythonVersion} " + "${files(configurations.sdkSourceTarball.files).singleFile} " + "ml_image_requirements.txt " + + "container/ml" + "[gcp,dataframe,test,tensorflow,torch,transformers] " + "${pipExtraOptions}" } diff --git a/sdks/python/container/ml/build.gradle b/sdks/python/container/ml/build.gradle new file mode 100644 index 000000000000..40d37274f24c --- /dev/null +++ b/sdks/python/container/ml/build.gradle @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } + +description = "Apache Beam :: SDKs :: Python :: Container :: ML" +// Keep these values in sync with sdks/python/container/build.gradle. +int min_python_version=9 +int max_python_version=12 + + +tasks.register("buildAll") { + for(int ver=min_python_version; ver<=max_python_version; ++ver) { + dependsOn ':sdks:python:container:ml:py3' + ver + ':docker' + } +} + +for(int i=min_python_version; i<=max_python_version; ++i) { + String min_version = "3" + min_python_version + String cur = "3" + i + String prev = "3" + (i-1) + tasks.register("push" + cur) { + if (cur != min_version) { + // Enforce ordering to allow the prune step to happen between runs. + // This will ensure we don't use up too much space (especially in CI environments) + if (!project.hasProperty("skip-python-3" + prev + "-images")) { + mustRunAfter(":sdks:python:container:ml:push" + prev) + } + } + dependsOn ':sdks:python:container:ml:py' + cur + ':docker' + + doLast { + if (project.hasProperty("prune-images")) { + exec { + executable("docker") + args("system", "prune", "-a", "--force") + } + } + } + } +} + +tasks.register("pushAll") { + for(int ver=min_python_version; ver<=max_python_version; ++ver) { + if (!project.hasProperty("skip-python-3" + ver + "-images")) { + dependsOn ':sdks:python:container:ml:push3' + ver + } + } +} diff --git a/sdks/python/container/ml/common.gradle b/sdks/python/container/ml/common.gradle new file mode 100644 index 000000000000..dff2b3fc7f97 --- /dev/null +++ b/sdks/python/container/ml/common.gradle @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') + +description = "Apache Beam :: SDKs :: Python :: Container :: ML :: Python ${pythonVersionSuffix} Container" + +configurations { + sdkSourceTarball + pythonHarnessLauncher +} + +dependencies { + sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") + pythonHarnessLauncher project(path: ":sdks:python:container", configuration: "pythonHarnessLauncher") +} + +def generatePythonRequirements = tasks.register("generatePythonRequirements") { + dependsOn ':sdks:python:sdist' + def pipExtraOptions = project.hasProperty("testRCDependencies") ? "--pre" : "" + def runScriptsPath = "${rootDir}/sdks/python/container/run_generate_requirements.sh" + doLast { + exec { + executable 'sh' + args '-c', "cd ${rootDir} && ${runScriptsPath} " + + "${project.ext.pythonVersion} " + + "${files(configurations.sdkSourceTarball.files).singleFile} " + + "base_image_requirements.txt " + + "[gcp,dataframe,test] " + + "${pipExtraOptions}" + } + // Generate versions for ML dependencies + exec { + executable 'sh' + args '-c', "cd ${rootDir} && ${runScriptsPath} " + + "${project.ext.pythonVersion} " + + "${files(configurations.sdkSourceTarball.files).singleFile} " + + "ml_image_requirements.txt " + + "[gcp,dataframe,test,tensorflow,torch,transformers] " + + "${pipExtraOptions}" + } + } +} + +def copyDockerfileDependencies = tasks.register("copyDockerfileDependencies", Copy) { + from configurations.sdkSourceTarball + from file("base_image_requirements.txt") + into "build/target" + if(configurations.sdkSourceTarball.isEmpty()) { + throw new StopExecutionException(); + } +} + +def copyLicenseScripts = tasks.register("copyLicenseScripts", Copy){ + from ("../license_scripts") + into "build/target/license_scripts" +} + +def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) { + from configurations.pythonHarnessLauncher + into "build/target/launcher" + + // Avoid seemingly gradle bug stated in https://github.com/apache/beam/issues/29220 + mustRunAfter "copyLicenses" + + if(configurations.pythonHarnessLauncher.isEmpty()) { + throw new StopExecutionException(); + } +} + +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + +docker { + name containerImageName( + name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk_ml", + root: project.rootProject.hasProperty(["docker-repository-root"]) ? + project.rootProject["docker-repository-root"] : + project.docker_image_default_repo_root, + tag: project.rootProject.hasProperty(["docker-tag"]) ? + project.rootProject["docker-tag"] : project.sdk_version) + // tags used by dockerTag task + tags containerImageTags() + files "../../Dockerfile", "./build" + buildArgs(['py_version': "${project.ext.pythonVersion}", + 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || + project.rootProject.hasProperty(["isRelease"])]) + buildx project.useBuildx() + platform(*project.containerPlatforms()) + load project.useBuildx() && !pushContainers + push pushContainers +} + +dockerPrepare.dependsOn copyLauncherDependencies +dockerPrepare.dependsOn copyDockerfileDependencies +dockerPrepare.dependsOn copyLicenseScripts + +if (project.rootProject.hasProperty("docker-pull-licenses")) { + def copyGolangLicenses = tasks.register("copyGolangLicenses", Copy) { + from "${project(':release:go-licenses:py').buildDir}/output" + into "build/target/go-licenses" + dependsOn ':release:go-licenses:py:createLicenses' + } + dockerPrepare.dependsOn copyGolangLicenses +} else { + def skipPullLicenses = tasks.register("skipPullLicenses", Exec) { + executable "sh" + // Touch a dummy file to ensure the directory exists. + args "-c", "mkdir -p build/target/go-licenses && touch build/target/go-licenses/skip" + } + dockerPrepare.dependsOn skipPullLicenses +} diff --git a/sdks/python/container/py310/ml_image_requirements.txt b/sdks/python/container/ml/py310/base_image_requirements.txt similarity index 100% rename from sdks/python/container/py310/ml_image_requirements.txt rename to sdks/python/container/ml/py310/base_image_requirements.txt diff --git a/sdks/python/container/ml/py310/build.gradle b/sdks/python/container/ml/py310/build.gradle new file mode 100644 index 000000000000..28894725165a --- /dev/null +++ b/sdks/python/container/ml/py310/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'base' + id 'org.apache.beam.module' +} +applyDockerNature() +applyPythonNature() + +pythonVersion = '3.10' + +apply from: "../common.gradle" diff --git a/sdks/python/container/py311/ml_image_requirements.txt b/sdks/python/container/ml/py311/base_image_requirements.txt similarity index 100% rename from sdks/python/container/py311/ml_image_requirements.txt rename to sdks/python/container/ml/py311/base_image_requirements.txt diff --git a/sdks/python/container/ml/py311/build.gradle b/sdks/python/container/ml/py311/build.gradle new file mode 100644 index 000000000000..24ccc85c6b25 --- /dev/null +++ b/sdks/python/container/ml/py311/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'base' + id 'org.apache.beam.module' +} +applyDockerNature() +applyPythonNature() + +pythonVersion = '3.11' + +apply from: "../common.gradle" diff --git a/sdks/python/container/py312/ml_image_requirements.txt b/sdks/python/container/ml/py312/base_image_requirements.txt similarity index 100% rename from sdks/python/container/py312/ml_image_requirements.txt rename to sdks/python/container/ml/py312/base_image_requirements.txt diff --git a/sdks/python/container/ml/py312/build.gradle b/sdks/python/container/ml/py312/build.gradle new file mode 100644 index 000000000000..d4009f12a489 --- /dev/null +++ b/sdks/python/container/ml/py312/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'base' + id 'org.apache.beam.module' +} +applyDockerNature() +applyPythonNature() + +pythonVersion = '3.12' + +apply from: "../common.gradle" diff --git a/sdks/python/container/ml/py313/build.gradle b/sdks/python/container/ml/py313/build.gradle new file mode 100644 index 000000000000..4552ba0ef62d --- /dev/null +++ b/sdks/python/container/ml/py313/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'base' + id 'org.apache.beam.module' +} +applyDockerNature() +applyPythonNature() + +pythonVersion = '3.13' + +apply from: "../common.gradle" diff --git a/sdks/python/container/py39/ml_image_requirements.txt b/sdks/python/container/ml/py39/base_image_requirements.txt similarity index 100% rename from sdks/python/container/py39/ml_image_requirements.txt rename to sdks/python/container/ml/py39/base_image_requirements.txt diff --git a/sdks/python/container/ml/py39/build.gradle b/sdks/python/container/ml/py39/build.gradle new file mode 100644 index 000000000000..c5f55ae53af7 --- /dev/null +++ b/sdks/python/container/ml/py39/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'base' + id 'org.apache.beam.module' +} +applyDockerNature() +applyPythonNature() + +pythonVersion = '3.9' + +apply from: "../common.gradle" diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index 02ff9d2ccd6d..ce4392818645 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -39,7 +39,8 @@ fi PY_VERSION=$1 SDK_TARBALL=$2 REQUIREMENTS_FILE_NAME=$3 -EXTRAS=$4 +BASE_PATH=$4 +EXTRAS=$5 # Use the PIP_EXTRA_OPTIONS environment variable to pass additional flags to the pip install command. # For example, you can include the --pre flag in $PIP_EXTRA_OPTIONS to download pre-release versions of packages. # Note that you can modify the behavior of the pip install command in this script by passing in your own $PIP_EXTRA_OPTIONS. @@ -59,6 +60,10 @@ if [ -z "$REQUIREMENTS_FILE_NAME" ]; then REQUIREMENTS_FILE_NAME="base_image_requirements.txt" fi +if [ -z "$BASE_PATH" ]; then + BASE_PATH="container" +fi + if [ -z "$EXTRAS" ]; then EXTRAS="[gcp,dataframe,test]" fi @@ -85,7 +90,7 @@ echo "Installed dependencies:" pip freeze --all PY_IMAGE="py${PY_VERSION//.}" -REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/$REQUIREMENTS_FILE_NAME +REQUIREMENTS_FILE=$PWD/sdks/python/$BASE_PATH/$PY_IMAGE/$REQUIREMENTS_FILE_NAME cat < "$REQUIREMENTS_FILE" # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with From c00b39d2f5bf9a4a21a9afa5834804ee505c5870 Mon Sep 17 00:00:00 2001 From: Danny Mccormick Date: Mon, 25 Aug 2025 14:43:37 -0400 Subject: [PATCH 2/2] Consolidate python version values --- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 4 ++++ sdks/python/container/build.gradle | 6 +++--- sdks/python/container/distroless/build.gradle | 6 +++--- sdks/python/container/ml/build.gradle | 6 +++--- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index a7f2b99b9a1f..bfafa44b7491 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -3030,6 +3030,10 @@ class BeamModulePlugin implements Plugin { project.ext.pythonVersion = project.hasProperty('pythonVersion') ? project.pythonVersion : '3.9' + // Set min/max python versions used for containers and supported versions. + project.ext.minPythonVersion = 9 + project.ext.maxPythonVersion = 13 + def setupVirtualenv = project.tasks.register('setupVirtualenv') { doLast { def virtualenvCmd = [ diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index 79cd0624b6d3..fe7bda553176 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -18,11 +18,11 @@ plugins { id 'org.apache.beam.module' } applyGoNature() +applyPythonNature() description = "Apache Beam :: SDKs :: Python :: Container" -// Keep these values in sync with sdks/python/container/distroless/build.gradle. -int min_python_version=9 -int max_python_version=12 +int min_python_version=project.ext.minPythonVersion +int max_python_version=project.ext.maxPythonVersion configurations { sdkSourceTarball diff --git a/sdks/python/container/distroless/build.gradle b/sdks/python/container/distroless/build.gradle index 314484ade61a..48255b98db3f 100644 --- a/sdks/python/container/distroless/build.gradle +++ b/sdks/python/container/distroless/build.gradle @@ -17,11 +17,11 @@ */ plugins { id 'org.apache.beam.module' } +applyPythonNature() description = "Apache Beam :: SDKs :: Python :: Container :: Distroless" -// Keep these values in sync with sdks/python/container/build.gradle. -int min_python_version=9 -int max_python_version=12 +int min_python_version=project.ext.minPythonVersion +int max_python_version=project.ext.maxPythonVersion tasks.register("buildAll") { diff --git a/sdks/python/container/ml/build.gradle b/sdks/python/container/ml/build.gradle index 40d37274f24c..f09cc2e80203 100644 --- a/sdks/python/container/ml/build.gradle +++ b/sdks/python/container/ml/build.gradle @@ -17,11 +17,11 @@ */ plugins { id 'org.apache.beam.module' } +applyPythonNature() description = "Apache Beam :: SDKs :: Python :: Container :: ML" -// Keep these values in sync with sdks/python/container/build.gradle. -int min_python_version=9 -int max_python_version=12 +int min_python_version=project.ext.minPythonVersion +int max_python_version=project.ext.maxPythonVersion tasks.register("buildAll") {