From 7c2a8496dfd632fd9f1e5fefd39dc30f7cb840e0 Mon Sep 17 00:00:00 2001 From: driazati Date: Fri, 10 Jun 2022 13:14:59 -0700 Subject: [PATCH] [ci][docker] Automatically update docker images This activates the code from #11329 behind a couple Jenkins flags: * `USE_AUTOUPDATED_DOCKER_IMAGES` - whether or not to use the dynamically determined docker images in the build * `PUSH_DOCKER_IMAGES` - whether or not to push built images to `tlcpack` on `main` These are there so we can quickly toggle this behavior off it breaks anything. Once this is on though, any docker changes in a PR should be reflected entirely in that build, so there would be no need for a separate PR to update the Docker images. We will have to see if this works in practice, as the docker image builds do download quite a bit of data (which can flake) and add some runtime overhead (about 30m), so when an update lands all PRs will end up needing to rebuild until the merged commit finishes. --- .gitignore | 3 ++ Jenkinsfile | 75 ++++++++++++++++++++------ jenkins/Deploy.groovy.j2 | 2 +- jenkins/Prepare.groovy.j2 | 28 +++++++--- tests/scripts/should_rebuild_docker.py | 17 ++++-- 5 files changed, 97 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 184ff17ab25e..e9b9743f1359 100644 --- a/.gitignore +++ b/.gitignore @@ -265,3 +265,6 @@ gallery/how_to/work_with_microtvm/micro_tvmc.py # Test sample data files !tests/python/ci/sample_prs/*.json + +# Used in CI to communicate between Python and Jenkins +.docker-image-names/ diff --git a/Jenkinsfile b/Jenkinsfile index ad7771b81745..bc25ab093fd1 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-06-10T12:12:40.419262 +// Generated at 2022-06-14T11:01:40.694929 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -253,18 +253,6 @@ def prepare() { ci_qemu = params.ci_qemu_param ?: ci_qemu ci_wasm = params.ci_wasm_param ?: ci_wasm - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_arm = ${ci_arm}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_hexagon = ${ci_hexagon}" - echo " ci_i386 = ${ci_i386}" - echo " ci_lint = ${ci_lint}" - echo " ci_qemu = ${ci_qemu}" - echo " ci_wasm = ${ci_wasm}" - """, label: 'Docker image names') - is_docs_only_build = sh ( returnStatus: true, script: './tests/scripts/git_change_docs.sh', @@ -274,13 +262,70 @@ def prepare() { skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) rebuild_docker_images = sh ( returnStatus: true, - script: './tests/scripts/git_change_docker.sh', + script: './tests/scripts/should_rebuild_docker.py', label: 'Check for any docker changes', ) if (skip_ci) { // Don't rebuild when skipping CI rebuild_docker_images = false } + if (!rebuild_docker_images) { + // Pull image names from the results of should_rebuild_docker.py + if (env.USE_AUTOUPDATED_DOCKER_IMAGES == 'yes') { + ci_arm = sh( + script: "cat .docker-image.names/ci_arm", + label: "Find docker image name for ci_arm", + returnStdout: true, + ).trim() + ci_cpu = sh( + script: "cat .docker-image.names/ci_cpu", + label: "Find docker image name for ci_cpu", + returnStdout: true, + ).trim() + ci_gpu = sh( + script: "cat .docker-image.names/ci_gpu", + label: "Find docker image name for ci_gpu", + returnStdout: true, + ).trim() + ci_hexagon = sh( + script: "cat .docker-image.names/ci_hexagon", + label: "Find docker image name for ci_hexagon", + returnStdout: true, + ).trim() + ci_i386 = sh( + script: "cat .docker-image.names/ci_i386", + label: "Find docker image name for ci_i386", + returnStdout: true, + ).trim() + ci_lint = sh( + script: "cat .docker-image.names/ci_lint", + label: "Find docker image name for ci_lint", + returnStdout: true, + ).trim() + ci_qemu = sh( + script: "cat .docker-image.names/ci_qemu", + label: "Find docker image name for ci_qemu", + returnStdout: true, + ).trim() + ci_wasm = sh( + script: "cat .docker-image.names/ci_wasm", + label: "Find docker image name for ci_wasm", + returnStdout: true, + ).trim() + } + } + + sh (script: """ + echo "Docker images being used in this build:" + echo " ci_arm = ${ci_arm}" + echo " ci_cpu = ${ci_cpu}" + echo " ci_gpu = ${ci_gpu}" + echo " ci_hexagon = ${ci_hexagon}" + echo " ci_i386 = ${ci_i386}" + echo " ci_lint = ${ci_lint}" + echo " ci_qemu = ${ci_qemu}" + echo " ci_wasm = ${ci_wasm}" + """, label: 'Docker image names') } } } @@ -3377,7 +3422,7 @@ def deploy() { } } } - if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { + if (env.BRANCH_NAME == 'main' && env.PUSH_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { node('CPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docker") { try { diff --git a/jenkins/Deploy.groovy.j2 b/jenkins/Deploy.groovy.j2 index 0c81f8f4724a..f309d2356ed1 100644 --- a/jenkins/Deploy.groovy.j2 +++ b/jenkins/Deploy.groovy.j2 @@ -86,7 +86,7 @@ def deploy() { } } } - if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { + if (env.BRANCH_NAME == 'main' && env.PUSH_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { node('CPU') { ws({{ m.per_exec_ws('tvm/deploy-docker') }}) { try { diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2 index 894ddc72eeb7..51c0c65b36c7 100644 --- a/jenkins/Prepare.groovy.j2 +++ b/jenkins/Prepare.groovy.j2 @@ -145,13 +145,6 @@ def prepare() { {{ image.name }} = params.{{ image.name }}_param ?: {{ image.name }} {% endfor %} - sh (script: """ - echo "Docker images being used in this build:" - {% for image in images %} - echo " {{ image.name }} = ${ {{- image.name -}} }" - {% endfor %} - """, label: 'Docker image names') - is_docs_only_build = sh ( returnStatus: true, script: './tests/scripts/git_change_docs.sh', @@ -161,13 +154,32 @@ def prepare() { skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) rebuild_docker_images = sh ( returnStatus: true, - script: './tests/scripts/git_change_docker.sh', + script: './tests/scripts/should_rebuild_docker.py', label: 'Check for any docker changes', ) if (skip_ci) { // Don't rebuild when skipping CI rebuild_docker_images = false } + if (!rebuild_docker_images) { + // Pull image names from the results of should_rebuild_docker.py + if (env.USE_AUTOUPDATED_DOCKER_IMAGES == 'yes') { + {% for image in images %} + {{ image.name }} = sh( + script: "cat .docker-image.names/{{ image.name }}", + label: "Find docker image name for {{ image.name }}", + returnStdout: true, + ).trim() + {% endfor %} + } + } + + sh (script: """ + echo "Docker images being used in this build:" + {% for image in images %} + echo " {{ image.name }} = ${ {{- image.name -}} }" + {% endfor %} + """, label: 'Docker image names') } } } diff --git a/tests/scripts/should_rebuild_docker.py b/tests/scripts/should_rebuild_docker.py index dc12c38de830..5e0f5facb9b1 100755 --- a/tests/scripts/should_rebuild_docker.py +++ b/tests/scripts/should_rebuild_docker.py @@ -25,10 +25,11 @@ from http_utils import get -from cmd_utils import Sh, init_log +from cmd_utils import Sh, init_log, REPO_ROOT DOCKER_API_BASE = "https://hub.docker.com/v2/" +DOCKER_USER = "tlcpack" PAGE_SIZE = 25 TEST_DATA = None @@ -111,7 +112,9 @@ def find_commit_in_repo(tags: List[Dict[str, Any]]): def main(): # Fetch all tlcpack images - images = docker_api("repositories/tlcpack") + images = docker_api(f"repositories/{DOCKER_USER}") + NAMES_DIR = REPO_ROOT / ".docker-image-names" + NAMES_DIR.mkdir(exist_ok=True) # Ignore all non-ci images relevant_images = [image for image in images["results"] if image["name"].startswith("ci-")] @@ -120,7 +123,7 @@ def main(): for image in relevant_images: # Check the tags for the image - tags = docker_api(f"repositories/tlcpack/{image['name']}/tags") + tags = docker_api(f"repositories/{DOCKER_USER}/{image['name']}/tags") # Find the hash of the most recent tag shorthash, tag = find_commit_in_repo(tags) @@ -132,6 +135,11 @@ def main(): logging.info(f"Found docker changes from {shorthash} when checking {name}") logging.info(diff) exit(2) + else: + fullname = f"{DOCKER_USER}/{image['name']}:{name}" + logging.info(f"Using image {fullname} for {image}") + with open(NAMES_DIR / image["name"], "w") as f: + f.write(fullname) logging.info("Did not find changes, no rebuild necessary") exit(0) @@ -140,7 +148,8 @@ def main(): if __name__ == "__main__": init_log() parser = argparse.ArgumentParser( - description="Exits 0 if Docker images don't need to be rebuilt, 1 otherwise" + description="Exits 0 if Docker images don't need to be rebuilt, 1 otherwise. " + "If 0, image names will be written to a .docker-image-names folder" ) parser.add_argument( "--testing-docker-data",