From 9b516c6235d9b0bac01490f340b1a12e6ff73edd Mon Sep 17 00:00:00 2001 From: Matt Boris Date: Wed, 20 Sep 2023 09:12:17 -0400 Subject: [PATCH] chore!: use `podman` as default build tool and remove `img` --- .taskcluster.yml | 48 ++------------------ Dockerfile | 3 -- Makefile | 4 +- README.md | 7 +-- taskboot/build.py | 7 +-- taskboot/cli.py | 4 +- taskboot/docker.py | 102 ------------------------------------------- tests/conftest.py | 10 ++--- tests/test_docker.py | 9 +--- 9 files changed, 18 insertions(+), 176 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index b6a7d75..3a3a93a 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -34,7 +34,7 @@ tasks: in: - taskId: {$eval: as_slugid("code_checks")} provisionerId: proj-relman - workerType: ci + workerType: generic-worker-ubuntu-22-04 created: {$fromNow: ''} deadline: {$fromNow: '1 hour'} payload: @@ -55,46 +55,6 @@ tasks: owner: bastien@mozilla.com source: https://github.com/mozilla/task-boot - - taskId: {$eval: as_slugid("docker_build")} - dependencies: - - {$eval: as_slugid("code_checks")} - provisionerId: proj-relman - workerType: ci - created: {$fromNow: ''} - deadline: {$fromNow: '1 hour'} - payload: - capabilities: - privileged: true - maxRunTime: 3600 - image: python:3.10 - env: - IMAGE: mozilla/taskboot - REGISTRY: registry.hub.docker.com - VERSION: "${tag}" - IMG_SHA256: cc9bf08794353ef57b400d32cd1065765253166b0a09fba360d927cfbd158088 - command: - - sh - - -lxce - - "apt-get -qq update && - apt-get -qq install -y zstd && - curl -fSL \"https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64\" -o \"/usr/local/bin/img\" && - echo \"$IMG_SHA256 /usr/local/bin/img\" | sha256sum -c - && chmod a+x \"/usr/local/bin/img\" && - git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot && - pip install --no-cache-dir --quiet . && - taskboot --target=/src build --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile" - artifacts: - public/taskboot/image.tar.zst: - expires: {$fromNow: '2 weeks'} - path: /image.tar.zst - type: file - scopes: - - docker-worker:capability:privileged - metadata: - name: TaskBoot docker build - description: Taskcluster boot utilities - build latest docker image - owner: bastien@mozilla.com - source: https://github.com/mozilla/task-boot - - taskId: {$eval: as_slugid("docker_build_podman")} dependencies: - {$eval: as_slugid("code_checks")} @@ -135,7 +95,7 @@ tasks: dependencies: - {$eval: as_slugid("docker_build_podman")} provisionerId: proj-relman - workerType: ci + workerType: generic-worker-ubuntu-22-04 created: {$fromNow: ''} deadline: {$fromNow: '1 hour'} payload: @@ -156,7 +116,7 @@ tasks: dependencies: - {$eval: as_slugid("docker_build")} provisionerId: proj-relman - workerType: ci + workerType: generic-worker-ubuntu-22-04 created: {$fromNow: ''} deadline: {$fromNow: '1 hour'} payload: @@ -190,7 +150,7 @@ tasks: dependencies: - {$eval: as_slugid("docker_push")} provisionerId: proj-relman - workerType: ci + workerType: generic-worker-ubuntu-22-04 created: {$fromNow: ''} deadline: {$fromNow: '1 hour'} payload: diff --git a/Dockerfile b/Dockerfile index fb683c1..5d065e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,6 @@ # syntax=docker/dockerfile:experimental FROM python:3.10-alpine -# Add img -RUN apk add --no-cache img --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing - # Setup other deps RUN apk add --no-cache git skopeo docker cargo podman cni-plugins fuse-overlayfs zstd \ && sed -i 's/^#mount_program/mount_program/' /etc/containers/storage.conf diff --git a/Makefile b/Makefile index 8e18e6d..6352955 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ TAG=mozilla/taskboot VERSION=$(shell cat $(ROOT_DIR)/VERSION) build: - img build -t $(TAG):latest -t $(TAG):$(VERSION) $(ROOT_DIR) + podman build -t $(TAG):latest -t $(TAG):$(VERSION) $(ROOT_DIR) publish: - img push $(TAG):latest + podman push $(TAG):latest diff --git a/README.md b/README.md index 8a8cdcd..9b02d41 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Features -------- * clone a git repo -* build a docker image in a Taskcluster task without `dind`, by using the excellent project [img](https://github.com/genuinetools/img/) +* build a docker image in a Taskcluster task without `dind`, by using [podman](https://podman.io/) * push that docker image to a Docker repo, reading credentials from a Taskcluster secret * build multiple docker images using a `docker-compose.yml` file * build/update a Taskcluster hook @@ -23,8 +23,3 @@ Documentation ------------- A more detailed documentation is available in this [project's wiki](https://github.com/mozilla/task-boot/wiki). - -System requirements -------------------- - -* `img` must be at least in version 0.5.2 to support multiple tags diff --git a/taskboot/build.py b/taskboot/build.py index 81811d1..0cafa03 100644 --- a/taskboot/build.py +++ b/taskboot/build.py @@ -15,7 +15,6 @@ from taskboot.config import Configuration from taskboot.docker import DinD from taskboot.docker import Docker -from taskboot.docker import Img from taskboot.docker import Podman from taskboot.docker import patch_dockerfile from taskboot.utils import retry @@ -52,9 +51,7 @@ def build_image(target, args): """ Build a docker image and allow save/push """ - if args.build_tool == "img": - build_tool = Img(cache=args.cache) - elif args.build_tool == "docker": + if args.build_tool == "docker": build_tool = Docker() elif args.build_tool == "podman": build_tool = Podman() @@ -112,7 +109,7 @@ def build_compose(target, args): Read a compose file and build each image described as buildable """ assert args.build_retries > 0, "Build retries must be a positive integer" - build_tool = Img(cache=args.cache) + build_tool = Podman(cache=args.cache) # Check the dockerfile is available in target composefile = target.check_path(args.composefile) diff --git a/taskboot/cli.py b/taskboot/cli.py index b8b8e8e..ea17278 100644 --- a/taskboot/cli.py +++ b/taskboot/cli.py @@ -100,8 +100,8 @@ def main() -> None: build.add_argument( "--build-tool", dest="build_tool", - choices=["img", "docker", "dind", "podman"], - default=os.environ.get("BUILD_TOOL") or "img", + choices=["docker", "dind", "podman"], + default=os.environ.get("BUILD_TOOL") or "podman", help="Tool to build docker images.", ) build.set_defaults(func=build_image) diff --git a/taskboot/docker.py b/taskboot/docker.py index b961cf2..ee88906 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -22,11 +22,6 @@ logger = logging.getLogger(__name__) -# docker.io/mozilla/taskboot:latest 172.3MiB 25 hours ago About an hour ago sha256:e339e39884d2a6f44b493e8f135e5275d0e47209b3f990b768228534944db6e7 -IMG_LS_REGEX = re.compile( - r"([\w\.]+)/(([\w\-_\.]+)/([\w\-_\.]+)):([\w\-_\.]+)\t+([\.\w]+)\t+([\w ]+)\t+([\w ]+)\t+sha256:(\w{64})" -) - IMG_NAME_REGEX = re.compile(r"(?P[\/\w\-\._]+):?(?P\S*)") # Taskcluster uses a really outdated version of Docker daemon API @@ -191,103 +186,6 @@ def push_archive(self, path, custom_tag=None): logger.info("Push successful") -class Img(Tool): - """ - Interface to the img tool, replacing docker daemon - """ - - def __init__(self, cache=None): - super().__init__("img") - - # Setup img state, using or creating a cache folder - if cache is not None: - self.state = os.path.join(os.path.realpath(cache), "img") - os.makedirs(self.state, exist_ok=True) - else: - self.state = tempfile.mkdtemp("-img") - logger.info("Docker state is using {}".format(self.state)) - - def login(self, registry, username, password): - """ - Login on remote registry - """ - cmd = [ - "login", - "--state", - self.state, - "--password-stdin", - "-u", - username, - registry, - ] - self.run(cmd, input=password.encode("utf-8")) - logger.info("Authenticated on {} as {}".format(registry, username)) - - def list_images(self): - """ - List images stored in current state - Parses the text output into usable dicts - """ - ls = self.run(["ls", "--state", self.state], stdout=subprocess.PIPE) - out = [] - for line in ls.stdout.splitlines()[1:]: - image = IMG_LS_REGEX.search(line.decode("utf-8")) - if image is not None: - out.append( - { - "registry": image.group(1), - "repository": image.group(2), - "tag": image.group(5), - "size": image.group(6), - "created": image.group(7), - "updated": image.group(8), - "digest": image.group(9), - } - ) - else: - logger.warn("Did not parse this image: {}".format(line)) - return out - - def build(self, context_dir, dockerfile, tags, build_args=[]): - logger.info("Building docker image {}".format(dockerfile)) - - command = ["build", "--state", self.state, "--no-console", "--file", dockerfile] - - for add_tag in tags: - command += ["--tag", add_tag] - - # We need to "de-parse" the build args - for single_build_arg in build_args: - command += ["--build-arg", single_build_arg] - - command.append(context_dir) - - logger.info("Running img command: {}".format(command)) - - self.run(command) - logger.info("Built image {}".format(", ".join(tags))) - - def save(self, tags, path): - assert isinstance(tags, list) - assert len(tags) > 0, "Missing tags to save" - - # First save the image using only one tag - # img does not support (yet) writing multiple tags - main_tag = tags[0] - logger.info("Saving image {} to {}".format(main_tag, path)) - self.run(["save", "--state", self.state, "--output", path, main_tag]) - - # Patch the produced image to add other tags - if len(tags) > 1: - manifest = read_manifest(path) - manifest[0]["RepoTags"] = tags - write_manifest(path, manifest) - - def push(self, tag): - logger.info("Pushing image {}".format(tag)) - self.run(["push", "--state", self.state, tag]) - - class DinD(Tool): """ Interface to the Docker In Docker Taskcluster feature diff --git a/tests/conftest.py b/tests/conftest.py index 5990cd1..ee40d80 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ import pytest -from taskboot.docker import Img +from taskboot.docker import Podman TESTS_DIR = os.path.realpath(os.path.dirname(__file__)) @@ -16,17 +16,17 @@ @pytest.fixture def mock_docker(tmpdir): """ - Mock the Docker tool class (img) with a fake state + Mock the Docker tool class (podman) with a fake state """ - class MockDocker(Img): + class MockDocker(Podman): def __init__(self): self.state = None self.images = [] def run(self, command, **kwargs): - # Fake img calls - if command[0] == "ls": + # Fake podman calls + if command[0] == "images": # Fake image listing output = "Headers\n" # will be skipped by parser output += "\n".join(["\t".join(image) for image in self.images]) diff --git a/tests/test_docker.py b/tests/test_docker.py index 35e1947..fc6062e 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -129,10 +129,8 @@ def test_list_local_images(mock_docker): mock_docker.images = [ ( - "registry.com/repo/test:latest", - "10.9MiB", - "1 days ago", - "1 days ago", + "registry.com/repo/test", + "latest", "sha256:991d19e5156799aa79cf7138b8b843601f180e68f625b892df40a1993b7ac7da", ) ] @@ -141,9 +139,6 @@ def test_list_local_images(mock_docker): "registry": "registry.com", "repository": "repo/test", "tag": "latest", - "size": "10.9MiB", - "created": "1 days ago", - "updated": "1 days ago", "digest": "991d19e5156799aa79cf7138b8b843601f180e68f625b892df40a1993b7ac7da", } ]