diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f1a94e..7e84ee3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,11 +17,20 @@ jobs: steps: - uses: actions/checkout@v3 - run: docker build --pull --tag oisupport/perl-bashbrew . - - uses: actions/checkout@v3 - with: - repository: docker-library/official-images - ref: b851f53b765e622928a94663d425ddbe37aceb3b - path: oi - - run: echo "BASHBREW_LIBRARY=$PWD/oi/library" >> "$GITHUB_ENV" - - run: docker run -dit --name registry --restart always -p 5000:5000 registry - - run: ./test-localhost.sh + - name: Install "bashbrew" + run: | + git clone --depth 1 https://github.com/docker-library/bashbrew.git -b master bashbrew + ./bashbrew/bashbrew.sh --version > /dev/null + export PATH="$PWD/bashbrew/bin:$PATH" + echo "PATH=$PATH" >> "$GITHUB_ENV" + bashbrew --version + - name: Clone DOI + run: | + git clone --depth 1 https://github.com/docker-library/official-images.git oi + export BASHBREW_LIBRARY="$PWD/oi/library" + echo "BASHBREW_LIBRARY=$BASHBREW_LIBRARY" >> "$GITHUB_ENV" + bashbrew from hello-world:latest > /dev/null + - run: docker run -dit --name registry --restart always -p 5000:5000 --env REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW='["^.*$"]' --env REGISTRY_VALIDATION_MANIFESTS_URLS_DENY='[]' registry + - run: ./test-localhost.sh hello-world:latest # includes Windows images + - run: ./test-localhost.sh hello-world:nanoserver-ltsc2022 # forces an "os.version from the config blob" lookup + - run: ./test-localhost.sh busybox:latest # includes three separate "linux/arm" variants diff --git a/bin/put-multiarch.pl b/bin/put-multiarch.pl index 4a6d7ac..47df2fa 100755 --- a/bin/put-multiarch.pl +++ b/bin/put-multiarch.pl @@ -31,39 +31,106 @@ sub get_arch_p ($targetRef, $arch, $archRef) { return $ua->get_manifest_p($archRef)->then(sub ($manifestData = undef) { return unless $manifestData; - my ($digest, $manifest, $size) = ($manifestData->{digest}, $manifestData->{manifest}, $manifestData->{size}); + my ($mediaType, $digest, $size, $manifest) = ( + $manifestData->{mediaType}, + $manifestData->{digest}, + $manifestData->{size}, + $manifestData->{manifest}, + ); - my $mediaType = $manifestData->{mediaType}; - if ($mediaType eq Bashbrew::RegistryUserAgent::MEDIA_OCI_INDEX_V1 || $mediaType eq Bashbrew::RegistryUserAgent::MEDIA_MANIFEST_LIST) { - # jackpot -- if it's already a manifest list, the hard work is done! - return ($archRef, $manifest->{manifests}); - # TODO we should validate the "platform" values here to make sure we're not violating the security boundary + my @manifests; + if (Bashbrew::RegistryUserAgent::is_media_image_list($mediaType)) { + push @manifests, @{ $manifest->{manifests} }; } - if ($mediaType eq Bashbrew::RegistryUserAgent::MEDIA_OCI_MANIFEST_V1 || $mediaType eq Bashbrew::RegistryUserAgent::MEDIA_MANIFEST_V1 || $mediaType eq Bashbrew::RegistryUserAgent::MEDIA_MANIFEST_V2) { - my $manifestListItem = { + elsif (Bashbrew::RegistryUserAgent::is_media_image_manifest($mediaType)) { + push @manifests, { mediaType => $mediaType, size => $size, digest => $digest, - platform => { - arch_to_platform($arch), - ($manifest->{'os.version'} ? ('os.version' => $manifest->{'os.version'}) : ()), - }, }; - if ($manifestListItem->{platform}{os} eq 'windows' && !$manifestListItem->{platform}{'os.version'} && $mediaType eq Bashbrew::RegistryUserAgent::MEDIA_MANIFEST_V2) { - # if we're on Windows, we need to make an effort to fetch the "os.version" value from the config for the platform object - return $ua->get_blob_p($archRef->clone->digest($manifest->{config}{digest}))->then(sub ($config = undef) { - if ($config && $config->{'os.version'}) { - $manifestListItem->{platform}{'os.version'} = $config->{'os.version'}; - } - return ($archRef, [ $manifestListItem ]); - }); + } + else { + die "unknown mediaType '$mediaType' for '$archRef'"; + } + + # filter out objects we know we don't want (not a hashref, missing required fields) + @manifests = grep { + # https://specs.opencontainers.org/image-spec/descriptor/?v=v1.0.1 + 'HASH' eq ref($_) && $_->{mediaType} && $_->{digest} && $_->{size} + } @manifests; + + # filter objects down to just fields we care about + @manifests = map { + # https://specs.opencontainers.org/image-spec/descriptor/?v=v1.0.1 + { %{ $_ }{qw{ + mediaType + digest + size + annotations + platform + }} } + } @manifests; + + my %platform = arch_to_platform($arch); + + # normalize the result a bit (delete empty annotations and make sure platform is an object and that every platform has at least "os" and "architecture") + @manifests = map { + $_->{platform} //= {}; + for my $key (qw( os architecture )) { + $_->{platform}{$key} //= $platform{$key}; } - else { - return ($archRef, [ $manifestListItem ]); + delete $_->{annotations} unless defined $_->{annotations}; + $_ + } @manifests; + + # now that we have a list of potential manifests, let's filter it based on %platform's "os" and "architecture" (avoids "riscv64" from being able to poison us with anything other than riscv64-tagged manifests) + my @filteredManifests = grep { + Bashbrew::RegistryUserAgent::is_media_image_manifest($_->{mediaType}) + && $_->{platform}{os} eq $platform{os} + && $_->{platform}{architecture} eq $platform{architecture} + } @manifests; + # normalize "platform" objects (esp. for "variant") + for my $item (@filteredManifests) { + for my $key (keys %platform) { + $item->{platform}{$key} = $platform{$key}; } } - die "unknown mediaType '$mediaType' for '$archRef'"; + # include any relevant "Docker-style" attachments (https://github.com/moby/buildkit/pull/2983, https://github.com/moby/buildkit/pull/3129, etc) + my %digests = map { $_ => 1 } map { $_->{digest} } @filteredManifests; + push @filteredManifests, grep { + $_->{mediaType} eq Bashbrew::RegistryUserAgent::MEDIA_OCI_MANIFEST_V1 + && $_->{platform}{os} eq 'unknown' + && $_->{platform}{architecture} eq 'unknown' + && $_->{annotations} + && $digests{$_->{annotations}{'vnd.docker.reference.digest'} // ''} + && $_->{annotations}{'vnd.docker.reference.type'} + } @manifests; + + # if we're looking at Windows, we need to make an effort to fetch the "os.version" value from the config for the platform object + return Mojo::Promise->map({ concurrency => 3 }, sub ($item) { + unless ( + Bashbrew::RegistryUserAgent::is_media_image_manifest($item->{mediaType}) + && $item->{platform}{os} eq 'windows' + && !$item->{platform}{'os.version'} + ) { + return Mojo::Promise->resolve($item); + } + return $ua->get_manifest_p($archRef->clone->digest($item->{digest}))->then(sub ($manifestData = undef) { + return $item unless $manifestData; + my $manifest = $manifestData->{manifest}; + return $item unless $manifest->{config} and $manifest->{config}{digest}; + return $ua->get_blob_p($archRef->clone->digest($manifest->{config}{digest}))->then(sub ($config = undef) { + if ($config && $config->{'os.version'}) { + $item->{platform}{'os.version'} = $config->{'os.version'}; + } + return $item; + }); + }); + }, @filteredManifests)->then(sub (@manifests) { + @manifests = map { @$_ } @manifests; + return ($archRef, \@manifests); + }); }); } diff --git a/lib/Bashbrew/RegistryUserAgent.pm b/lib/Bashbrew/RegistryUserAgent.pm index 2b05095..66ec57b 100644 --- a/lib/Bashbrew/RegistryUserAgent.pm +++ b/lib/Bashbrew/RegistryUserAgent.pm @@ -36,6 +36,13 @@ use constant MEDIA_FOREIGN_LAYER => 'application/vnd.docker.image.rootfs.foreign use constant MEDIA_OCI_INDEX_V1 => 'application/vnd.oci.image.index.v1+json'; use constant MEDIA_OCI_MANIFEST_V1 => 'application/vnd.oci.image.manifest.v1+json'; +sub is_media_image_manifest ($mediaType) { + return $mediaType eq MEDIA_OCI_MANIFEST_V1 || $mediaType eq MEDIA_MANIFEST_V2 || $mediaType eq MEDIA_MANIFEST_V1; +} +sub is_media_image_list ($mediaType) { + return $mediaType eq MEDIA_OCI_INDEX_V1 || $mediaType eq MEDIA_MANIFEST_LIST; +} + # this is "normally" handled for us by https://github.com/tianon/dockerhub-public-proxy but is necessary for alternative registries my $acceptHeader = [ MEDIA_MANIFEST_LIST, diff --git a/test-localhost.sh b/test-localhost.sh index d29cce4..b88b82e 100755 --- a/test-localhost.sh +++ b/test-localhost.sh @@ -3,17 +3,49 @@ set -Eeuo pipefail # docker run -dit --name registry --restart always -p 5000:5000 registry -arches=( amd64 arm32v5 arm32v6 arm32v7 arm64v8 i386 mips64le ppc64le riscv64 s390x ) -image='busybox:latest' +# docker run -dit --name registry --restart always -p 5000:5000 --env REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW='["^.*$"]' --env REGISTRY_VALIDATION_MANIFESTS_URLS_DENY='[]' registry + +image="${1:-'hello-world:latest'}" +registry='docker.io' target='localhost:5000' +arches="$(bashbrew cat --format '{{- range .Entries }}{{ json .Architectures -}}{{- end -}}' "$image")" +arches="$(jq <<<"$arches" -sr 'flatten | unique | map(@sh) | join(" ")')" +eval "arches=( $arches )" + +if ! command -v crane &> /dev/null; then + if ! docker image inspect --format '.' gcr.io/go-containerregistry/crane &> /dev/null; then + docker pull gcr.io/go-containerregistry/crane + fi + crane() { + local args=( + --interactive --rm + --user "$RANDOM:$RANDOM" + --network host + --security-opt no-new-privileges + ) + if [ -t 0 ] && [ -t 1 ]; then + args+=( --tty ) + fi + docker run "${args[@]}" gcr.io/go-containerregistry/crane "$@" + } +fi + BASHBREW_ARCH_NAMESPACES= for arch in "${arches[@]}"; do - docker image inspect "$arch/$image" &> /dev/null || docker pull "$arch/$image" - docker tag "$arch/$image" "$target/$arch/$image" - docker push "$target/$arch/$image" - #skopeo copy --format oci "docker://docker.io/$arch/$image" --dest-tls-verify=false "docker://$target/$arch/$image" - #skopeo copy --format v2s2 "docker://docker.io/$arch/$image" --dest-tls-verify=false "docker://$target/$arch/$image" + if [ "$arch" = 'windows-amd64' ]; then + src="$registry/winamd64/$image" + else + src="$registry/$arch/$image" + fi + trg="$target/$arch/$image" + + crane copy "$src" --insecure "$trg" + + # skopeo appears to be the only one of these "registry copy" tools willing to do format conversions between Docker and OCI 👀 + #skopeo copy --multi-arch all --format oci "docker://$src" --dest-tls-verify=false "docker://$trg" + #skopeo copy --multi-arch all --format v2s2 "docker://$src" --dest-tls-verify=false "docker://$trg" + [ -z "$BASHBREW_ARCH_NAMESPACES" ] || BASHBREW_ARCH_NAMESPACES+=', ' BASHBREW_ARCH_NAMESPACES+="$arch = $target/$arch" done