update fork and add test

Author: mafredri

Makefile

@@ -67,7 +67,7 @@ test-registry-container: .registry-cache
 
 # Pulls images referenced in integration tests and pushes them to the local cache.
 .PHONY: test-images-push
-test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server
+test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server .registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown
 
 .PHONY: test-images-pull
 test-images-pull:
@@ -77,6 +77,7 @@ test-images-pull:
 	docker tag ubuntu:latest localhost:5000/envbuilder-test-ubuntu:latest
 	docker pull codercom/code-server:latest
 	docker tag codercom/code-server:latest localhost:5000/envbuilder-test-codercom-code-server:latest
+	docker build -t localhost:5000/envbuilder-test-blob-unknown:latest -f integration/testdata/blob-unknown/Dockerfile integration/testdata/blob-unknown
 
 .registry-cache:
 	mkdir -p .registry-cache && chmod -R ag+w .registry-cache
@@ -89,3 +90,6 @@ test-images-pull:
 
 .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server:
 	docker push localhost:5000/envbuilder-test-codercom-code-server:latest
+
+.registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown:
+	docker push localhost:5000/envbuilder-test-blob-unknown:latest

envbuilder.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	stdlog "log"
 	"maps"
 	"net"
 	"net/http"
@@ -44,7 +43,6 @@ import (
 	dockerconfig "github.com/docker/cli/cli/config"
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/fatih/color"
-	"github.com/google/go-containerregistry/pkg/logs"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/kballard/go-shellquote"
@@ -543,7 +541,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
 				NoPush:             !opts.PushImage || len(destinations) == 0,
 				CacheRunLayers:     true,
 				CacheCopyLayers:    true,
-				ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing.
+				ForceBuildMetadata: opts.PushImage, // Force layers with no changes to be cached, required for cache probing.
 				CompressedCaching:  true,
 				Compression:        config.ZStd,
 				// Maps to "default" level, ~100-300 MB/sec according to
@@ -585,19 +583,16 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
 			endStage("🏗️ Built image!")
 			if opts.PushImage {
 				endStage = startStage("🏗️ Pushing image...")
-				kOpts.PushRetry = 3
-				// kOpts.PushIgnoreImmutableTagErrors = true
-				logs.Debug = stdlog.New(os.Stderr, "", 0)
-				logs.Warn = stdlog.New(os.Stderr, "", 0)
-				logs.Progress = stdlog.New(os.Stderr, "", 0)
-				layers, _ := image.Layers()
-				for _, layer := range layers {
-					mediaType, _ := layer.MediaType()
-					diffID, _ := layer.DiffID()
-					digest, _ := layer.Digest()
-					size, _ := layer.Size()
-					opts.Logger(log.LevelDebug, "Layer: %s %s %s %d", mediaType, diffID, digest, size)
-				}
+
+				// To debug registry issues, enable logging:
+				//
+				// 	import (
+				// 		stdlog "log"
+				// 		reglogs "github.com/google/go-containerregistry/pkg/logs"
+				// 	)
+				// 	reglogs.Debug = stdlog.New(os.Stderr, "", 0)
+				// 	reglogs.Warn = stdlog.New(os.Stderr, "", 0)
+				// 	reglogs.Progress = stdlog.New(os.Stderr, "", 0)
 				if err := executor.DoPush(image, kOpts); err == nil {
 					endStage("🏗️ Pushed image!")
 				} else if !opts.ExitOnPushFailure {
@@ -1284,7 +1279,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
 		NoPush:             true,
 		CacheRunLayers:     true,
 		CacheCopyLayers:    true,
-		ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing.
+		ForceBuildMetadata: true, // Force layers with no changes to be cached, required for cache probing.
 		CompressedCaching:  true,
 		Compression:        config.ZStd,
 		// Maps to "default" level, ~100-300 MB/sec according to

go.mod

@@ -4,7 +4,7 @@ go 1.22.4
 
 // There are a few options we need added to Kaniko!
 // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
-replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d
+replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802
 
 // Required to import codersdk due to gvisor dependency.
 replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374

go.sum

@@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0=
 github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo=
-github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d h1:qqoyQMtIHWYRw8RbDPT+pdOQER9UKm36tsJtiy9MVc8=
-github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
+github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802 h1:6vcTuyI9pLlmEkSOw0cT+T74DuqYWMcBJFGj+32C7qk=
+github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
 github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ=

integration/integration_test.go

@@ -57,9 +57,10 @@ import (
 )
 
 const (
-	testContainerLabel = "envbox-integration-test"
-	testImageAlpine    = "localhost:5000/envbuilder-test-alpine:latest"
-	testImageUbuntu    = "localhost:5000/envbuilder-test-ubuntu:latest"
+	testContainerLabel   = "envbox-integration-test"
+	testImageAlpine      = "localhost:5000/envbuilder-test-alpine:latest"
+	testImageUbuntu      = "localhost:5000/envbuilder-test-ubuntu:latest"
+	testImageBlobUnknown = "localhost:5000/envbuilder-test-blob-unknown:latest"
 
 	// nolint:gosec // Throw-away key for testing. DO NOT REUSE.
 	testSSHKey = `-----BEGIN OPENSSH PRIVATE KEY-----
@@ -2354,6 +2355,38 @@ USER devalot
 		}
 		require.Fail(t, "expected pid 1 to be running as devalot")
 	})
+
+	t.Run("PushDuplicateLayersNoBlobUnknown", func(t *testing.T) {
+		t.Parallel()
+
+		srv := gittest.CreateGitServer(t, gittest.Options{
+			Files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s
+USER root
+RUN echo "hi i r empty"
+RUN echo "who u"
+`, testImageBlobUnknown),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// NOTE(mafredri): The in-memory registry doesn't catch this error so we
+		// have to use registry:2.
+		ref, err := name.ParseReference(fmt.Sprintf("localhost:5000/test-blob-unknown-%s", uuid.NewString()))
+		require.NoError(t, err)
+		opts := []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", ref.String()),
+			envbuilderEnv("VERBOSE", "1"),
+		}
+
+		_ = pushImage(t, ref, nil, opts...)
+	})
 }
 
 func TestChownHomedir(t *testing.T) {
@@ -2532,6 +2565,8 @@ func getCachedImage(ctx context.Context, t *testing.T, cli *client.Client, env .
 }
 
 func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client, ref name.Reference) container.CreateResponse {
+	t.Helper()
+
 	// Ensure that we can pull the image.
 	rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{})
 	require.NoError(t, err)

integration/testdata/blob-unknown/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest
+
+WORKDIR /home