diff --git a/cmd/nerdctl/compose.go b/cmd/nerdctl/compose.go index 9b61d4b84ab..1aa28dfc8f5 100644 --- a/cmd/nerdctl/compose.go +++ b/cmd/nerdctl/compose.go @@ -103,6 +103,11 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo if err != nil { return nil, err } + hostsDirs, err := cmd.Flags().GetStringSlice("hosts-dir") + if err != nil { + return nil, err + } + o := composer.Options{ ProjectOptions: composecli.ProjectOptions{ WorkingDir: projectDirectory, @@ -186,7 +191,7 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo pullMode, ocispecPlatforms, nil, quiet) } else { _, imgErr = imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), cmd.ErrOrStderr(), snapshotter, imageName, - pullMode, insecure, ocispecPlatforms, nil, quiet) + pullMode, insecure, hostsDirs, ocispecPlatforms, nil, quiet) } return imgErr } diff --git a/cmd/nerdctl/compose_up_linux_test.go b/cmd/nerdctl/compose_up_linux_test.go index a710ed44f4d..af0f06d8f9e 100644 --- a/cmd/nerdctl/compose_up_linux_test.go +++ b/cmd/nerdctl/compose_up_linux_test.go @@ -27,6 +27,7 @@ import ( "github.com/docker/go-connections/nat" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" "gotest.tools/v3/assert" ) @@ -81,7 +82,7 @@ func testComposeUp(t *testing.T, base *testutil.Base, dockerComposeYAML string) base.Cmd("network", "inspect", fmt.Sprintf("%s_default", projectName)).AssertOK() checkWordpress := func() error { - resp, err := httpGet("http://127.0.0.1:8080", 10) + resp, err := nettestutil.HTTPGet("http://127.0.0.1:8080", 10, false) if err != nil { return err } @@ -144,7 +145,7 @@ COPY index.html /usr/share/nginx/html/index.html base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d", "--build").AssertOK() defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run() - resp, err := httpGet("http://127.0.0.1:8080", 50) + resp, err := nettestutil.HTTPGet("http://127.0.0.1:8080", 50, false) assert.NilError(t, err) respBody, err := io.ReadAll(resp.Body) assert.NilError(t, err) diff --git a/cmd/nerdctl/image_encrypt_linux_test.go b/cmd/nerdctl/image_encrypt_linux_test.go index f9ab93f2016..151768d8a5c 100644 --- a/cmd/nerdctl/image_encrypt_linux_test.go +++ b/cmd/nerdctl/image_encrypt_linux_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/testregistry" "gotest.tools/v3/assert" ) @@ -73,10 +74,10 @@ func TestImageEncryptJWE(t *testing.T) { defer keyPair.cleanup() base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() base.Cmd("pull", testutil.CommonImage).AssertOK() - encryptImageRef := fmt.Sprintf("127.0.0.1:%d/%s:encrypted", reg.listenPort, tID) + encryptImageRef := fmt.Sprintf("127.0.0.1:%d/%s:encrypted", reg.ListenPort, tID) defer base.Cmd("rmi", encryptImageRef).Run() base.Cmd("image", "encrypt", "--recipient=jwe:"+keyPair.pub, testutil.CommonImage, encryptImageRef).AssertOK() base.Cmd("image", "inspect", "--mode=native", "--format={{len .Index.Manifests}}", encryptImageRef).AssertOutExactly("1\n") diff --git a/cmd/nerdctl/ipfs_compose_linux_test.go b/cmd/nerdctl/ipfs_compose_linux_test.go index 332781dfb41..5d3a00bea28 100644 --- a/cmd/nerdctl/ipfs_compose_linux_test.go +++ b/cmd/nerdctl/ipfs_compose_linux_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" "gotest.tools/v3/assert" ) @@ -129,7 +130,7 @@ COPY index.html /usr/share/nginx/html/index.html defer base.Cmd("ipfs", "registry", "down").AssertOK() defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run() - resp, err := httpGet("http://127.0.0.1:8080", 50) + resp, err := nettestutil.HTTPGet("http://127.0.0.1:8080", 50, false) assert.NilError(t, err) respBody, err := io.ReadAll(resp.Body) assert.NilError(t, err) diff --git a/cmd/nerdctl/login.go b/cmd/nerdctl/login.go index 5e64f0c3fd7..d80e810629d 100644 --- a/cmd/nerdctl/login.go +++ b/cmd/nerdctl/login.go @@ -21,16 +21,20 @@ import ( "errors" "fmt" "io" - "runtime" + "net/http" + "net/url" "strings" - "github.com/containerd/nerdctl/pkg/version" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/remotes/docker" + dockerconfig "github.com/containerd/containerd/remotes/docker/config" + "github.com/containerd/nerdctl/pkg/imgutil/dockerconfigresolver" dockercliconfig "github.com/docker/cli/cli/config" clitypes "github.com/docker/cli/cli/config/types" dockercliconfigtypes "github.com/docker/cli/cli/config/types" "github.com/docker/docker/api/types" - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/registry" + "golang.org/x/net/context/ctxhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -76,7 +80,7 @@ func loginAction(cmd *cobra.Command, args []string) error { serverAddress = options.serverAddress } - var response registrytypes.AuthenticateOKBody + var responseIdentityToken string ctx := cmd.Context() isDefaultRegistry := serverAddress == registry.IndexServer authConfig, err := GetDefaultAuthConfig(options.username == "" && options.password == "", serverAddress, isDefaultRegistry) @@ -85,7 +89,7 @@ func loginAction(cmd *cobra.Command, args []string) error { } if err == nil && authConfig.Username != "" && authConfig.Password != "" { //login With StoreCreds - response, err = loginClientSide(ctx, cmd, *authConfig) + responseIdentityToken, err = loginClientSide(ctx, cmd, *authConfig) } if err != nil || authConfig.Username == "" || authConfig.Password == "" { @@ -94,16 +98,16 @@ func loginAction(cmd *cobra.Command, args []string) error { return err } - response, err = loginClientSide(ctx, cmd, *authConfig) + responseIdentityToken, err = loginClientSide(ctx, cmd, *authConfig) if err != nil { return err } } - if response.IdentityToken != "" { + if responseIdentityToken != "" { authConfig.Password = "" - authConfig.IdentityToken = response.IdentityToken + authConfig.IdentityToken = responseIdentityToken } dockerConfigFile, err := dockercliconfig.Load("") @@ -115,9 +119,10 @@ func loginAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("error saving credentials: %w", err) } - if response.Status != "" { - fmt.Fprintln(cmd.OutOrStdout(), response.Status) + if _, err = loginClientSide(ctx, cmd, *authConfig); err != nil { + return err } + fmt.Fprintln(cmd.OutOrStdout(), "Login Succeeded") return nil } @@ -169,33 +174,119 @@ func GetDefaultAuthConfig(checkCredStore bool, serverAddress string, isDefaultRe return &res, nil } -// Code from github.com/cli/cli/command/registry/login.go -func loginClientSide(ctx context.Context, cmd *cobra.Command, auth types.AuthConfig) (registrytypes.AuthenticateOKBody, error) { +func loginClientSide(ctx context.Context, cmd *cobra.Command, auth types.AuthConfig) (string, error) { + host := auth.ServerAddress + if strings.Contains(host, "://") { + u, err := url.Parse(host) + if err != nil { + return "", err + } + host = u.Host + } - var insecureRegistries []string - insecureRegistry, err := cmd.Flags().GetBool("insecure-registry") + var dOpts []dockerconfigresolver.Opt + insecure, err := cmd.Flags().GetBool("insecure-registry") + if err != nil { + return "", err + } + if insecure { + logrus.Warnf("skipping verifying HTTPS certs for %q", host) + dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true)) + } + hostsDirs, err := cmd.Flags().GetStringSlice("hosts-dir") if err != nil { - return registrytypes.AuthenticateOKBody{}, err + return "", err } - if insecureRegistry { - insecureRegistries = append(insecureRegistries, auth.ServerAddress) + dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(hostsDirs)) + + authCreds := func(acArg string) (string, string, error) { + if acArg == host { + if auth.RegistryToken != "" { + // Even containerd/CRI does not support RegistryToken as of v1.4.3, + // so, nobody is actually using RegistryToken? + logrus.Warnf("RegistryToken (for %q) is not supported yet (FIXME)", host) + } + if auth.IdentityToken != "" { + return "", auth.IdentityToken, nil + } + return auth.Username, auth.Password, nil + } + return "", "", fmt.Errorf("expected acArg to be %q, got %q", host, acArg) } - svc, err := registry.NewService(registry.ServiceOptions{ - InsecureRegistries: insecureRegistries, - }) + dOpts = append(dOpts, dockerconfigresolver.WithAuthCreds(authCreds)) + ho, err := dockerconfigresolver.NewHostOptions(ctx, host, dOpts...) + if err != nil { + return "", err + } + fetchedRefreshTokens := make(map[string]string) // key: req.URL.Host + // onFetchRefreshToken is called when tryLoginWithRegHost calls rh.Authorizer.Authorize() + onFetchRefreshToken := func(ctx context.Context, s string, req *http.Request) { + fetchedRefreshTokens[req.URL.Host] = s + } + ho.AuthorizerOpts = append(ho.AuthorizerOpts, docker.WithFetchRefreshToken(onFetchRefreshToken)) + regHosts, err := dockerconfig.ConfigureHosts(ctx, *ho)(host) if err != nil { - return registrytypes.AuthenticateOKBody{}, err + return "", err + } + logrus.Debugf("len(regHosts)=%d", len(regHosts)) + if len(regHosts) == 0 { + return "", fmt.Errorf("got empty []docker.RegistryHost for %q", host) + } + for i, rh := range regHosts { + err = tryLoginWithRegHost(ctx, rh) + identityToken := fetchedRefreshTokens[rh.Host] // can be empty + if err == nil { + return identityToken, nil + } + logrus.WithError(err).WithField("i", i).Error("failed to call tryLoginWithRegHost") } + return "", err +} - userAgent := fmt.Sprintf("Docker-Client/nerdctl-%s (%s)", version.Version, runtime.GOOS) +func tryLoginWithRegHost(ctx context.Context, rh docker.RegistryHost) error { + if rh.Authorizer == nil { + return errors.New("got nil Authorizer") + } + u := url.URL{ + Scheme: rh.Scheme, + Host: rh.Host, + Path: rh.Path, + } + ctx = docker.WithScope(ctx, "") + var ress []*http.Response + for i := 0; i < 10; i++ { + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return err + } + for k, v := range rh.Header.Clone() { + for _, vv := range v { + req.Header.Add(k, vv) + } + } + if err := rh.Authorizer.Authorize(ctx, req); err != nil { + return fmt.Errorf("failed to call rh.Authorizer.Authorize: %w", err) + } + res, err := ctxhttp.Do(ctx, rh.Client, req) + if err != nil { + return fmt.Errorf("failed to call rh.Client.Do: %w", err) + } + ress = append(ress, res) + if res.StatusCode == 401 { + if err := rh.Authorizer.AddResponses(ctx, ress); err != nil && !errdefs.IsNotImplemented(err) { + return fmt.Errorf("failed to call rh.Authorizer.AddResponses: %w", err) + } + continue + } + if res.StatusCode/100 != 2 { + return fmt.Errorf("unexpected status code %d", res.StatusCode) + } - status, token, err := svc.Auth(ctx, &auth, userAgent) + return nil + } - return registrytypes.AuthenticateOKBody{ - Status: status, - IdentityToken: token, - }, err + return errors.New("too many 401 (probably)") } func ConfigureAuthentification(authConfig *types.AuthConfig, options *loginOptions) error { diff --git a/cmd/nerdctl/login_linux_test.go b/cmd/nerdctl/login_linux_test.go new file mode 100644 index 00000000000..3e155c27baf --- /dev/null +++ b/cmd/nerdctl/login_linux_test.go @@ -0,0 +1,43 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "net" + "strconv" + "testing" + + "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/testregistry" +) + +func TestLogin(t *testing.T) { + // Skip docker, because Docker doesn't have `--hosts-dir` option, and we don't want to contaminate the global /etc/docker/certs.d during this test + testutil.DockerIncompatible(t) + + base := testutil.NewBase(t) + reg := testregistry.NewHTTPS(base, "admin", "validTestPassword") + defer reg.Cleanup() + + regHost := net.JoinHostPort(reg.IP.String(), strconv.Itoa(reg.ListenPort)) + + t.Logf("Good password") + base.Cmd("--debug-full", "--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "validTestPassword", regHost).AssertOK() + + t.Logf("Bad password") + base.Cmd("--debug-full", "--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "invalidTestPassword", regHost).AssertFail() +} diff --git a/cmd/nerdctl/main.go b/cmd/nerdctl/main.go index 2c1655a4e49..fd25ac79f48 100644 --- a/cmd/nerdctl/main.go +++ b/cmd/nerdctl/main.go @@ -87,16 +87,17 @@ func xmain() error { // Config corresponds to nerdctl.toml . // See docs/config.md . type Config struct { - Debug bool `toml:"debug"` - DebugFull bool `toml:"debug_full"` - Address string `toml:"address"` - Namespace string `toml:"namespace"` - Snapshotter string `toml:"snapshotter"` - CNIPath string `toml:"cni_path"` - CNINetConfPath string `toml:"cni_netconfpath"` - DataRoot string `toml:"data_root"` - CgroupManager string `toml:"cgroup_manager"` - InsecureRegistry bool `toml:"insecure_registry"` + Debug bool `toml:"debug"` + DebugFull bool `toml:"debug_full"` + Address string `toml:"address"` + Namespace string `toml:"namespace"` + Snapshotter string `toml:"snapshotter"` + CNIPath string `toml:"cni_path"` + CNINetConfPath string `toml:"cni_netconfpath"` + DataRoot string `toml:"data_root"` + CgroupManager string `toml:"cgroup_manager"` + InsecureRegistry bool `toml:"insecure_registry"` + HostsDir []string `toml:"hosts_dir"` } // NewConfig creates a default Config object statically, @@ -113,6 +114,7 @@ func NewConfig() *Config { DataRoot: ncdefaults.DataRoot(), CgroupManager: ncdefaults.CgroupManager(), InsecureRegistry: false, + HostsDir: ncdefaults.HostsDirs(), } } @@ -148,6 +150,8 @@ func initRootCmdFlags(rootCmd *cobra.Command, tomlPath string) error { rootCmd.PersistentFlags().String("cgroup-manager", cfg.CgroupManager, `Cgroup manager to use ("cgroupfs"|"systemd")`) rootCmd.RegisterFlagCompletionFunc("cgroup-manager", shellCompleteCgroupManagerNames) rootCmd.PersistentFlags().Bool("insecure-registry", cfg.InsecureRegistry, "skips verifying HTTPS certs, and allows falling back to plain HTTP") + // hosts-dir is defined as StringSlice, not StringArray, to allow specifying "--hosts-dir=/etc/containerd/certs.d,/etc/docker/certs.d" + rootCmd.PersistentFlags().StringSlice("hosts-dir", cfg.HostsDir, "A directory that contains /hosts.toml (containerd style) or /{ca.cert, cert.pem, key.pem} (docker style)") return nil } diff --git a/cmd/nerdctl/multi_platform_linux_test.go b/cmd/nerdctl/multi_platform_linux_test.go index b65551d4c71..b814f19bd47 100644 --- a/cmd/nerdctl/multi_platform_linux_test.go +++ b/cmd/nerdctl/multi_platform_linux_test.go @@ -24,6 +24,8 @@ import ( "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" + "github.com/containerd/nerdctl/pkg/testutil/testregistry" "gotest.tools/v3/assert" ) @@ -55,10 +57,10 @@ func TestMultiPlatformBuildPush(t *testing.T) { testutil.RequireExecPlatform(t, "linux/amd64", "linux/arm64", "linux/arm/v7") base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() - imageName := fmt.Sprintf("localhost:%d/%s:latest", reg.listenPort, tID) + imageName := fmt.Sprintf("localhost:%d/%s:latest", reg.ListenPort, tID) defer base.Cmd("rmi", imageName).Run() dockerfile := fmt.Sprintf(`FROM %s @@ -78,10 +80,10 @@ func TestMultiPlatformPullPushAllPlatforms(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() - pushImageName := fmt.Sprintf("localhost:%d/%s:latest", reg.listenPort, tID) + pushImageName := fmt.Sprintf("localhost:%d/%s:latest", reg.ListenPort, tID) defer base.Cmd("rmi", pushImageName).Run() base.Cmd("pull", "--all-platforms", testutil.AlpineImage).AssertOK() @@ -133,7 +135,7 @@ RUN uname -m > /usr/share/nginx/html/index.html } for testURL, expectedIndexHTML := range testCases { - resp, err := httpGet(testURL, 50) + resp, err := nettestutil.HTTPGet(testURL, 50, false) assert.NilError(t, err) respBody, err := io.ReadAll(resp.Body) assert.NilError(t, err) diff --git a/cmd/nerdctl/pull.go b/cmd/nerdctl/pull.go index 9d3b39fa2e4..795dd7ae5de 100644 --- a/cmd/nerdctl/pull.go +++ b/cmd/nerdctl/pull.go @@ -125,6 +125,10 @@ func ensureImage(cmd *cobra.Command, ctx context.Context, client *containerd.Cli if err != nil { return nil, err } + hostsDirs, err := cmd.Flags().GetStringSlice("hosts-dir") + if err != nil { + return nil, err + } verifier, err := cmd.Flags().GetString("verify") if err != nil { return nil, err @@ -155,7 +159,7 @@ func ensureImage(cmd *cobra.Command, ctx context.Context, client *containerd.Cli return nil, err } - ref, err = verifyCosign(ctx, rawRef, keyRef) + ref, err = verifyCosign(ctx, rawRef, keyRef, hostsDirs) if err != nil { return nil, err } @@ -166,15 +170,15 @@ func ensureImage(cmd *cobra.Command, ctx context.Context, client *containerd.Cli } ensured, err = imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), cmd.ErrOrStderr(), snapshotter, ref, - pull, insecureRegistry, ocispecPlatforms, unpack, quiet) + pull, insecureRegistry, hostsDirs, ocispecPlatforms, unpack, quiet) if err != nil { return nil, err } return ensured, err } -func verifyCosign(ctx context.Context, rawRef string, keyRef string) (string, error) { - digest, err := imgutil.ResolveDigest(ctx, rawRef, false) +func verifyCosign(ctx context.Context, rawRef string, keyRef string, hostsDirs []string) (string, error) { + digest, err := imgutil.ResolveDigest(ctx, rawRef, false, hostsDirs) if err != nil { logrus.WithError(err).Errorf("unable to resolve digest for an image %s: %v", rawRef, err) return rawRef, err diff --git a/cmd/nerdctl/pull_linux_test.go b/cmd/nerdctl/pull_linux_test.go index 137364b7bb8..bde63b99541 100644 --- a/cmd/nerdctl/pull_linux_test.go +++ b/cmd/nerdctl/pull_linux_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/testregistry" "gotest.tools/v3/assert" ) @@ -66,12 +67,12 @@ func TestImageVerifyWithCosign(t *testing.T) { defer keyPair.cleanup() base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) testImageRef := fmt.Sprintf("%s:%d/%s", - localhostIP, reg.listenPort, tID) + localhostIP, reg.ListenPort, tID) t.Logf("testImageRef=%q", testImageRef) dockerfile := fmt.Sprintf(`FROM %s @@ -98,12 +99,12 @@ func TestImageVerifyWithCosignShouldFailWhenKeyIsNotCorrect(t *testing.T) { defer keyPair.cleanup() base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) testImageRef := fmt.Sprintf("%s:%d/%s", - localhostIP, reg.listenPort, tID) + localhostIP, reg.ListenPort, tID) t.Logf("testImageRef=%q", testImageRef) dockerfile := fmt.Sprintf(`FROM %s diff --git a/cmd/nerdctl/push.go b/cmd/nerdctl/push.go index 42a27ce2ec2..b6b0980d580 100644 --- a/cmd/nerdctl/push.go +++ b/cmd/nerdctl/push.go @@ -173,7 +173,12 @@ func pushAction(cmd *cobra.Command, args []string) error { logrus.Warnf("skipping verifying HTTPS certs for %q", refDomain) dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true)) } - resolver, err := dockerconfigresolver.New(refDomain, dOpts...) + hostsDirs, err := cmd.Flags().GetStringSlice("hosts-dir") + if err != nil { + return err + } + dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(hostsDirs)) + resolver, err := dockerconfigresolver.New(ctx, refDomain, dOpts...) if err != nil { return err } @@ -184,7 +189,7 @@ func pushAction(cmd *cobra.Command, args []string) error { if insecure { logrus.WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain) dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true)) - resolver, err = dockerconfigresolver.New(refDomain, dOpts...) + resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...) if err != nil { return err } diff --git a/cmd/nerdctl/push_linux_test.go b/cmd/nerdctl/push_linux_test.go index 26842001229..79661828ea8 100644 --- a/cmd/nerdctl/push_linux_test.go +++ b/cmd/nerdctl/push_linux_test.go @@ -17,187 +17,23 @@ package main import ( - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "fmt" - "math/big" - "net" - "net/http" - "os" - "runtime" "strings" "testing" - "time" - "github.com/containerd/containerd/errdefs" "github.com/containerd/nerdctl/pkg/testutil" - - "golang.org/x/crypto/bcrypt" + "github.com/containerd/nerdctl/pkg/testutil/testregistry" "gotest.tools/v3/assert" ) -func getNonLoopbackIPv4() (net.IP, error) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return nil, err - } - for _, addr := range addrs { - ip, _, err := net.ParseCIDR(addr.String()) - if err != nil { - continue - } - ipv4 := ip.To4() - if ipv4 == nil { - continue - } - if ipv4.IsLoopback() { - continue - } - return ipv4, nil - } - return nil, fmt.Errorf("non-loopback IPv4 address not found, attempted=%+v: %w", addrs, errdefs.ErrNotFound) -} - -type testRegistry struct { - ip net.IP - listenIP net.IP - listenPort int - cleanup func() -} - -func newTestRegistry(base *testutil.Base) *testRegistry { - if runtime.GOOS != "linux" { - base.T.Skip("only linux is supported, currently") - } - hostIP, err := getNonLoopbackIPv4() - assert.NilError(base.T, err) - // listen on 0.0.0.0 to enable 127.0.0.1 - listenIP := net.ParseIP("0.0.0.0") - const listenPort = 5000 // TODO: choose random empty port - base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d", hostIP, listenIP, listenPort) - - registryContainerName := "reg-" + testutil.Identifier(base.T) - cmd := base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5000", listenIP, listenPort), - "--name", registryContainerName, - testutil.RegistryImage) - cmd.AssertOK() - if _, err = httpGet(fmt.Sprintf("http://%s:%d/v2", hostIP.String(), listenPort), 30); err != nil { - base.Cmd("rm", "-f", registryContainerName).Run() - base.T.Fatal(err) - } - return &testRegistry{ - ip: hostIP, - listenIP: listenIP, - listenPort: listenPort, - cleanup: func() { base.Cmd("rm", "-f", registryContainerName).AssertOK() }, - } -} - -func newTestInsecureRegistry(base *testutil.Base, user, pass string) *testRegistry { - if runtime.GOOS != "linux" { - base.T.Skip("only linux is supported, currently") - } - name := testutil.Identifier(base.T) - hostIP, err := getNonLoopbackIPv4() - assert.NilError(base.T, err) - // listen on 0.0.0.0 to enable 127.0.0.1 - listenIP := net.ParseIP("0.0.0.0") - const listenPort = 5000 // TODO: choose random empty port - const authPort = 5100 // TODO: choose random empty port - base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d, authPort=%d", hostIP, listenIP, listenPort, authPort) - - registryCert, registryKey, registryClose := generateTestCert(base, hostIP.String()) - authCert, authKey, authClose := generateTestCert(base, hostIP.String()) - - // Prepare configuration file for authentication server - // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml - authConfigFile, err := os.CreateTemp("", "authconfig") - assert.NilError(base.T, err) - bpass, err := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost) - assert.NilError(base.T, err) - authConfigFileName := authConfigFile.Name() - _, err = authConfigFile.Write([]byte(fmt.Sprintf(` -server: - addr: ":5100" - certificate: "/auth/domain.crt" - key: "/auth/domain.key" -token: - issuer: "Acme auth server" - expiration: 900 -users: - "%s": - password: "%s" -acl: - - match: {account: "%s"} - actions: ["*"] -`, user, string(bpass), user))) - assert.NilError(base.T, err) - - // Run authentication server - authContainerName := "auth-" + name - cmd := base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5100", listenIP, authPort), - "--name", authContainerName, - "-v", authCert+":/auth/domain.crt", - "-v", authKey+":/auth/domain.key", - "-v", authConfigFileName+":/config/auth_config.yml", - testutil.DockerAuthImage, - "/config/auth_config.yml") - cmd.AssertOK() - - // Run docker_auth-enabled registry - // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml - registryContainerName := "reg-" + name - cmd = base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5000", listenIP, listenPort), - "--name", registryContainerName, - "--env", "REGISTRY_AUTH=token", - "--env", "REGISTRY_AUTH_TOKEN_REALM="+fmt.Sprintf("https://%s:%d/auth", hostIP.String(), authPort), - "--env", "REGISTRY_AUTH_TOKEN_SERVICE=Docker registry", - "--env", "REGISTRY_AUTH_TOKEN_ISSUER=Acme auth server", - "--env", "REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE=/auth/domain.crt", - "--env", "REGISTRY_HTTP_TLS_CERTIFICATE=/registry/domain.crt", - "--env", "REGISTRY_HTTP_TLS_KEY=/registry/domain.key", - "-v", authCert+":/auth/domain.crt", - "-v", registryCert+":/registry/domain.crt", - "-v", registryKey+":/registry/domain.key", - testutil.RegistryImage) - cmd.AssertOK() - if _, err = httpInsecureGet(fmt.Sprintf("https://%s:%d/v2", hostIP.String(), listenPort), 30); err != nil { - base.Cmd("rm", "-f", registryContainerName).Run() - base.T.Fatal(err) - } - return &testRegistry{ - ip: hostIP, - listenIP: listenIP, - listenPort: listenPort, - cleanup: func() { - base.Cmd("rm", "-f", registryContainerName).AssertOK() - base.Cmd("rm", "-f", authContainerName).AssertOK() - assert.NilError(base.T, registryClose()) - assert.NilError(base.T, authClose()) - assert.NilError(base.T, authConfigFile.Close()) - os.Remove(authConfigFileName) - }, - } -} - func TestPushPlainHTTPFails(t *testing.T) { base := testutil.NewBase(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.ip.String(), reg.listenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -210,14 +46,14 @@ func TestPushPlainHTTPFails(t *testing.T) { func TestPushPlainHTTPLocalhost(t *testing.T) { base := testutil.NewBase(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - localhostIP, reg.listenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + localhostIP, reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -229,12 +65,12 @@ func TestPushPlainHTTPInsecure(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.ip.String(), reg.listenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -246,90 +82,36 @@ func TestPushInsecureWithLogin(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := newTestInsecureRegistry(base, "admin", "badmin") - defer reg.cleanup() + reg := testregistry.NewHTTPS(base, "admin", "badmin") + defer reg.Cleanup() base.Cmd("--insecure-registry", "login", "-u", "admin", "-p", "badmin", - fmt.Sprintf("%s:%d", reg.ip.String(), reg.listenPort)).AssertOK() + fmt.Sprintf("%s:%d", reg.IP.String(), reg.ListenPort)).AssertOK() base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.ip.String(), reg.listenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() + base.Cmd("push", testImageRef).AssertFail() base.Cmd("--insecure-registry", "push", testImageRef).AssertOK() } -func generateTestCert(base *testutil.Base, host string) (crtPath, keyPath string, closeFn func() error) { - certF, err := os.CreateTemp("", "certtemp") - assert.NilError(base.T, err) - keyF, err := os.CreateTemp("", "keytemp") - assert.NilError(base.T, err) +func TestPushWithHostsDir(t *testing.T) { + // Skip docker, because Docker doesn't have `--hosts-dir` option, and we don't want to contaminate the global /etc/docker/certs.d during this test + testutil.DockerIncompatible(t) - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 60) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - assert.NilError(base.T, err) - template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{Organization: []string{"test"}}, - NotBefore: time.Now(), - NotAfter: time.Now().Add(365 * 24 * time.Hour), - DNSNames: []string{host}, - } - privatekey, err := rsa.GenerateKey(rand.Reader, 2048) - assert.NilError(base.T, err) - publickey := &privatekey.PublicKey - cert, err := x509.CreateCertificate(rand.Reader, &template, &template, publickey, privatekey) - assert.NilError(base.T, err) - privBytes, err := x509.MarshalPKCS8PrivateKey(privatekey) - assert.NilError(base.T, err) + base := testutil.NewBase(t) + reg := testregistry.NewHTTPS(base, "admin", "badmin") + defer reg.Cleanup() - assert.NilError(base.T, pem.Encode(certF, &pem.Block{Type: "CERTIFICATE", Bytes: cert})) - assert.NilError(base.T, pem.Encode(keyF, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes})) + base.Cmd("--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "badmin", fmt.Sprintf("%s:%d", reg.IP.String(), reg.ListenPort)).AssertOK() - return certF.Name(), keyF.Name(), func() error { - var errors []error - certFName := certF.Name() - keyFName := keyF.Name() - for _, f := range []func() error{ - certF.Close, - keyF.Close, - func() error { return os.Remove(certFName) }, - func() error { return os.Remove(keyFName) }, - } { - if err := f(); err != nil { - errors = append(errors, err) - } - } - if len(errors) > 0 { - return fmt.Errorf("failed to close tmpfile: %v", errors) - } - return nil - } -} + base.Cmd("pull", testutil.CommonImage).AssertOK() + testImageRef := fmt.Sprintf("%s:%d/%s:%s", + reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + t.Logf("testImageRef=%q", testImageRef) + base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() -func httpInsecureGet(urlStr string, attempts int) (*http.Response, error) { - var ( - resp *http.Response - err error - ) - if attempts < 1 { - return nil, errdefs.ErrInvalidArgument - } - client := &http.Client{ - Timeout: 3 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - for i := 0; i < attempts; i++ { - resp, err = client.Get(urlStr) - if err == nil { - return resp, nil - } - time.Sleep(100 * time.Millisecond) - } - return nil, fmt.Errorf("error after %d attempts: %w", attempts, err) + base.Cmd("--debug", "--hosts-dir", reg.HostsDir, "push", testImageRef).AssertOK() } diff --git a/cmd/nerdctl/run_network_linux_test.go b/cmd/nerdctl/run_network_linux_test.go index 3b55e5e8ea8..90b437b4406 100644 --- a/cmd/nerdctl/run_network_linux_test.go +++ b/cmd/nerdctl/run_network_linux_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" "gotest.tools/v3/assert" ) @@ -140,7 +141,7 @@ func valuesOfMapStringString(m map[string]string) map[string]struct{} { } func TestRunPort(t *testing.T) { - hostIP, err := getNonLoopbackIPv4() + hostIP, err := nettestutil.NonLoopbackIPv4() assert.NilError(t, err) type testCase struct { listenIP net.IP @@ -307,7 +308,7 @@ func TestRunPort(t *testing.T) { return } - resp, err := httpGet(connectURL, 30) + resp, err := nettestutil.HTTPGet(connectURL, 30, false) if tc.err != "" { assert.ErrorContains(t, err, tc.err) return diff --git a/cmd/nerdctl/run_restart_linux_test.go b/cmd/nerdctl/run_restart_linux_test.go index 697550a3dff..cfcbba1bc86 100644 --- a/cmd/nerdctl/run_restart_linux_test.go +++ b/cmd/nerdctl/run_restart_linux_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" "gotest.tools/v3/assert" ) @@ -51,7 +52,7 @@ func TestRunRestart(t *testing.T) { testutil.NginxAlpineImage).AssertOK() check := func(httpGetRetry int) error { - resp, err := httpGet(fmt.Sprintf("http://127.0.0.1:%d", hostPort), httpGetRetry) + resp, err := nettestutil.HTTPGet(fmt.Sprintf("http://127.0.0.1:%d", hostPort), httpGetRetry, false) if err != nil { return err } diff --git a/cmd/nerdctl/run_verify_linux_test.go b/cmd/nerdctl/run_verify_linux_test.go index c478a08344c..e7372c4b7ae 100644 --- a/cmd/nerdctl/run_verify_linux_test.go +++ b/cmd/nerdctl/run_verify_linux_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/testregistry" "gotest.tools/v3/assert" ) @@ -37,12 +38,12 @@ func TestRunVerifyCosign(t *testing.T) { defer keyPair.cleanup() base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := newTestRegistry(base) - defer reg.cleanup() + reg := testregistry.NewPlainHTTP(base) + defer reg.Cleanup() localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) testImageRef := fmt.Sprintf("%s:%d/%s", - localhostIP, reg.listenPort, tID) + localhostIP, reg.ListenPort, tID) t.Logf("testImageRef=%q", testImageRef) dockerfile := fmt.Sprintf(`FROM %s diff --git a/cmd/nerdctl/stop_linux_test.go b/cmd/nerdctl/stop_linux_test.go index 4defddd028e..7a7f65f667f 100644 --- a/cmd/nerdctl/stop_linux_test.go +++ b/cmd/nerdctl/stop_linux_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" "gotest.tools/v3/assert" ) @@ -42,7 +43,7 @@ func TestStopStart(t *testing.T) { testutil.NginxAlpineImage).AssertOK() check := func(httpGetRetry int) error { - resp, err := httpGet(fmt.Sprintf("http://127.0.0.1:%d", hostPort), httpGetRetry) + resp, err := nettestutil.HTTPGet(fmt.Sprintf("http://127.0.0.1:%d", hostPort), httpGetRetry, false) if err != nil { return err } diff --git a/docs/config.md b/docs/config.md index 05bb4a06349..5b7936b964b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -24,6 +24,7 @@ address = "unix:///run/k3s/containerd/containerd.sock" namespace = "k8s.io" snapshotter = "stargz" cgroup_manager = "cgroupfs" +hosts_dir = ["/etc/containerd/certs.d", "/etc/docker/certs.d"] ``` ## Properties @@ -40,6 +41,7 @@ cgroup_manager = "cgroupfs" | `data_root` | `--data-root` | | Persistent state directory | Since 0.16.0 | | `cgroup_manager` | `--cgroup-manager` | | cgroup manager | Since 0.16.0 | | `insecure_registry` | `--insecure-registry` | | Allow insecure registry | Since 0.16.0 | +| `hosts_dir` | `--hosts-dir` | | `certs.d` directory | Since 0.16.0 | The properties are parsed in the following precedence: 1. CLI flag diff --git a/docs/faq.md b/docs/faq.md index b6872f1cf08..59bd90290e2 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -13,6 +13,7 @@ - [nerdctl ignores `[plugins."io.containerd.grpc.v1.cri"]` config](#nerdctl-ignores-pluginsiocontainerdgrpcv1cri-config) - [How to login to a registry?](#how-to-login-to-a-registry) - [How to use a non-HTTPS registry?](#how-to-use-a-non-https-registry) + - [How to specify the CA certificate of the registry?](#how-to-specify-the-ca-certificate-of-the-registry) - [How to change the cgroup driver?](#how-to-change-the-cgroup-driver) - [How to change the snapshotter?](#how-to-change-the-snapshotter) - [How to change the runtime?](#how-to-change-the-runtime) @@ -99,6 +100,29 @@ nerdctl also supports credential helper binaries such as `docker-credential-ecr- Use `nerdctl --insecure-registry run `. See also [`registry.md`](./registry.md). +### How to specify the CA certificate of the registry? + +| :zap: Requirement | nerdctl >= 0.16 | +|-------------------|-----------------| + +Create `~/.config/containerd/certs.d//hosts.toml` (or `/etc/containerd/certs.d/...` for rootful) to specify `ca` certificates. + +```toml +# An example of ~/.config/containerd/certs.d/192.168.12.34:5000/hosts.toml +# (The path is "/etc/containerd/certs.d/192.168.12.34:5000/hosts.toml" for rootful) + +server = "https://192.168.12.34:5000" +[host."https://192.168.12.34:5000"] + ca = "/path/to/ca.crt" +``` + +See https://github.com/containerd/containerd/blob/main/docs/hosts.md for the syntax of `hosts.toml` . + +Docker-style directories are also supported. +The path is `~/.config/docker/certs.d` for rootless, `/etc/docker/certs.d` for rootful. + +See also [`registry.md`](./registry.md). + ### How to change the cgroup driver? - Option 1: `nerdctl --cgroup-manager=(cgroupfs|systemd|none)`. diff --git a/docs/registry.md b/docs/registry.md index 8dff774e53f..d50bf8bb319 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -13,6 +13,29 @@ e.g., $ nerdctl --insecure-registry run --rm 192.168.12.34:5000/foo ``` +## Specifying certificates + + +| :zap: Requirement | nerdctl >= 0.16 | +|-------------------|-----------------| + + +Create `~/.config/containerd/certs.d//hosts.toml` (or `/etc/containerd/certs.d/...` for rootful) to specify `ca` certificates. + +```toml +# An example of ~/.config/containerd/certs.d/192.168.12.34:5000/hosts.toml +# (The path is "/etc/containerd/certs.d/192.168.12.34:5000/hosts.toml" for rootful) + +server = "https://192.168.12.34:5000" +[host."https://192.168.12.34:5000"] + ca = "/path/to/ca.crt" +``` + +See https://github.com/containerd/containerd/blob/main/docs/hosts.md for the syntax of `hosts.toml` . + +Docker-style directories are also supported. +The path is `~/.config/docker/certs.d` for rootless, `/etc/docker/certs.d` for rootful. + ## Accessing 127.0.0.1 from rootless nerdctl Currently, rootless nerdctl cannot pull images from 127.0.0.1, because diff --git a/go.mod b/go.mod index 2770d7bed1f..2ae18dd6e92 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/compose-spec/compose-go v1.0.8 github.com/containerd/cgroups v1.0.2 github.com/containerd/console v1.0.3 - github.com/containerd/containerd v1.6.0-beta.5 + github.com/containerd/containerd v1.6.0-beta.5.0.20220106193438-3ccd43c8f685 github.com/containerd/containerd/api v1.6.0-beta.3 github.com/containerd/continuity v0.2.2-0.20211201162329-8e53e7cac79d github.com/containerd/go-cni v1.1.1-0.20211215152955-2d9d28f46d8e @@ -46,6 +46,7 @@ require ( github.com/vishvananda/netlink v1.1.1-0.20211129163951-9ada19101fc5 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + golang.org/x/net v0.0.0-20211216030914-fe4d6282115f golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 @@ -161,7 +162,6 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.17.0 // indirect - golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect diff --git a/go.sum b/go.sum index 2490e0c2b25..54b0c1cb0bb 100644 --- a/go.sum +++ b/go.sum @@ -296,8 +296,8 @@ github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTV github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= github.com/containerd/containerd v1.6.0-beta.2.0.20211117185425-a776a27af54a/go.mod h1:0AwP8LDBKEIaCT48IETmHkY1+YX7c/ALcN1kkLGBLtk= -github.com/containerd/containerd v1.6.0-beta.5 h1:07rR8S3D5/8sfrbJ9rloLua4U0BPAZC0qvssdYF2RRk= -github.com/containerd/containerd v1.6.0-beta.5/go.mod h1:PnZPHEkIpTckm5JGL2+DPssQDN6lNE9jncwXRYktpXg= +github.com/containerd/containerd v1.6.0-beta.5.0.20220106193438-3ccd43c8f685 h1:+vrXOSMO8SA5hbRbp1/QPRoYVIi1lsPROsYQ6G2f890= +github.com/containerd/containerd v1.6.0-beta.5.0.20220106193438-3ccd43c8f685/go.mod h1:PnZPHEkIpTckm5JGL2+DPssQDN6lNE9jncwXRYktpXg= github.com/containerd/containerd/api v1.6.0-beta.1/go.mod h1:XDzkCoLyj2hn24f13Jcuq/U2bHb2LjJ2qWlklgL0Ofg= github.com/containerd/containerd/api v1.6.0-beta.3 h1:+w8zh0hbn4cNIkAtt4v95dBylcwp1hEsFJ5lxbr8wgY= github.com/containerd/containerd/api v1.6.0-beta.3/go.mod h1:fkctx1jj7m92mQDI6mIEXF+SH3tt2Rv/azUHqrOxYPc= diff --git a/pkg/defaults/defaults_freebsd.go b/pkg/defaults/defaults_freebsd.go index 9b1bacc68e7..bc12c3a3e01 100644 --- a/pkg/defaults/defaults_freebsd.go +++ b/pkg/defaults/defaults_freebsd.go @@ -51,3 +51,7 @@ func CgroupnsMode() string { func NerdctlTOML() string { return "/etc/nerdctl/nerdctl.toml" } + +func HostsDirs() []string { + return []string{"/etc/containerd/certs.d", "/etc/docker/certs.d"} +} diff --git a/pkg/defaults/defaults_linux.go b/pkg/defaults/defaults_linux.go index 54f121d4505..b5e12710c84 100644 --- a/pkg/defaults/defaults_linux.go +++ b/pkg/defaults/defaults_linux.go @@ -104,3 +104,17 @@ func NerdctlTOML() string { } return filepath.Join(xch, "nerdctl/nerdctl.toml") } + +func HostsDirs() []string { + if !rootlessutil.IsRootless() { + return []string{"/etc/containerd/certs.d", "/etc/docker/certs.d"} + } + xch, err := rootlessutil.XDGConfigHome() + if err != nil { + panic(err) + } + return []string{ + filepath.Join(xch, "containerd/certs.d"), + filepath.Join(xch, "docker/certs.d"), + } +} diff --git a/pkg/defaults/defaults_windows.go b/pkg/defaults/defaults_windows.go index 1566ceb21de..3094abe45f5 100644 --- a/pkg/defaults/defaults_windows.go +++ b/pkg/defaults/defaults_windows.go @@ -60,3 +60,14 @@ func NerdctlTOML() string { } return filepath.Join(ucd, "nerdctl\\nerdctl.toml") } + +func HostsDirs() []string { + programData := os.Getenv("ProgramData") + if programData == "" { + panic("%ProgramData% needs to be set") + } + return []string{ + filepath.Join(programData, "containerd\\certs.d"), + filepath.Join(programData, "docker\\certs.d"), + } +} diff --git a/pkg/imgutil/dockerconfigresolver/dockerconfigresolver.go b/pkg/imgutil/dockerconfigresolver/dockerconfigresolver.go index 0f0f419fa08..aa8924783a4 100644 --- a/pkg/imgutil/dockerconfigresolver/dockerconfigresolver.go +++ b/pkg/imgutil/dockerconfigresolver/dockerconfigresolver.go @@ -17,22 +17,27 @@ package dockerconfigresolver import ( + "context" "crypto/tls" + "errors" "fmt" - "net/http" + "os" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" + dockerconfig "github.com/containerd/containerd/remotes/docker/config" dockercliconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/credentials" dockercliconfigtypes "github.com/docker/cli/cli/config/types" - + "github.com/docker/docker/errdefs" "github.com/sirupsen/logrus" ) type opts struct { plainHTTP bool skipVerifyCerts bool + hostsDirs []string + authCreds AuthCreds } // Opt for New @@ -52,55 +57,102 @@ func WithSkipVerifyCerts(b bool) Opt { } } -// New instantiates a resolver using $DOCKER_CONFIG/config.json . +// WithHostsDirs specifies directories like /etc/containerd/certs.d and /etc/docker/certs.d +func WithHostsDirs(orig []string) Opt { + var ss []string + if len(orig) == 0 { + logrus.Debug("no hosts dir was specified") + } + for _, v := range orig { + if _, err := os.Stat(v); err == nil { + logrus.Debugf("Found hosts dir %q", v) + ss = append(ss, v) + } else { + if errors.Is(err, os.ErrNotExist) { + logrus.WithError(err).Debugf("Ignoring hosts dir %q", v) + } else { + logrus.WithError(err).Warnf("Ignoring hosts dir %q", v) + } + } + } + return func(o *opts) { + o.hostsDirs = ss + } +} + +func WithAuthCreds(ac AuthCreds) Opt { + return func(o *opts) { + o.authCreds = ac + } +} + +// NewHostOptions instantiates a HostOptions struct using $DOCKER_CONFIG/config.json . // // $DOCKER_CONFIG defaults to "~/.docker". // // refHostname is like "docker.io". -func New(refHostname string, optFuncs ...Opt) (remotes.Resolver, error) { +func NewHostOptions(ctx context.Context, refHostname string, optFuncs ...Opt) (*dockerconfig.HostOptions, error) { var o opts for _, of := range optFuncs { of(&o) } - var authzOpts []docker.AuthorizerOpt - var insecureClient *http.Client - if o.skipVerifyCerts { - insecureClient = newInsecureClient() - authzOpts = append(authzOpts, docker.WithAuthClient(insecureClient)) + var ho dockerconfig.HostOptions + + ho.HostDir = func(s string) (string, error) { + for _, hostsDir := range o.hostsDirs { + found, err := dockerconfig.HostDirFromRoot(hostsDir)(s) + if (err != nil && !errdefs.IsNotFound(err)) || (found != "") { + return found, err + } + } + return "", nil } - if authCreds, err := NewAuthCreds(refHostname); err != nil { - return nil, err + + if o.authCreds != nil { + ho.Credentials = o.authCreds } else { - authzOpts = append(authzOpts, docker.WithAuthCreds(authCreds)) - } - authz := docker.NewDockerAuthorizer(authzOpts...) - plainHTTPFunc := docker.MatchLocalhost - if o.plainHTTP { - plainHTTPFunc = docker.MatchAllHosts - } - regOpts := []docker.RegistryOpt{ - docker.WithAuthorizer(authz), - docker.WithPlainHTTP(plainHTTPFunc), + if authCreds, err := NewAuthCreds(refHostname); err != nil { + return nil, err + } else { + ho.Credentials = authCreds + } } + if o.skipVerifyCerts { - regOpts = append(regOpts, docker.WithClient(insecureClient)) + ho.DefaultTLS = &tls.Config{ + InsecureSkipVerify: true, + } } - resovlerOpts := docker.ResolverOptions{ - Hosts: docker.ConfigureDefaultRegistries(regOpts...), + + if o.plainHTTP { + ho.DefaultScheme = "http" + } else { + if isLocalHost, err := docker.MatchLocalhost(refHostname); err != nil { + return nil, err + } else if isLocalHost { + ho.DefaultScheme = "http" + } } - resolver := docker.NewResolver(resovlerOpts) - return resolver, nil + return &ho, nil } -func newInsecureClient() *http.Client { - tr := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, +// New instantiates a resolver using $DOCKER_CONFIG/config.json . +// +// $DOCKER_CONFIG defaults to "~/.docker". +// +// refHostname is like "docker.io". +func New(ctx context.Context, refHostname string, optFuncs ...Opt) (remotes.Resolver, error) { + ho, err := NewHostOptions(ctx, refHostname, optFuncs...) + if err != nil { + return nil, err } - return &http.Client{ - Transport: tr, + + resovlerOpts := docker.ResolverOptions{ + Hosts: dockerconfig.ConfigureHosts(ctx, *ho), } + + resolver := docker.NewResolver(resovlerOpts) + return resolver, nil } // AuthCreds is for docker.WithAuthCreds diff --git a/pkg/imgutil/imgutil.go b/pkg/imgutil/imgutil.go index 3a325a96830..5971d3dc9e3 100644 --- a/pkg/imgutil/imgutil.go +++ b/pkg/imgutil/imgutil.go @@ -101,7 +101,9 @@ func GetExistingImage(ctx context.Context, client *containerd.Client, snapshotte // EnsureImage ensures the image. // // When insecure is set, skips verifying certs, and also falls back to HTTP when the registry does not speak HTTPS -func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr io.Writer, snapshotter, rawRef string, mode PullMode, insecure bool, ocispecPlatforms []ocispec.Platform, unpack *bool, quiet bool) (*EnsuredImage, error) { +// +// FIXME: this func has too many args +func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr io.Writer, snapshotter, rawRef string, mode PullMode, insecure bool, hostsDirs []string, ocispecPlatforms []ocispec.Platform, unpack *bool, quiet bool) (*EnsuredImage, error) { switch mode { case "always", "missing", "never": // NOP @@ -134,7 +136,8 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr logrus.Warnf("skipping verifying HTTPS certs for %q", refDomain) dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true)) } - resolver, err := dockerconfigresolver.New(refDomain, dOpts...) + dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(hostsDirs)) + resolver, err := dockerconfigresolver.New(ctx, refDomain, dOpts...) if err != nil { return nil, err } @@ -147,7 +150,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr if insecure { logrus.WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain) dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true)) - resolver, err = dockerconfigresolver.New(refDomain, dOpts...) + resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...) if err != nil { return nil, err } @@ -161,7 +164,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr return img, nil } -func ResolveDigest(ctx context.Context, rawRef string, insecure bool) (string, error) { +func ResolveDigest(ctx context.Context, rawRef string, insecure bool, hostsDirs []string) (string, error) { named, err := refdocker.ParseDockerRef(rawRef) if err != nil { return "", err @@ -174,7 +177,8 @@ func ResolveDigest(ctx context.Context, rawRef string, insecure bool) (string, e logrus.Warnf("skipping verifying HTTPS certs for %q", refDomain) dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true)) } - resolver, err := dockerconfigresolver.New(refDomain, dOpts...) + dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(hostsDirs)) + resolver, err := dockerconfigresolver.New(ctx, refDomain, dOpts...) if err != nil { return "", err } diff --git a/cmd/nerdctl/run_network_test.go b/pkg/testutil/nettestutil/nettestutil.go similarity index 60% rename from cmd/nerdctl/run_network_test.go rename to pkg/testutil/nettestutil/nettestutil.go index 449e4e20650..3662cd0c921 100644 --- a/cmd/nerdctl/run_network_test.go +++ b/pkg/testutil/nettestutil/nettestutil.go @@ -14,17 +14,19 @@ limitations under the License. */ -package main +package nettestutil import ( + "crypto/tls" "fmt" + "net" "net/http" "time" "github.com/containerd/containerd/errdefs" ) -func httpGet(urlStr string, attempts int) (*http.Response, error) { +func HTTPGet(urlStr string, attempts int, insecure bool) (*http.Response, error) { var ( resp *http.Response err error @@ -34,6 +36,11 @@ func httpGet(urlStr string, attempts int) (*http.Response, error) { } client := &http.Client{ Timeout: 3 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecure, + }, + }, } for i := 0; i < attempts; i++ { resp, err = client.Get(urlStr) @@ -44,3 +51,25 @@ func httpGet(urlStr string, attempts int) (*http.Response, error) { } return nil, fmt.Errorf("error after %d attempts: %w", attempts, err) } + +func NonLoopbackIPv4() (net.IP, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return nil, err + } + for _, addr := range addrs { + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + continue + } + ipv4 := ip.To4() + if ipv4 == nil { + continue + } + if ipv4.IsLoopback() { + continue + } + return ipv4, nil + } + return nil, fmt.Errorf("non-loopback IPv4 address not found, attempted=%+v: %w", addrs, errdefs.ErrNotFound) +} diff --git a/pkg/testutil/testca/testca.go b/pkg/testutil/testca/testca.go new file mode 100644 index 00000000000..6b9a9536ddc --- /dev/null +++ b/pkg/testutil/testca/testca.go @@ -0,0 +1,156 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testca + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "testing" + "time" + + "gotest.tools/v3/assert" +) + +type CA struct { + KeyPath string + CertPath string + + t testing.TB + key *rsa.PrivateKey + cert *x509.Certificate + closeF func() error +} + +func (c *CA) Close() error { + return c.closeF() +} + +const keyLength = 4096 + +func New(t testing.TB) *CA { + key, err := rsa.GenerateKey(rand.Reader, keyLength) + assert.NilError(t, err) + + cert := &x509.Certificate{ + SerialNumber: serialNumber(t), + Subject: pkix.Name{ + Organization: []string{"nerdctl test organization"}, + CommonName: fmt.Sprintf("nerdctl CA (%s)", t.Name()), + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + dir, err := os.MkdirTemp(t.TempDir(), "ca") + assert.NilError(t, err) + keyPath := filepath.Join(dir, "ca.key") + certPath := filepath.Join(dir, "ca.cert") + writePair(t, keyPath, certPath, cert, cert, key, key) + + return &CA{ + KeyPath: keyPath, + CertPath: certPath, + t: t, + key: key, + cert: cert, + closeF: func() error { + return os.RemoveAll(dir) + }, + } +} + +type Cert struct { + KeyPath string + CertPath string + closeF func() error +} + +func (c *Cert) Close() error { + return c.closeF() +} + +func (ca *CA) NewCert(host string) *Cert { + t := ca.t + + key, err := rsa.GenerateKey(rand.Reader, keyLength) + assert.NilError(t, err) + + cert := &x509.Certificate{ + SerialNumber: serialNumber(t), + Subject: pkix.Name{ + Organization: []string{"nerdctl test organization"}, + CommonName: fmt.Sprintf("nerdctl %s (%s)", host, t.Name()), + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCRLSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{host}, + } + if ip := net.ParseIP(host); ip != nil { + cert.IPAddresses = append(cert.IPAddresses, ip) + } + + dir, err := os.MkdirTemp(t.TempDir(), "cert") + assert.NilError(t, err) + certPath := filepath.Join(dir, "a.cert") + keyPath := filepath.Join(dir, "a.key") + writePair(t, keyPath, certPath, cert, ca.cert, key, ca.key) + + return &Cert{ + CertPath: certPath, + KeyPath: keyPath, + closeF: func() error { + return os.RemoveAll(dir) + }, + } +} + +func writePair(t testing.TB, keyPath, certPath string, cert, caCert *x509.Certificate, key, caKey *rsa.PrivateKey) { + keyF, err := os.Create(keyPath) + assert.NilError(t, err) + defer keyF.Close() + assert.NilError(t, pem.Encode(keyF, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})) + assert.NilError(t, keyF.Close()) + + certB, err := x509.CreateCertificate(rand.Reader, cert, caCert, &key.PublicKey, caKey) + assert.NilError(t, err) + certF, err := os.Create(certPath) + assert.NilError(t, err) + defer certF.Close() + assert.NilError(t, pem.Encode(certF, &pem.Block{Type: "CERTIFICATE", Bytes: certB})) + assert.NilError(t, certF.Close()) +} + +func serialNumber(t testing.TB) *big.Int { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 60) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + assert.NilError(t, err) + return serialNumber +} diff --git a/pkg/testutil/testregistry/testregistry_linux.go b/pkg/testutil/testregistry/testregistry_linux.go new file mode 100644 index 00000000000..667003a0ff4 --- /dev/null +++ b/pkg/testutil/testregistry/testregistry_linux.go @@ -0,0 +1,179 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testregistry + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strconv" + + "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" + "github.com/containerd/nerdctl/pkg/testutil/testca" + + "golang.org/x/crypto/bcrypt" + "gotest.tools/v3/assert" +) + +type TestRegistry struct { + IP net.IP + ListenIP net.IP + ListenPort int + HostsDir string // contains ":/hosts.toml" + Cleanup func() + Logs func() +} + +func NewPlainHTTP(base *testutil.Base) *TestRegistry { + hostIP, err := nettestutil.NonLoopbackIPv4() + assert.NilError(base.T, err) + // listen on 0.0.0.0 to enable 127.0.0.1 + listenIP := net.ParseIP("0.0.0.0") + const listenPort = 5000 // TODO: choose random empty port + base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d", hostIP, listenIP, listenPort) + + registryContainerName := "reg-" + testutil.Identifier(base.T) + cmd := base.Cmd("run", + "-d", + "-p", fmt.Sprintf("%s:%d:5000", listenIP, listenPort), + "--name", registryContainerName, + testutil.RegistryImage) + cmd.AssertOK() + if _, err = nettestutil.HTTPGet(fmt.Sprintf("http://%s:%d/v2", hostIP.String(), listenPort), 30, false); err != nil { + base.Cmd("rm", "-f", registryContainerName).Run() + base.T.Fatal(err) + } + return &TestRegistry{ + IP: hostIP, + ListenIP: listenIP, + ListenPort: listenPort, + Cleanup: func() { base.Cmd("rm", "-f", registryContainerName).AssertOK() }, + } +} + +func NewHTTPS(base *testutil.Base, user, pass string) *TestRegistry { + name := testutil.Identifier(base.T) + hostIP, err := nettestutil.NonLoopbackIPv4() + assert.NilError(base.T, err) + // listen on 0.0.0.0 to enable 127.0.0.1 + listenIP := net.ParseIP("0.0.0.0") + const listenPort = 5000 // TODO: choose random empty port + const authPort = 5100 // TODO: choose random empty port + base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d, authPort=%d", hostIP, listenIP, listenPort, authPort) + + ca := testca.New(base.T) + registryCert := ca.NewCert(hostIP.String()) + authCert := ca.NewCert(hostIP.String()) + + // Prepare configuration file for authentication server + // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml + authConfigFile, err := os.CreateTemp("", "authconfig") + assert.NilError(base.T, err) + bpass, err := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost) + assert.NilError(base.T, err) + authConfigFileName := authConfigFile.Name() + _, err = authConfigFile.Write([]byte(fmt.Sprintf(` +server: + addr: ":5100" + certificate: "/auth/domain.crt" + key: "/auth/domain.key" +token: + issuer: "Acme auth server" + expiration: 900 +users: + "%s": + password: "%s" +acl: + - match: {account: "%s"} + actions: ["*"] +`, user, string(bpass), user))) + assert.NilError(base.T, err) + + // Run authentication server + authContainerName := "auth-" + name + cmd := base.Cmd("run", + "-d", + "-p", fmt.Sprintf("%s:%d:5100", listenIP, authPort), + "--name", authContainerName, + "-v", authCert.CertPath+":/auth/domain.crt", + "-v", authCert.KeyPath+":/auth/domain.key", + "-v", authConfigFileName+":/config/auth_config.yml", + testutil.DockerAuthImage, + "/config/auth_config.yml") + cmd.AssertOK() + + // Run docker_auth-enabled registry + // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml + registryContainerName := "reg-" + name + cmd = base.Cmd("run", + "-d", + "-p", fmt.Sprintf("%s:%d:5000", listenIP, listenPort), + "--name", registryContainerName, + "--env", "REGISTRY_AUTH=token", + "--env", "REGISTRY_AUTH_TOKEN_REALM="+fmt.Sprintf("https://%s:%d/auth", hostIP.String(), authPort), + "--env", "REGISTRY_AUTH_TOKEN_SERVICE=Docker registry", + "--env", "REGISTRY_AUTH_TOKEN_ISSUER=Acme auth server", + "--env", "REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE=/auth/domain.crt", + "--env", "REGISTRY_HTTP_TLS_CERTIFICATE=/registry/domain.crt", + "--env", "REGISTRY_HTTP_TLS_KEY=/registry/domain.key", + // rootcertbundle is not CA cert: https://github.com/distribution/distribution/issues/1143 + "-v", authCert.CertPath+":/auth/domain.crt", + "-v", registryCert.CertPath+":/registry/domain.crt", + "-v", registryCert.KeyPath+":/registry/domain.key", + testutil.RegistryImage) + cmd.AssertOK() + joined := net.JoinHostPort(hostIP.String(), strconv.Itoa(listenPort)) + if _, err = nettestutil.HTTPGet(fmt.Sprintf("https://%s/v2", joined), 30, true); err != nil { + base.Cmd("rm", "-f", registryContainerName).Run() + base.T.Fatal(err) + } + hostsDir, err := os.MkdirTemp(base.T.TempDir(), "certs.d") + assert.NilError(base.T, err) + hostsSubDir := filepath.Join(hostsDir, joined) + err = os.MkdirAll(hostsSubDir, 0700) + assert.NilError(base.T, err) + hostsTOMLPath := filepath.Join(hostsSubDir, "hosts.toml") + // See https://github.com/containerd/containerd/blob/main/docs/hosts.md + hostsTOML := fmt.Sprintf(` +server = "https://%s" +[host."https://%s"] + ca = %q + `, joined, joined, ca.CertPath) + base.T.Logf("Writing %q: %q", hostsTOMLPath, hostsTOML) + err = os.WriteFile(hostsTOMLPath, []byte(hostsTOML), 0700) + assert.NilError(base.T, err) + return &TestRegistry{ + IP: hostIP, + ListenIP: listenIP, + ListenPort: listenPort, + HostsDir: hostsDir, + Cleanup: func() { + base.Cmd("rm", "-f", registryContainerName).AssertOK() + base.Cmd("rm", "-f", authContainerName).AssertOK() + assert.NilError(base.T, registryCert.Close()) + assert.NilError(base.T, authCert.Close()) + assert.NilError(base.T, authConfigFile.Close()) + os.Remove(authConfigFileName) + }, + Logs: func() { + base.T.Logf("%s: %q", registryContainerName, base.Cmd("logs", registryContainerName).Run().String()) + base.T.Logf("%s: %q", authContainerName, base.Cmd("logs", authContainerName).Run().String()) + }, + } +}