From 8916e773a2b147f375621f9026b5f756b3165f28 Mon Sep 17 00:00:00 2001 From: Caleb Boylan Date: Fri, 29 Aug 2025 17:42:01 -0700 Subject: [PATCH] fix: fix the registry functionality (#542) Signed-off-by: Caleb Boylan --- pkg/kind/cluster.go | 10 +++++++ pkg/kind/cluster_test.go | 34 ++++++++++++++---------- pkg/kind/config.go | 42 ++++++++++++++++++++++++++++++ pkg/kind/resources/hosts.toml.tmpl | 13 +++++++++ pkg/kind/resources/kind.yaml.tmpl | 17 ++++-------- 5 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 pkg/kind/resources/hosts.toml.tmpl diff --git a/pkg/kind/cluster.go b/pkg/kind/cluster.go index 3ab82586a..f8bde7879 100644 --- a/pkg/kind/cluster.go +++ b/pkg/kind/cluster.go @@ -55,11 +55,20 @@ type IProvider interface { func (c *Cluster) getConfig() ([]byte, error) { rawConfigTempl, err := loadConfig(c.kindConfigPath, c.httpClient) + if err != nil { + return nil, fmt.Errorf("loading config template: %w", err) + } portMappingPairs := parsePortMappings(c.extraPortsMapping) registryConfig := findRegistryConfig(c.registryConfig) + registryCertsDir, err := renderRegistryCertsDir(c.cfg) + + if err != nil { + return nil, fmt.Errorf("rendering insecure registry config: %w", err) + } + if len(c.registryConfig) > 0 && registryConfig == "" { return nil, errors.New("--registry-config flag used but no registry config was found") } @@ -70,6 +79,7 @@ func (c *Cluster) getConfig() ([]byte, error) { KubernetesVersion: c.kubeVersion, ExtraPortsMapping: portMappingPairs, RegistryConfig: registryConfig, + RegistryCertsDir: registryCertsDir, }); err != nil { return nil, err } diff --git a/pkg/kind/cluster_test.go b/pkg/kind/cluster_test.go index 753a88f6e..20248b768 100644 --- a/pkg/kind/cluster_test.go +++ b/pkg/kind/cluster_test.go @@ -12,10 +12,13 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "regexp" "sigs.k8s.io/kind/pkg/cluster/nodes" "sigs.k8s.io/kind/pkg/exec" ) +var re = regexp.MustCompile(`(.*?)hostPath: /tmp/idpbuilder-registry-certs.d-.*(.*?)`) + func TestGetConfig(t *testing.T) { type tc struct { @@ -47,12 +50,12 @@ nodes: - containerPort: 32222 hostPort: 32222 protocol: TCP + extraMounts: + - containerPath: /etc/containerd/certs.d containerdConfigPatches: - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."gitea.cnoe.localtest.me:8443"] - endpoint = ["https://gitea.cnoe.localtest.me"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."gitea.cnoe.localtest.me".tls] - insecure_skip_verify = true`, + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d"`, }, { host: "cnoe.localtest.me", @@ -75,14 +78,13 @@ nodes: hostPort: 32222 protocol: TCP extraMounts: + - containerPath: /etc/containerd/certs.d - containerPath: /var/lib/kubelet/config.json hostPath: testdata/empty.json containerdConfigPatches: - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."cnoe.localtest.me:8443"] - endpoint = ["https://cnoe.localtest.me"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."cnoe.localtest.me".tls] - insecure_skip_verify = true`, + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d"`, }, } @@ -97,7 +99,9 @@ containerdConfigPatches: cfg, err := cluster.getConfig() assert.NoError(t, err) - assert.YAMLEq(t, c.expectConfig, string(cfg)) + expectStripped := re.ReplaceAllString(c.expectConfig, `$1$2`) + cfgStripped := re.ReplaceAllString(string(cfg), `$1$2`) + assert.YAMLEq(t, expectStripped, cfgStripped) } } @@ -134,14 +138,16 @@ nodes: - containerPort: 32222 hostPort: 22 protocol: TCP + extraMounts: + - containerPath: /etc/containerd/certs.d containerdConfigPatches: - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."gitea.cnoe.localtest.me:8443"] - endpoint = ["https://gitea.cnoe.localtest.me"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."gitea.cnoe.localtest.me".tls] - insecure_skip_verify = true` + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d"` - assert.YAMLEq(t, expectConfig, string(cfg)) + expectStripped := re.ReplaceAllString(expectConfig, `$1$2`) + cfgStripped := re.ReplaceAllString(string(cfg), `$1$2`) + assert.YAMLEq(t, expectStripped, cfgStripped) } func TestGetConfigCustom(t *testing.T) { diff --git a/pkg/kind/config.go b/pkg/kind/config.go index 76dfd3702..06c874993 100644 --- a/pkg/kind/config.go +++ b/pkg/kind/config.go @@ -6,9 +6,11 @@ import ( "io" "io/fs" "os" + "path/filepath" "strings" "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/cnoe-io/idpbuilder/pkg/util/files" ) type PortMapping struct { @@ -21,6 +23,7 @@ type TemplateConfig struct { KubernetesVersion string ExtraPortsMapping []PortMapping RegistryConfig string + RegistryCertsDir string } //go:embed resources/* testdata/custom-kind.yaml.tmpl @@ -83,3 +86,42 @@ func findRegistryConfig(registryConfigPaths []string) string { } return "" } + +func renderRegistryCertsDir(cfg v1alpha1.BuildCustomizationSpec) (string, error) { + // Render out the template + rawConfigTempl, err := fs.ReadFile(configFS, "resources/hosts.toml.tmpl") + if err != nil { + return "", fmt.Errorf("reading insecure registry config %w", err) + } + + var retBuff []byte + if retBuff, err = files.ApplyTemplate(rawConfigTempl, cfg); err != nil { + return "", fmt.Errorf("templating insecure registry config %w", err) + } + + // Generate the directory structure and write the file to hosts.toml + dir, err := os.MkdirTemp("", "idpbuilder-registry-certs.d-*") + if err != nil { + return "", fmt.Errorf("creating temp dir %w", err) + } + + var hostAndPort string + if cfg.UsePathRouting { + hostAndPort = fmt.Sprintf("%s:%s", cfg.Host, cfg.Port) + } else { + hostAndPort = fmt.Sprintf("gitea.%s:%s", cfg.Host, cfg.Port) + } + hostCertsDir := filepath.Join(dir, hostAndPort) + err = os.Mkdir(hostCertsDir, 0700) + if err != nil { + return "", fmt.Errorf("creating temp dir for host %w", err) + } + hostsFile := filepath.Join(hostCertsDir, "hosts.toml") + + err = os.WriteFile(hostsFile, retBuff, 0700) + if err != nil { + return "", fmt.Errorf("writing insecure registry config %w", err) + } + + return dir, nil +} diff --git a/pkg/kind/resources/hosts.toml.tmpl b/pkg/kind/resources/hosts.toml.tmpl new file mode 100644 index 000000000..4ca5d1477 --- /dev/null +++ b/pkg/kind/resources/hosts.toml.tmpl @@ -0,0 +1,13 @@ +{{ if .UsePathRouting -}} +server = "https://{{ .Host }}:{{ .Port }}" + +[host."https://{{ .Host }}"] + capabilities = ["pull", "resolve"] + skip_verify = true +{{ else -}} +server = "https://gitea.{{ .Host }}:{{ .Port }}" + +[host."https://gitea.{{ .Host }}"] + capabilities = ["pull", "resolve"] + skip_verify = true +{{ end -}} diff --git a/pkg/kind/resources/kind.yaml.tmpl b/pkg/kind/resources/kind.yaml.tmpl index a48f15ffa..51a6e88aa 100644 --- a/pkg/kind/resources/kind.yaml.tmpl +++ b/pkg/kind/resources/kind.yaml.tmpl @@ -20,21 +20,14 @@ nodes: hostPort: {{ .HostPort }} protocol: TCP {{- end }} -{{- if .RegistryConfig }} extraMounts: + - containerPath: /etc/containerd/certs.d + hostPath: {{ .RegistryCertsDir }} +{{- if .RegistryConfig }} - containerPath: /var/lib/kubelet/config.json hostPath: {{ .RegistryConfig }} {{- end }} containerdConfigPatches: - |- - {{ if .UsePathRouting -}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ .Host }}:{{ .Port }}"] - endpoint = ["https://{{ .Host }}"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .Host }}".tls] - insecure_skip_verify = true - {{- else -}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."gitea.{{ .Host }}:{{ .Port }}"] - endpoint = ["https://gitea.{{ .Host }}"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."gitea.{{ .Host }}".tls] - insecure_skip_verify = true - {{- end -}} + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d"