From 635519f2622f7e753ab4d01d8272ecf6cea0e8a4 Mon Sep 17 00:00:00 2001 From: Jorge Morales Pou Date: Thu, 15 Jan 2026 11:34:41 +0100 Subject: [PATCH] Fixed ips for local registry and mirrors --- .../pkg/cmd/local_cluster_create_cmd.go | 113 +++----- client-programs/pkg/registry/network.go | 245 ++++++++++++++++++ client-programs/pkg/registry/registry.go | 64 +++-- project-docs/release-notes/version-3.5.2.md | 13 + 4 files changed, 334 insertions(+), 101 deletions(-) create mode 100644 client-programs/pkg/registry/network.go create mode 100644 project-docs/release-notes/version-3.5.2.md diff --git a/client-programs/pkg/cmd/local_cluster_create_cmd.go b/client-programs/pkg/cmd/local_cluster_create_cmd.go index 876e4392..7157a8a6 100644 --- a/client-programs/pkg/cmd/local_cluster_create_cmd.go +++ b/client-programs/pkg/cmd/local_cluster_create_cmd.go @@ -1,21 +1,17 @@ package cmd import ( - "context" _ "embed" "fmt" - "io" - "os" + "net" + "strconv" + "time" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/image" - "github.com/docker/go-connections/nat" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/educates/educates-training-platform/client-programs/pkg/cluster" "github.com/educates/educates-training-platform/client-programs/pkg/config" - "github.com/educates/educates-training-platform/client-programs/pkg/docker" "github.com/educates/educates-training-platform/client-programs/pkg/installer" "github.com/educates/educates-training-platform/client-programs/pkg/registry" "github.com/educates/educates-training-platform/client-programs/pkg/secrets" @@ -82,13 +78,9 @@ func (o *LocalClusterCreateOptions) Run() error { return err } - httpAvailable, err := checkPortAvailability(fullConfig.LocalKindCluster.ListenAddress, []uint{80, 443}, o.Verbose) + available := checkPortAvailability(fullConfig.LocalKindCluster.ListenAddress, []uint{80, 443}, o.Verbose) - if err != nil { - return errors.Wrap(err, "couldn't test whether ports 80/443 available") - } - - if !httpAvailable { + if !available { return errors.New("ports 80/443 not available") } @@ -112,7 +104,12 @@ func (o *LocalClusterCreateOptions) Run() error { } } - if err = registry.DeployRegistryAndLinkToCluster(o.RegistryBindIP, client); err != nil { + localRegistryIP, err := registry.ResolveLocalRegistryIP() + if err != nil { + return errors.Wrap(err, "failed to resolve local registry IP") + } + + if err = registry.DeployRegistryAndLinkToCluster(o.RegistryBindIP, localRegistryIP, client); err != nil { return errors.Wrap(err, "failed to deploy registry") } @@ -134,6 +131,10 @@ func (o *LocalClusterCreateOptions) Run() error { } if !o.ClusterOnly { + if !o.SkipImageResolution && !isImageResolutionPossible() { + fmt.Println("🔴 No network connectivity detected; skipping image resolution") + o.SkipImageResolution = true + } installer := installer.NewInstaller() err = installer.Run(o.Version, o.PackageRepository, fullConfig, &clusterConfig.Config, o.Verbose, false, o.SkipImageResolution, false) if err != nil { @@ -228,31 +229,10 @@ func (p *ProjectInfo) NewLocalClusterCreateCmd() *cobra.Command { return c } -func checkPortAvailability(listenAddress string, ports []uint, verbose bool) (bool, error) { - ctx := context.Background() - - cli, err := docker.NewDockerClient() - - if err != nil { - return false, errors.Wrap(err, "unable to create docker client") - } - - cli.ContainerRemove(ctx, "educates-port-availability-check", container.RemoveOptions{}) - - reader, err := cli.ImagePull(ctx, "docker.io/library/busybox:latest", image.PullOptions{}) - if err != nil { - return false, errors.Wrap(err, "cannot pull busybox image") - } - - defer reader.Close() - - if verbose { - io.Copy(os.Stdout, reader) - } else { - io.Copy(io.Discard, reader) - } - +func checkPortAvailability(listenAddress string, ports []uint, verbose bool) bool { + // Handle empty address default if listenAddress == "" { + var err error listenAddress, err = config.HostIP() if err != nil { @@ -260,49 +240,32 @@ func checkPortAvailability(listenAddress string, ports []uint, verbose bool) (bo } } - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{}, - } - - exposedPorts := nat.PortSet{} - for _, port := range ports { - key := nat.Port(fmt.Sprintf("%d/tcp", port)) - hostConfig.PortBindings[key] = []nat.PortBinding{ - { - HostIP: listenAddress, - HostPort: fmt.Sprintf("%d", port), - }, - } - exposedPorts[key] = struct{}{} - } + // Format the address:port string + address := net.JoinHostPort(listenAddress, strconv.Itoa(int(port))) - resp, err := cli.ContainerCreate(ctx, &container.Config{ - Image: "docker.io/library/busybox:latest", - Cmd: []string{"/bin/true"}, - Tty: false, - ExposedPorts: exposedPorts, - }, hostConfig, nil, nil, "educates-port-availability-check") + // Try to create a server listener + listener, err := net.Listen("tcp", address) + if err != nil { + // If we get an error, the port is likely in use (or we lack permission) + return false + } - if err != nil { - return false, errors.Wrap(err, "cannot create busybox container") + // Important: Close the listener immediately so we don't hog the port! + listener.Close() } - defer cli.ContainerRemove(ctx, "educates-port-availability-check", container.RemoveOptions{}) - - if err := cli.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil { - return false, errors.Wrap(err, "cannot start busybox container") - } + return true +} - statusCh, errCh := cli.ContainerWait(ctx, "educates-port-availability-check", container.WaitConditionNotRunning) +func isImageResolutionPossible() bool { + timeout := 2 * time.Second + target := net.JoinHostPort("registry-1.docker.io", "443") - select { - case err := <-errCh: - if err != nil { - return false, nil - } - case <-statusCh: + conn, err := net.DialTimeout("tcp", target, timeout) + if err != nil { + return false } - - return true, nil + conn.Close() + return true } diff --git a/client-programs/pkg/registry/network.go b/client-programs/pkg/registry/network.go new file mode 100644 index 00000000..120057a5 --- /dev/null +++ b/client-programs/pkg/registry/network.go @@ -0,0 +1,245 @@ +package registry + +import ( + "context" + "encoding/binary" + "fmt" + "net/netip" + + "github.com/docker/docker/api/types/network" + "github.com/pkg/errors" + + "github.com/educates/educates-training-platform/client-programs/pkg/docker" +) + +const ( + dockerNetworkFixedIPOffsetBase = 200 * 256 + localRegistryIPOffset = dockerNetworkFixedIPOffsetBase + 1 + localMirrorIPOffsetStart = dockerNetworkFixedIPOffsetBase + 2 + localMirrorIPOffsetRange = 200 +) + +func ResolveLocalRegistryIP() (string, error) { + ctx := context.Background() + + cli, err := docker.NewDockerClient() + if err != nil { + return "", errors.Wrap(err, "unable to create docker client") + } + + networkInfo, err := cli.NetworkInspect(ctx, KindNetworkName, network.InspectOptions{}) + if err != nil { + return "", errors.Wrap(err, "unable to inspect kind network") + } + + prefix, gateway, err := dockerNetworkIPv4Prefix(KindNetworkName, networkInfo) + if err != nil { + return "", err + } + + registryIP, err := fixedIPForOffset(KindNetworkName, prefix, gateway, networkInfo.Containers, localRegistryIPOffset, EducatesRegistryContainer) + if err != nil { + return "", errors.Wrap(err, "unable to resolve fixed kind IP for registry") + } + + return registryIP.String(), nil +} + +func ResolveLocalMirrorIP(containerName string) (string, error) { + ctx := context.Background() + + cli, err := docker.NewDockerClient() + if err != nil { + return "", errors.Wrap(err, "unable to create docker client") + } + + networkInfo, err := cli.NetworkInspect(ctx, KindNetworkName, network.InspectOptions{}) + if err != nil { + return "", errors.Wrap(err, "unable to inspect kind network") + } + + prefix, gateway, err := dockerNetworkIPv4Prefix(KindNetworkName, networkInfo) + if err != nil { + return "", err + } + + offset, err := mirrorOffsetForContainer(containerName) + if err != nil { + return "", err + } + + for i := uint32(0); i < localMirrorIPOffsetRange; i++ { + candidateOffset := localMirrorIPOffsetStart + ((offset + i) % localMirrorIPOffsetRange) + if candidateOffset == localRegistryIPOffset { + continue + } + + candidateIP, available, err := candidateFixedIP(KindNetworkName, prefix, gateway, networkInfo.Containers, candidateOffset, containerName) + if err != nil { + return "", err + } + if !available { + continue + } + return candidateIP.String(), nil + } + + return "", errors.New("unable to allocate fixed kind IP for mirror") +} + +func EnsureContainerKindNetworkIP(containerName string, fixedIP string) error { + ctx := context.Background() + + cli, err := docker.NewDockerClient() + if err != nil { + return errors.Wrap(err, "unable to create docker client") + } + + containerInfo, err := cli.ContainerInspect(ctx, containerName) + if err != nil { + return errors.Wrap(err, "unable to inspect container") + } + + if kindNetwork, exists := containerInfo.NetworkSettings.Networks[KindNetworkName]; exists { + if fixedIP == "" || kindNetwork.IPAddress == fixedIP { + return nil + } + } + + cli.NetworkDisconnect(ctx, KindNetworkName, containerName, false) + + endpointSettings := &network.EndpointSettings{} + if fixedIP != "" { + endpointSettings.IPAddress = fixedIP + endpointSettings.IPAMConfig = &network.EndpointIPAMConfig{ + IPv4Address: fixedIP, + } + } + + if err := cli.NetworkConnect(ctx, KindNetworkName, containerName, endpointSettings); err != nil { + return errors.Wrapf(err, "unable to connect container to %s network", KindNetworkName) + } + + return nil +} +func dockerNetworkIPv4Prefix(networkName string, networkInfo network.Inspect) (netip.Prefix, *netip.Addr, error) { + for _, config := range networkInfo.IPAM.Config { + if config.Subnet == "" { + continue + } + + prefix, err := netip.ParsePrefix(config.Subnet) + if err != nil || !prefix.Addr().Is4() { + continue + } + + var gateway *netip.Addr + if config.Gateway != "" { + if addr, err := netip.ParseAddr(config.Gateway); err == nil && addr.Is4() { + gateway = &addr + } + } + + return prefix.Masked(), gateway, nil + } + + return netip.Prefix{}, nil, errors.Errorf( "%s network has no IPv4 subnet", networkName) +} + +func fixedIPForOffset(networkName string, prefix netip.Prefix, gateway *netip.Addr, containers map[string]network.EndpointResource, offset uint32, allowedContainerName string) (netip.Addr, error) { + addr, available, err := candidateFixedIP(networkName, prefix, gateway, containers, offset, allowedContainerName) + if err != nil { + return netip.Addr{}, err + } + if !available { + return netip.Addr{}, fmt.Errorf("%s network already uses fixed IP %s", networkName, addr.String()) + } + return addr, nil +} + +func candidateFixedIP(networkName string, prefix netip.Prefix, gateway *netip.Addr, containers map[string]network.EndpointResource, offset uint32, allowedContainerName string) (netip.Addr, bool, error) { + base := prefix.Addr() + if !base.Is4() { + return netip.Addr{}, false, errors.New("kind network base is not IPv4") + } + + addr, err := addIPv4Offset(base, offset) + if err != nil { + return netip.Addr{}, false, err + } + + if !prefix.Contains(addr) { + return netip.Addr{}, false, fmt.Errorf("%s network does not include fixed IP %s", networkName, addr.String()) + } + + if gateway != nil && *gateway == addr { + return netip.Addr{}, false, fmt.Errorf("%s network gateway conflicts with fixed IP %s", networkName, addr.String()) + } + + if containerName, inUse := containerNameForIP(containers, addr); inUse { + if allowedContainerName != "" && containerName == allowedContainerName { + return addr, true, nil + } + return addr, false, nil + } + + return addr, true, nil +} + +func addIPv4Offset(base netip.Addr, offset uint32) (netip.Addr, error) { + if !base.Is4() { + return netip.Addr{}, errors.New("base address is not IPv4") + } + + baseBytes := base.As4() + baseValue := binary.BigEndian.Uint32(baseBytes[:]) + targetValue := baseValue + offset + + if targetValue < baseValue { + return netip.Addr{}, errors.New("fixed IP offset overflows IPv4 range") + } + + var targetBytes [4]byte + binary.BigEndian.PutUint32(targetBytes[:], targetValue) + + return netip.AddrFrom4(targetBytes), nil +} + + +func containerNameForIP(containers map[string]network.EndpointResource, addr netip.Addr) (string, bool) { + for _, container := range containers { + if container.IPv4Address == "" { + continue + } + + parsed, err := netip.ParsePrefix(container.IPv4Address) + if err != nil { + continue + } + + if parsed.Addr() == addr { + return container.Name, true + } + } + + return "", false +} + +func mirrorOffsetForContainer(containerName string) (uint32, error) { + hash := fnv32a(containerName) + return hash % localMirrorIPOffsetRange, nil +} + +func fnv32a(value string) uint32 { + const ( + offset32 = 2166136261 + prime32 = 16777619 + ) + + hash := uint32(offset32) + for i := 0; i < len(value); i++ { + hash ^= uint32(value[i]) + hash *= prime32 + } + return hash +} diff --git a/client-programs/pkg/registry/registry.go b/client-programs/pkg/registry/registry.go index 8ed27cf7..0df930d7 100644 --- a/client-programs/pkg/registry/registry.go +++ b/client-programs/pkg/registry/registry.go @@ -38,6 +38,8 @@ const hostRegistryTomlTemplate = `[host."http://%s:5000"]` const ( RegistryImageV3 = "docker.io/library/registry:3" RegistryConfigTargetPath = "/etc/distribution/config.yml" + KindNetworkName = "kind" + // Used for docker-based workshop deployments to discover the local registry. EducatesNetworkName = "educates" EducatesRegistryContainer = "educates-registry" EducatesControlPlaneContainer = "educates-control-plane" @@ -50,9 +52,9 @@ const ( * This function is used to deploy the registry and link it to the cluster. * It is used when creating a new local cluster. */ -func DeployRegistryAndLinkToCluster(bindIP string, client *kubernetes.Clientset) error { +func DeployRegistryAndLinkToCluster(bindIP string, kindRegistryIP string, client *kubernetes.Clientset) error { - err := createRegistryContainer(bindIP) + err := createRegistryContainer(bindIP, kindRegistryIP) if err != nil { return errors.Wrap(err, "failed to deploy registry") } @@ -80,7 +82,7 @@ func DeployRegistryAndLinkToCluster(bindIP string, client *kubernetes.Clientset) * It will not link the registry to the cluster. */ func DeployRegistry(bindIP string) error { - err := createRegistryContainer(bindIP) + err := createRegistryContainer(bindIP, "") if err != nil { return errors.Wrap(err, "failed to deploy registry") } @@ -91,7 +93,7 @@ func DeployRegistry(bindIP string) error { /** * This private function only creates the registry container. */ -func createRegistryContainer(bindIP string) error { +func createRegistryContainer(bindIP string, kindRegistryIP string) error { ctx := context.Background() fmt.Println("Deploying local image registry") @@ -175,7 +177,7 @@ func createRegistryContainer(bindIP string) error { return errors.Wrap(err, "unable to connect registry to educates network") } - if err = linkRegistryToClusterNetwork(EducatesRegistryContainer); err != nil { + if err = linkRegistryToClusterNetwork(EducatesRegistryContainer, kindRegistryIP); err != nil { return errors.Wrap(err, "failed to link registry to cluster") } @@ -226,7 +228,13 @@ func createMirrorContainer(mirrorConfig *config.RegistryMirrorConfig) error { // have exited and container was not removed, but if that is the case // then leave it up to the user to sort out. fmt.Printf("Registry mirror %s already exists\n", mirrorConfig.Mirror) - + kindMirrorIP, err := ResolveLocalMirrorIP(mirrorContainerName) + if err != nil { + return errors.Wrap(err, "failed to resolve fixed kind IP for local registry mirror") + } + if err = linkRegistryToClusterNetwork(mirrorContainerName, kindMirrorIP); err != nil { + return errors.Wrap(err, "failed to link local registry mirror to cluster") + } return nil } @@ -300,7 +308,12 @@ func createMirrorContainer(mirrorConfig *config.RegistryMirrorConfig) error { return errors.Wrap(err, "unable to connect local registry mirror to educates network") } - if err = linkRegistryToClusterNetwork(mirrorContainerName); err != nil { + kindMirrorIP, err := ResolveLocalMirrorIP(mirrorContainerName) + if err != nil { + return errors.Wrap(err, "failed to resolve fixed kind IP for local registry mirror") + } + + if err = linkRegistryToClusterNetwork(mirrorContainerName, kindMirrorIP); err != nil { return errors.Wrap(err, "failed to link local registry mirror to cluster") } @@ -447,25 +460,11 @@ func documentLocalRegistry(client *kubernetes.Clientset) error { * This function is used to link the registry to the cluster network, which is the kind network. * It is used when creating a new local registry or registry mirror containers. */ -func linkRegistryToClusterNetwork(containerName string) error { - ctx := context.Background() - +func linkRegistryToClusterNetwork(containerName string, kindFixedIP string) error { fmt.Println("Linking local image registry to cluster") - - cli, err := docker.NewDockerClient() - - if err != nil { - return errors.Wrap(err, "unable to create docker client") - } - - cli.NetworkDisconnect(ctx, "kind", containerName, false) - - err = cli.NetworkConnect(ctx, "kind", containerName, &network.EndpointSettings{}) - - if err != nil { + if err := EnsureContainerKindNetworkIP(containerName, kindFixedIP); err != nil { return errors.Wrap(err, "unable to connect registry to cluster network") } - return nil } @@ -647,10 +646,23 @@ func UpdateRegistryK8SService(k8sclient *kubernetes.Clientset) error { return errors.Wrapf(err, "unable to inspect container for registry") } - kindNetwork, exists := registryInfo.NetworkSettings.Networks["kind"] - + kindNetwork, exists := registryInfo.NetworkSettings.Networks[KindNetworkName] if !exists { - return errors.New("registry is not attached to kind network") + kindRegistryIP, err := ResolveLocalRegistryIP() + if err != nil { + return errors.Wrap(err, "failed to resolve kind registry IP") + } + if err := EnsureContainerKindNetworkIP(EducatesRegistryContainer, kindRegistryIP); err != nil { + return errors.Wrap(err, "failed to attach registry to kind network") + } + registryInfo, err = cli.ContainerInspect(ctx, EducatesRegistryContainer) + if err != nil { + return errors.Wrapf(err, "unable to inspect container for registry after reattach") + } + kindNetwork, exists = registryInfo.NetworkSettings.Networks[KindNetworkName] + if !exists { + return errors.New("registry is not attached to kind network") + } } endpointAddresses := []string{kindNetwork.IPAddress} diff --git a/project-docs/release-notes/version-3.5.2.md b/project-docs/release-notes/version-3.5.2.md new file mode 100644 index 00000000..078c6211 --- /dev/null +++ b/project-docs/release-notes/version-3.5.2.md @@ -0,0 +1,13 @@ +Version 3.5.2 +============= + +Features Changed +---------------- + +* Local registry and local mirrors now get a fixed IP on the kind network + for better support Docker Desktop restarts. + +* Initial check of port 80/443 are now done in a simpler (more performant) way + +* When local cluster detects a disconnected install it will skip-image-resolution + automatically. A message will be printed in the output.