From ce7072f6ec81489e50a081c1a7317922efa7b641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Min=C3=A1=C5=99?= Date: Tue, 27 Jun 2017 15:06:07 +0200 Subject: [PATCH 1/2] image-pruner: Determine protocol just once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Determine the registry protocol once. Do not change to other protocol during the run. This will produce nicer output without unrelated protocol fallback errors. Do not default to insecure connection when the --registry-url is empty. Move registry client initialization just before the start of the pruner - so we can precisely determine whether to allow for insecure fall-back based on collected images and image streams. Move ping() outside of pruner. Instead, determine the registry URL before the pruner starts and assume it won't change during the run. Signed-off-by: Michal Minář --- pkg/image/prune/doc.go | 2 + pkg/image/prune/helper.go | 208 ++++++++++++ pkg/image/prune/helper_test.go | 217 +++++++++++++ pkg/image/prune/prune.go | 301 ++++------------- pkg/image/prune/prune_test.go | 524 +++++++++++++++++------------- pkg/oc/admin/prune/images.go | 143 +++++--- pkg/oc/admin/prune/images_test.go | 6 +- 7 files changed, 892 insertions(+), 509 deletions(-) create mode 100644 pkg/image/prune/doc.go create mode 100644 pkg/image/prune/helper.go create mode 100644 pkg/image/prune/helper_test.go diff --git a/pkg/image/prune/doc.go b/pkg/image/prune/doc.go new file mode 100644 index 000000000000..f910671d2c07 --- /dev/null +++ b/pkg/image/prune/doc.go @@ -0,0 +1,2 @@ +// Package prune contains logic for pruning images and interoperating with the integrated Docker registry. +package prune diff --git a/pkg/image/prune/helper.go b/pkg/image/prune/helper.go new file mode 100644 index 000000000000..57380919bdaf --- /dev/null +++ b/pkg/image/prune/helper.go @@ -0,0 +1,208 @@ +package prune + +import ( + "fmt" + "net/http" + "net/url" + "sort" + "strings" + + "github.com/docker/distribution/registry/api/errcode" + "github.com/golang/glog" + + kerrors "k8s.io/apimachinery/pkg/util/errors" + + imageapi "github.com/openshift/origin/pkg/image/apis/image" + "github.com/openshift/origin/pkg/util/netutils" +) + +// order younger images before older +type imgByAge []*imageapi.Image + +func (ba imgByAge) Len() int { return len(ba) } +func (ba imgByAge) Swap(i, j int) { ba[i], ba[j] = ba[j], ba[i] } +func (ba imgByAge) Less(i, j int) bool { + return ba[i].CreationTimestamp.After(ba[j].CreationTimestamp.Time) +} + +// order younger image stream before older +type isByAge []imageapi.ImageStream + +func (ba isByAge) Len() int { return len(ba) } +func (ba isByAge) Swap(i, j int) { ba[i], ba[j] = ba[j], ba[i] } +func (ba isByAge) Less(i, j int) bool { + return ba[i].CreationTimestamp.After(ba[j].CreationTimestamp.Time) +} + +// DetermineRegistryHost returns registry host embedded in a pull-spec of the latest unmanaged image or the +// latest imagestream from the provided lists. If no such pull-spec is found, error is returned. +func DetermineRegistryHost(images *imageapi.ImageList, imageStreams *imageapi.ImageStreamList) (string, error) { + var pullSpec string + var managedImages []*imageapi.Image + + // 1st try to determine registry url from a pull spec of the youngest managed image + for i := range images.Items { + image := &images.Items[i] + if image.Annotations[imageapi.ManagedByOpenShiftAnnotation] != "true" { + continue + } + managedImages = append(managedImages, image) + } + // be sure to pick up the newest managed image which should have an up to date information + sort.Sort(imgByAge(managedImages)) + + if len(managedImages) > 0 { + pullSpec = managedImages[0].DockerImageReference + } else { + // 2nd try to get the pull spec from any image stream + // Sorting by creation timestamp may not get us up to date info. Modification time would be much + // better if there were such an attribute. + sort.Sort(isByAge(imageStreams.Items)) + for _, is := range imageStreams.Items { + if len(is.Status.DockerImageRepository) == 0 { + continue + } + pullSpec = is.Status.DockerImageRepository + } + } + + if len(pullSpec) == 0 { + return "", fmt.Errorf("no managed image found") + } + + ref, err := imageapi.ParseDockerImageReference(pullSpec) + if err != nil { + return "", fmt.Errorf("unable to parse %q: %v", pullSpec, err) + } + + if len(ref.Registry) == 0 { + return "", fmt.Errorf("%s does not include a registry", pullSpec) + } + + return ref.Registry, nil +} + +// RegistryPinger performs a health check against a registry. +type RegistryPinger interface { + // Ping performs a health check against registry. It returns registry url qualified with schema unless an + // error occurs. + Ping(registry string) (*url.URL, error) +} + +// DefaultRegistryPinger implements RegistryPinger. +type DefaultRegistryPinger struct { + Client *http.Client + Insecure bool +} + +// Ping verifies that the integrated registry is ready, determines its transport protocol and returns its url +// or error. +func (drp *DefaultRegistryPinger) Ping(registry string) (*url.URL, error) { + var ( + registryURL *url.URL + err error + ) + +pathLoop: + // first try the new default / path, then fall-back to the obsolete /healthz endpoint + for _, path := range []string{"/", "/healthz"} { + registryURL, err = TryProtocolsWithRegistryURL(registry, drp.Insecure, func(u url.URL) error { + u.Path = path + healthResponse, err := drp.Client.Get(u.String()) + if err != nil { + return err + } + defer healthResponse.Body.Close() + + if healthResponse.StatusCode != http.StatusOK { + return &retryPath{err: fmt.Errorf("unexpected status: %s", healthResponse.Status)} + } + + return nil + }) + + // determine whether to retry with another endpoint + switch t := err.(type) { + case *retryPath: + // return the nested error if this is the last ping attempt + err = t.err + continue pathLoop + case kerrors.Aggregate: + // if any aggregated error indicates a possible retry, do it + for _, err := range t.Errors() { + if _, ok := err.(*retryPath); ok { + continue pathLoop + } + } + } + + break + } + + return registryURL, err +} + +// DryRunRegistryPinger implements RegistryPinger. +type DryRunRegistryPinger struct { +} + +// Ping implements Ping method. +func (*DryRunRegistryPinger) Ping(registry string) (*url.URL, error) { + return url.Parse("https://" + registry) +} + +// TryProtocolsWithRegistryURL runs given action with different protocols until no error is returned. The +// https protocol is the first attempt. If it fails and allowInsecure is true, http will be the next. Obtained +// errors will be concatenated and returned. +func TryProtocolsWithRegistryURL(registry string, allowInsecure bool, action func(registryURL url.URL) error) (*url.URL, error) { + var errs []error + + if !strings.Contains(registry, "://") { + registry = "unset://" + registry + } + url, err := url.Parse(registry) + if err != nil { + return nil, err + } + var protos []string + switch { + case len(url.Scheme) > 0 && url.Scheme != "unset": + protos = []string{url.Scheme} + case allowInsecure || netutils.IsPrivateAddress(registry): + protos = []string{"https", "http"} + default: + protos = []string{"https"} + } + registry = url.Host + + for _, proto := range protos { + glog.V(4).Infof("Trying protocol %s for the registry URL %s", proto, registry) + url.Scheme = proto + err := action(*url) + if err == nil { + return url, nil + } + + if err != nil { + glog.V(4).Infof("Error with %s for %s: %v", proto, registry, err) + } + + if _, ok := err.(*errcode.Errors); ok { + // we got a response back from the registry, so return it + return url, err + } + errs = append(errs, err) + if proto == "https" && strings.Contains(err.Error(), "server gave HTTP response to HTTPS client") && !allowInsecure { + errs = append(errs, fmt.Errorf("\n* Append --force-insecure if you really want to prune the registry using insecure connection.")) + } else if proto == "http" && strings.Contains(err.Error(), "malformed HTTP response") { + errs = append(errs, fmt.Errorf("\n* Are you trying to connect to a TLS-enabled registry without TLS?")) + } + } + + return nil, kerrors.NewAggregate(errs) +} + +// retryPath is an error indicating that another connection attempt may be retried with a different path +type retryPath struct{ err error } + +func (rp *retryPath) Error() string { return rp.err.Error() } diff --git a/pkg/image/prune/helper_test.go b/pkg/image/prune/helper_test.go new file mode 100644 index 000000000000..79bb91daf4b5 --- /dev/null +++ b/pkg/image/prune/helper_test.go @@ -0,0 +1,217 @@ +package prune + +import ( + "crypto/tls" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + + knet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/diff" +) + +type requestStats struct { + lock sync.Mutex + requests []string +} + +func (rs *requestStats) addRequest(r *http.Request) { + rs.lock.Lock() + defer rs.lock.Unlock() + rs.requests = append(rs.requests, r.URL.String()) +} +func (rs *requestStats) clear() { + rs.lock.Lock() + defer rs.lock.Unlock() + rs.requests = rs.requests[:0] +} +func (rs *requestStats) getRequests() []string { + rs.lock.Lock() + defer rs.lock.Unlock() + res := make([]string, 0, len(rs.requests)) + for _, r := range rs.requests { + res = append(res, r) + } + return res +} + +func TestDefaultImagePinger(t *testing.T) { + rs := requestStats{requests: []string{}} + + type statusForPath map[string]int + + rt := knet.SetTransportDefaults(&http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }) + insecureClient := http.Client{Transport: rt} + secureClient := http.Client{} + + for _, tc := range []struct { + name string + schemePrefix string + securedRegistry bool + insecure bool + statusForPath statusForPath + expectedErrorSubstring string + expectedRequests []string + }{ + { + name: "tls secured registry with insecure fallback", + securedRegistry: true, + insecure: true, + statusForPath: statusForPath{"/": http.StatusOK}, + expectedRequests: []string{"/"}, + }, + + { + name: "tls secured registry prefixed by scheme with insecure fallback", + schemePrefix: "https://", + securedRegistry: true, + insecure: true, + statusForPath: statusForPath{"/": http.StatusOK}, + expectedRequests: []string{"/"}, + }, + + { + name: "tls secured registry prefixed by http scheme with insecure fallback", + schemePrefix: "http://", + securedRegistry: true, + insecure: true, + statusForPath: statusForPath{"/": http.StatusOK}, + expectedErrorSubstring: "malformed HTTP response", + }, + + { + name: "tls secured registry with no fallback", + securedRegistry: true, + insecure: false, + statusForPath: statusForPath{"/": http.StatusOK, "/healthz": http.StatusOK}, + expectedErrorSubstring: "x509: certificate signed by unknown authority", + }, + + { + name: "tls secured registry with old healthz endpoint", + securedRegistry: true, + insecure: true, + statusForPath: statusForPath{"/healthz": http.StatusOK}, + expectedRequests: []string{"/", "/healthz"}, + }, + + { + name: "insecure registry with insecure fallback", + securedRegistry: false, + insecure: true, + statusForPath: statusForPath{"/": http.StatusOK}, + expectedRequests: []string{"/"}, + }, + + { + name: "insecure registry prefixed by scheme with insecure fallback", + schemePrefix: "http://", + securedRegistry: false, + insecure: true, + statusForPath: statusForPath{"/": http.StatusOK}, + expectedRequests: []string{"/"}, + }, + + { + name: "insecure registry prefixed by https scheme with insecure fallback", + schemePrefix: "https://", + securedRegistry: false, + insecure: true, + statusForPath: statusForPath{"/": http.StatusOK}, + expectedErrorSubstring: "server gave HTTP response to HTTPS client", + }, + + { + name: "insecure registry with no fallback", + securedRegistry: false, + statusForPath: statusForPath{"/": http.StatusOK, "/healthz": http.StatusOK}, + expectedErrorSubstring: "server gave HTTP response to HTTPS client", + }, + + { + name: "insecure registry with old healthz endpoint", + securedRegistry: false, + insecure: true, + statusForPath: statusForPath{"/healthz": http.StatusOK}, + expectedRequests: []string{"/", "/healthz"}, + }, + + { + name: "initializing insecure registry", + securedRegistry: false, + insecure: true, + statusForPath: statusForPath{}, + expectedErrorSubstring: "server gave HTTP response to HTTPS client, unexpected status: 404 Not Found", + expectedRequests: []string{"/", "/healthz"}, + }, + } { + func() { + defer rs.clear() + + handler := func(w http.ResponseWriter, r *http.Request) { + rs.addRequest(r) + if s, ok := tc.statusForPath[r.URL.Path]; ok { + w.WriteHeader(s) + } else { + w.WriteHeader(http.StatusNotFound) + } + } + + var server *httptest.Server + if tc.securedRegistry { + server = httptest.NewTLSServer(http.HandlerFunc(handler)) + } else { + server = httptest.NewServer(http.HandlerFunc(handler)) + } + defer server.Close() + serverHost := strings.TrimLeft(strings.TrimLeft(server.URL, "http://"), "https://") + + client := &secureClient + if tc.insecure { + client = &insecureClient + } + + pinger := DefaultRegistryPinger{ + Client: client, + Insecure: tc.insecure, + } + + registryURL, err := pinger.Ping(tc.schemePrefix + serverHost) + if err != nil { + if len(tc.expectedErrorSubstring) == 0 { + t.Errorf("[%s] got unexpected ping error of type %T: %v", tc.name, err, err) + } else if !strings.Contains(err.Error(), tc.expectedErrorSubstring) { + t.Errorf("[%s] expected substring %q not found in error message: %s", tc.name, tc.expectedErrorSubstring, err.Error()) + } + } else if len(tc.expectedErrorSubstring) > 0 { + t.Errorf("[%s] unexpected non-error", tc.name) + } + + e := server.URL + if len(tc.expectedErrorSubstring) > 0 { + // the pinger should return unchanged input in case of error + e = "" + } + a := "" + if registryURL != nil { + a = registryURL.String() + } + if a != e { + t.Errorf("[%s] unexpected registry url: %q != %q", tc.name, a, e) + } + + ers := tc.expectedRequests + if ers == nil { + ers = []string{} + } + if a := rs.getRequests(); !reflect.DeepEqual(a, ers) { + t.Errorf("[%s] got unexpected requests: %s", tc.name, diff.ObjectDiff(a, ers)) + } + }() + } +} diff --git a/pkg/image/prune/prune.go b/pkg/image/prune/prune.go index f24b3ba0e291..28af53bd3379 100644 --- a/pkg/image/prune/prune.go +++ b/pkg/image/prune/prune.go @@ -4,9 +4,8 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "reflect" - "sort" - "strings" "time" "github.com/docker/distribution/manifest/schema2" @@ -32,7 +31,6 @@ import ( deploygraph "github.com/openshift/origin/pkg/deploy/graph/nodes" imageapi "github.com/openshift/origin/pkg/image/apis/image" imagegraph "github.com/openshift/origin/pkg/image/graph/nodes" - "github.com/openshift/origin/pkg/util/netutils" ) // TODO these edges should probably have an `Add***Edges` method in images/graph and be moved there @@ -82,14 +80,14 @@ type ImageStreamDeleter interface { type BlobDeleter interface { // DeleteBlob uses registryClient to ask the registry at registryURL // to remove the blob. - DeleteBlob(registryClient *http.Client, registryURL, blob string) error + DeleteBlob(registryClient *http.Client, registryURL *url.URL, blob string) error } // LayerLinkDeleter knows how to delete a repository layer link from the Docker registry. type LayerLinkDeleter interface { // DeleteLayerLink uses registryClient to ask the registry at registryURL to // delete the repository layer link. - DeleteLayerLink(registryClient *http.Client, registryURL, repo, linkName string) error + DeleteLayerLink(registryClient *http.Client, registryURL *url.URL, repo, linkName string) error } // ManifestDeleter knows how to delete image manifest data for a repository from @@ -97,7 +95,7 @@ type LayerLinkDeleter interface { type ManifestDeleter interface { // DeleteManifest uses registryClient to ask the registry at registryURL to // delete the repository's image manifest data. - DeleteManifest(registryClient *http.Client, registryURL, repo, manifest string) error + DeleteManifest(registryClient *http.Client, registryURL *url.URL, repo, manifest string) error } // PrunerOptions contains the fields used to initialize a new Pruner. @@ -112,7 +110,6 @@ type PrunerOptions struct { // will be considered as candidates for pruning. PruneOverSizeLimit *bool // AllImages considers all images for pruning, not just those pushed directly to the registry. - // Requires RegistryURL be set. AllImages *bool // Namespace to be pruned, if specified it should never remove Images. Namespace string @@ -142,10 +139,8 @@ type PrunerOptions struct { DryRun bool // RegistryClient is the http.Client to use when contacting the registry. RegistryClient *http.Client - // RegistryURL is the URL for the registry. - RegistryURL string - // Allow a fallback to insecure transport when contacting the registry. - Insecure bool + // RegistryURL is the URL of the integrated Docker registry. + RegistryURL *url.URL } // Pruner knows how to prune istags, images, layers and image configs. @@ -161,68 +156,12 @@ type Pruner interface { type pruner struct { g graph.Graph algorithm pruneAlgorithm - registryPinger registryPinger registryClient *http.Client - registryURL string + registryURL *url.URL } var _ Pruner = &pruner{} -// registryPinger performs a health check against a registry. -type registryPinger interface { - // ping performs a health check against registry. - ping(registry string) error -} - -// defaultRegistryPinger implements registryPinger. -type defaultRegistryPinger struct { - client *http.Client - insecure bool -} - -func (drp *defaultRegistryPinger) ping(registry string) error { - healthCheck := func(proto, registry string) error { - // TODO: `/healthz` route is deprecated by `/`; remove it in future versions - healthResponse, err := drp.client.Get(fmt.Sprintf("%s://%s/healthz", proto, registry)) - if err != nil { - return err - } - defer healthResponse.Body.Close() - - if healthResponse.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status: %s", healthResponse.Status) - } - - return nil - } - - var errs []error - protos := make([]string, 0, 2) - protos = append(protos, "https") - if drp.insecure || netutils.IsPrivateAddress(registry) { - protos = append(protos, "http") - } - for _, proto := range protos { - glog.V(4).Infof("Trying %s for %s", proto, registry) - err := healthCheck(proto, registry) - if err == nil { - return nil - } - errs = append(errs, err) - glog.V(4).Infof("Error with %s for %s: %v", proto, registry, err) - } - - return kerrors.NewAggregate(errs) -} - -// dryRunRegistryPinger implements registryPinger. -type dryRunRegistryPinger struct { -} - -func (*dryRunRegistryPinger) ping(registry string) error { - return nil -} - // NewPruner creates a Pruner. // // Images younger than keepYoungerThan and images referenced by image streams @@ -296,20 +235,9 @@ func NewPruner(options PrunerOptions) Pruner { addBuildsToGraph(g, options.Builds) addDeploymentConfigsToGraph(g, options.DCs) - var rp registryPinger - if options.DryRun { - rp = &dryRunRegistryPinger{} - } else { - rp = &defaultRegistryPinger{ - client: options.RegistryClient, - insecure: options.Insecure, - } - } - return &pruner{ g: g, algorithm: algorithm, - registryPinger: rp, registryClient: options.RegistryClient, registryURL: options.RegistryURL, } @@ -812,73 +740,6 @@ func pruneImages(g graph.Graph, imageNodes []*imagegraph.ImageNode, imagePruner return errs } -// order younger images before older -type imgByAge []*imageapi.Image - -func (ba imgByAge) Len() int { return len(ba) } -func (ba imgByAge) Swap(i, j int) { ba[i], ba[j] = ba[j], ba[i] } -func (ba imgByAge) Less(i, j int) bool { - return ba[i].CreationTimestamp.After(ba[j].CreationTimestamp.Time) -} - -// order younger image stream before older -type isByAge []*imagegraph.ImageStreamNode - -func (ba isByAge) Len() int { return len(ba) } -func (ba isByAge) Swap(i, j int) { ba[i], ba[j] = ba[j], ba[i] } -func (ba isByAge) Less(i, j int) bool { - return ba[i].ImageStream.CreationTimestamp.After(ba[j].ImageStream.CreationTimestamp.Time) -} - -func (p *pruner) determineRegistry(imageNodes []*imagegraph.ImageNode, isNodes []*imagegraph.ImageStreamNode) (string, error) { - if len(p.registryURL) > 0 { - return p.registryURL, nil - } - - var pullSpec string - var managedImages []*imageapi.Image - - // 1st try to determine registry url from a pull spec of the youngest managed image - for _, node := range imageNodes { - if node.Image.Annotations[imageapi.ManagedByOpenShiftAnnotation] != "true" { - continue - } - managedImages = append(managedImages, node.Image) - } - // be sure to pick up the newest managed image which should have an up to date information - sort.Sort(imgByAge(managedImages)) - - if len(managedImages) > 0 { - pullSpec = managedImages[0].DockerImageReference - } else { - // 2nd try to get the pull spec from any image stream - // Sorting by creation timestamp may not get us up to date info. Modification time would be much - // better if there were such an attribute. - sort.Sort(isByAge(isNodes)) - for _, node := range isNodes { - if len(node.ImageStream.Status.DockerImageRepository) == 0 { - continue - } - pullSpec = node.ImageStream.Status.DockerImageRepository - } - } - - if len(pullSpec) == 0 { - return "", fmt.Errorf("no managed image found") - } - - ref, err := imageapi.ParseDockerImageReference(pullSpec) - if err != nil { - return "", fmt.Errorf("unable to parse %q: %v", pullSpec, err) - } - - if len(ref.Registry) == 0 { - return "", fmt.Errorf("%s does not include a registry", pullSpec) - } - - return ref.Registry, nil -} - // Run identifies images eligible for pruning, invoking imagePruner for each image, and then it identifies // image configs and layers eligible for pruning, invoking layerLinkPruner for each registry URL that has // layers or configs that can be pruned. @@ -896,16 +757,6 @@ func (p *pruner) Prune( return nil } - registryURL, err := p.determineRegistry(imageNodes, getImageStreamNodes(allNodes)) - if err != nil { - return fmt.Errorf("unable to determine registry: %v", err) - } - glog.V(1).Infof("Using registry: %s", registryURL) - - if err := p.registryPinger.ping(registryURL); err != nil { - return fmt.Errorf("error communicating with registry %s: %v", registryURL, err) - } - prunableImageNodes, prunableImageIDs := calculatePrunableImages(p.g, imageNodes) errs := []error{} @@ -917,9 +768,9 @@ func (p *pruner) Prune( graphWithoutPrunableImages := subgraphWithoutPrunableImages(p.g, prunableImageIDs) prunableComponents := calculatePrunableImageComponents(graphWithoutPrunableImages) - errs = append(errs, pruneImageComponents(p.g, p.registryClient, registryURL, prunableComponents, layerLinkPruner)...) - errs = append(errs, pruneBlobs(p.g, p.registryClient, registryURL, prunableComponents, blobPruner)...) - errs = append(errs, pruneManifests(p.g, p.registryClient, registryURL, prunableImageNodes, manifestPruner)...) + errs = append(errs, pruneImageComponents(p.g, p.registryClient, p.registryURL, prunableComponents, layerLinkPruner)...) + errs = append(errs, pruneBlobs(p.g, p.registryClient, p.registryURL, prunableComponents, blobPruner)...) + errs = append(errs, pruneManifests(p.g, p.registryClient, p.registryURL, prunableImageNodes, manifestPruner)...) if len(errs) > 0 { // If we had any errors removing image references from image streams or deleting @@ -964,7 +815,7 @@ func streamsReferencingImageComponent(g graph.Graph, cn *imagegraph.ImageCompone func pruneImageComponents( g graph.Graph, registryClient *http.Client, - registryURL string, + registryURL *url.URL, imageComponents []*imagegraph.ImageComponentNode, layerLinkDeleter LayerLinkDeleter, ) []error { @@ -975,12 +826,10 @@ func pruneImageComponents( streamNodes := streamsReferencingImageComponent(g, cn) for _, streamNode := range streamNodes { - stream := streamNode.ImageStream - streamName := fmt.Sprintf("%s/%s", stream.Namespace, stream.Name) - - glog.V(4).Infof("Pruning registry=%q, repo=%q, %s", registryURL, streamName, cn.Describe()) + streamName := getName(streamNode.ImageStream) + glog.V(4).Infof("Pruning repository %s/%s: %s", registryURL.Host, streamName, cn.Describe()) if err := layerLinkDeleter.DeleteLayerLink(registryClient, registryURL, streamName, cn.Component); err != nil { - errs = append(errs, fmt.Errorf("error pruning layer link %s in repo %q: %v", cn.Component, streamName, err)) + errs = append(errs, fmt.Errorf("error pruning layer link %s in the repository %s: %v", cn.Component, streamName, err)) } } } @@ -993,7 +842,7 @@ func pruneImageComponents( func pruneBlobs( g graph.Graph, registryClient *http.Client, - registryURL string, + registryURL *url.URL, componentNodes []*imagegraph.ImageComponentNode, blobPruner BlobDeleter, ) []error { @@ -1001,8 +850,8 @@ func pruneBlobs( for _, cn := range componentNodes { if err := blobPruner.DeleteBlob(registryClient, registryURL, cn.Component); err != nil { - errs = append(errs, fmt.Errorf("error removing blob from registry %s: blob %q: %v", - registryURL, cn.Component, err)) + errs = append(errs, fmt.Errorf("error removing blob %s from the registry %s: %v", + cn.Component, registryURL.Host, err)) } } @@ -1011,7 +860,13 @@ func pruneBlobs( // pruneManifests invokes manifestPruner.DeleteManifest for each repository // manifest to be deleted from the registry. -func pruneManifests(g graph.Graph, registryClient *http.Client, registryURL string, imageNodes []*imagegraph.ImageNode, manifestPruner ManifestDeleter) []error { +func pruneManifests( + g graph.Graph, + registryClient *http.Client, + registryURL *url.URL, + imageNodes []*imagegraph.ImageNode, + manifestPruner ManifestDeleter, +) []error { errs := []error{} for _, imageNode := range imageNodes { @@ -1021,12 +876,12 @@ func pruneManifests(g graph.Graph, registryClient *http.Client, registryURL stri continue } - stream := streamNode.ImageStream - repoName := fmt.Sprintf("%s/%s", stream.Namespace, stream.Name) + repoName := getName(streamNode.ImageStream) - glog.V(4).Infof("Pruning manifest for registry %q, repo %q, image %q", registryURL, repoName, imageNode.Image.Name) + glog.V(4).Infof("Pruning manifest %s in the repository %s/%s", imageNode.Image.Name, registryURL.Host, repoName) if err := manifestPruner.DeleteManifest(registryClient, registryURL, repoName, imageNode.Image.Name); err != nil { - errs = append(errs, fmt.Errorf("error pruning manifest for registry %q, repo %q, image %q: %v", registryURL, repoName, imageNode.Image.Name, err)) + errs = append(errs, fmt.Errorf("error pruning manifest %s in the repository %s/%s: %v", + imageNode.Image.Name, registryURL.Host, repoName, err)) } } } @@ -1069,71 +924,53 @@ func NewImageStreamDeleter(streams client.ImageStreamsNamespacer) ImageStreamDel func (p *imageStreamDeleter) DeleteImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error) { glog.V(4).Infof("Updating ImageStream %s", getName(stream)) - glog.V(5).Infof("Updated stream: %#v", stream) - return p.streams.ImageStreams(stream.Namespace).UpdateStatus(stream) + is, err := p.streams.ImageStreams(stream.Namespace).UpdateStatus(stream) + if err == nil { + glog.V(5).Infof("Updated ImageStream: %#v", is) + } + return is, err } // deleteFromRegistry uses registryClient to send a DELETE request to the // provided url. It attempts an https request first; if that fails, it fails // back to http. func deleteFromRegistry(registryClient *http.Client, url string) error { - deleteFunc := func(proto, url string) error { - req, err := http.NewRequest("DELETE", url, nil) - if err != nil { - return err - } - - glog.V(4).Infof("Sending request to registry") - resp, err := registryClient.Do(req) - if err != nil { - if proto != "https" && strings.Contains(err.Error(), "malformed HTTP response") { - return fmt.Errorf("%v.\n* Are you trying to connect to a TLS-enabled registry without TLS?", err) - } - return err - } - defer resp.Body.Close() + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } - // TODO: investigate why we're getting non-existent layers, for now we're logging - // them out and continue working - if resp.StatusCode == http.StatusNotFound { - glog.Warningf("Unable to prune layer %s, returned %v", url, resp.Status) - return nil - } - // non-2xx/3xx response doesn't cause an error, so we need to check for it - // manually and return it to caller - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { - return fmt.Errorf(resp.Status) - } - if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusAccepted { - glog.V(1).Infof("Unexpected status code in response: %d", resp.StatusCode) - var response errcode.Errors - decoder := json.NewDecoder(resp.Body) - if err := decoder.Decode(&response); err != nil { - return err - } - glog.V(1).Infof("Response: %#v", response) - return &response - } + glog.V(5).Infof(`Sending request "%s %s" to the registry`, req.Method, req.URL.String()) + resp, err := registryClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + // TODO: investigate why we're getting non-existent layers, for now we're logging + // them out and continue working + if resp.StatusCode == http.StatusNotFound { + glog.Warningf("Unable to prune layer %s, returned %v", url, resp.Status) return nil } - var err error - for _, proto := range []string{"https", "http"} { - glog.V(4).Infof("Trying %s for %s", proto, url) - err = deleteFunc(proto, fmt.Sprintf("%s://%s", proto, url)) - if err == nil { - return nil - } + // non-2xx/3xx response doesn't cause an error, so we need to check for it + // manually and return it to caller + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { + return fmt.Errorf(resp.Status) + } - if _, ok := err.(*errcode.Errors); ok { - // we got a response back from the registry, so return it + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusAccepted { + glog.V(1).Infof("Unexpected status code in response: %d", resp.StatusCode) + var response errcode.Errors + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(&response); err != nil { return err } - - // we didn't get a success or a errcode.Errors response back from the registry - glog.V(4).Infof("Error with %s for %s: %v", proto, url, err) + glog.V(1).Infof("Response: %#v", response) + return &response } + return err } @@ -1147,9 +984,9 @@ func NewLayerLinkDeleter() LayerLinkDeleter { return &layerLinkDeleter{} } -func (p *layerLinkDeleter) DeleteLayerLink(registryClient *http.Client, registryURL, repoName, linkName string) error { - glog.V(4).Infof("Deleting layer link from registry %q: repo %q, layer link %q", registryURL, repoName, linkName) - return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, repoName, linkName)) +func (p *layerLinkDeleter) DeleteLayerLink(registryClient *http.Client, registryURL *url.URL, repoName, linkName string) error { + glog.V(4).Infof("Deleting layer link %s from repository %s/%s", linkName, registryURL.Host, repoName) + return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL.String(), repoName, linkName)) } // blobDeleter removes a blob from the registry. @@ -1162,9 +999,9 @@ func NewBlobDeleter() BlobDeleter { return &blobDeleter{} } -func (p *blobDeleter) DeleteBlob(registryClient *http.Client, registryURL, blob string) error { - glog.V(4).Infof("Deleting blob from registry %q: blob %q", registryURL, blob) - return deleteFromRegistry(registryClient, fmt.Sprintf("%s/admin/blobs/%s", registryURL, blob)) +func (p *blobDeleter) DeleteBlob(registryClient *http.Client, registryURL *url.URL, blob string) error { + glog.V(4).Infof("Deleting blob %s from registry %s", blob, registryURL.Host) + return deleteFromRegistry(registryClient, fmt.Sprintf("%s/admin/blobs/%s", registryURL.String(), blob)) } // manifestDeleter deletes repository manifest data from the registry. @@ -1177,9 +1014,9 @@ func NewManifestDeleter() ManifestDeleter { return &manifestDeleter{} } -func (p *manifestDeleter) DeleteManifest(registryClient *http.Client, registryURL, repoName, manifest string) error { - glog.V(4).Infof("Deleting manifest from registry %q: repo %q, manifest %q", registryURL, repoName, manifest) - return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL, repoName, manifest)) +func (p *manifestDeleter) DeleteManifest(registryClient *http.Client, registryURL *url.URL, repoName, manifest string) error { + glog.V(4).Infof("Deleting manifest %s from repository %s/%s", manifest, registryURL.Host, repoName) + return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL.String(), repoName, manifest)) } func getName(obj runtime.Object) string { diff --git a/pkg/image/prune/prune_test.go b/pkg/image/prune/prune_test.go index 0d5c8c4ce098..8398741de895 100644 --- a/pkg/image/prune/prune_test.go +++ b/pkg/image/prune/prune_test.go @@ -2,11 +2,11 @@ package prune import ( "bytes" - "errors" "flag" "fmt" "io/ioutil" "net/http" + "net/url" "reflect" "testing" "time" @@ -21,6 +21,7 @@ import ( "k8s.io/client-go/rest/fake" clientgotesting "k8s.io/client-go/testing" kapi "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/diff" "github.com/openshift/origin/pkg/api/graph" buildapi "github.com/openshift/origin/pkg/build/apis/build" @@ -30,16 +31,6 @@ import ( imagegraph "github.com/openshift/origin/pkg/image/graph/nodes" ) -type fakeRegistryPinger struct { - err error - requests []string -} - -func (f *fakeRegistryPinger) ping(registry string) error { - f.requests = append(f.requests, registry) - return f.err -} - func imageList(images ...imageapi.Image) imageapi.ImageList { return imageapi.ImageList{ Items: images, @@ -386,8 +377,8 @@ type fakeBlobDeleter struct { var _ BlobDeleter = &fakeBlobDeleter{} -func (p *fakeBlobDeleter) DeleteBlob(registryClient *http.Client, registryURL, blob string) error { - p.invocations.Insert(fmt.Sprintf("%s|%s", registryURL, blob)) +func (p *fakeBlobDeleter) DeleteBlob(registryClient *http.Client, registryURL *url.URL, blob string) error { + p.invocations.Insert(fmt.Sprintf("%s|%s", registryURL.String(), blob)) return p.err } @@ -398,8 +389,8 @@ type fakeLayerLinkDeleter struct { var _ LayerLinkDeleter = &fakeLayerLinkDeleter{} -func (p *fakeLayerLinkDeleter) DeleteLayerLink(registryClient *http.Client, registryURL, repo, layer string) error { - p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL, repo, layer)) +func (p *fakeLayerLinkDeleter) DeleteLayerLink(registryClient *http.Client, registryURL *url.URL, repo, layer string) error { + p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL.String(), repo, layer)) return p.err } @@ -410,8 +401,8 @@ type fakeManifestDeleter struct { var _ ManifestDeleter = &fakeManifestDeleter{} -func (p *fakeManifestDeleter) DeleteManifest(registryClient *http.Client, registryURL, repo, manifest string) error { - p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL, repo, manifest)) +func (p *fakeManifestDeleter) DeleteManifest(registryClient *http.Client, registryURL *url.URL, repo, manifest string) error { + p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL.String(), repo, manifest)) return p.err } @@ -420,11 +411,12 @@ var testCase = flag.String("testcase", "", "") func TestImagePruning(t *testing.T) { flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel)) - registryURL := "registry.io" + registryHost := "registry.io" + registryURL := "https://" + registryHost tests := map[string]struct { pruneOverSizeLimit *bool - registryURLs []string + allImages *bool namespace string images imageapi.ImageList pods kapi.PodList @@ -440,36 +432,40 @@ func TestImagePruning(t *testing.T) { expectedBlobDeletions []string }{ "1 pod - phase pending - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - pods: podList(pod("foo", "pod1", kapi.PodPending, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + pods: podList(pod("foo", "pod1", kapi.PodPending, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "3 pods - last phase pending - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), pods: podList( - pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod2", kapi.PodSucceeded, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod3", kapi.PodPending, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod1", kapi.PodSucceeded, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod2", kapi.PodSucceeded, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod3", kapi.PodPending, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), ), expectedImageDeletions: []string{}, }, + "1 pod - phase running - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - pods: podList(pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + pods: podList(pod("foo", "pod1", kapi.PodRunning, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "3 pods - last phase running - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), pods: podList( - pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod2", kapi.PodSucceeded, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod3", kapi.PodRunning, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod1", kapi.PodSucceeded, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod2", kapi.PodSucceeded, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod3", kapi.PodRunning, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), ), expectedImageDeletions: []string{}, }, + "pod phase succeeded - prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - pods: podList(pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + pods: podList(pod("foo", "pod1", kapi.PodSucceeded, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, expectedBlobDeletions: []string{ registryURL + "|" + layer1, @@ -479,22 +475,25 @@ func TestImagePruning(t *testing.T) { registryURL + "|" + layer5, }, }, + "pod phase succeeded, pod less than min pruning age - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - pods: podList(agedPod("foo", "pod1", kapi.PodSucceeded, 5, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + pods: podList(agedPod("foo", "pod1", kapi.PodSucceeded, 5, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "pod phase succeeded, image less than min pruning age - don't prune": { - images: imageList(agedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", 5)), - pods: podList(pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(agedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", 5)), + pods: podList(pod("foo", "pod1", kapi.PodSucceeded, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "pod phase failed - prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), pods: podList( - pod("foo", "pod1", kapi.PodFailed, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod2", kapi.PodFailed, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod3", kapi.PodFailed, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod1", kapi.PodFailed, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod2", kapi.PodFailed, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod3", kapi.PodFailed, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), ), expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, expectedBlobDeletions: []string{ @@ -505,12 +504,13 @@ func TestImagePruning(t *testing.T) { registryURL + "|" + layer5, }, }, + "pod phase unknown - prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), pods: podList( - pod("foo", "pod1", kapi.PodUnknown, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod2", kapi.PodUnknown, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - pod("foo", "pod3", kapi.PodUnknown, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod1", kapi.PodUnknown, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod2", kapi.PodUnknown, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + pod("foo", "pod3", kapi.PodUnknown, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), ), expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, expectedBlobDeletions: []string{ @@ -521,8 +521,9 @@ func TestImagePruning(t *testing.T) { registryURL + "|" + layer5, }, }, + "pod container image not parsable": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), pods: podList( pod("foo", "pod1", kapi.PodRunning, "a/b/c/d/e"), ), @@ -535,8 +536,9 @@ func TestImagePruning(t *testing.T) { registryURL + "|" + layer5, }, }, + "pod container image doesn't have an id": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), pods: podList( pod("foo", "pod1", kapi.PodRunning, "foo/bar:latest"), ), @@ -549,10 +551,11 @@ func TestImagePruning(t *testing.T) { registryURL + "|" + layer5, }, }, + "pod refers to image not in graph": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), pods: podList( - pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@otherid"), + pod("foo", "pod1", kapi.PodRunning, registryHost+"/foo/bar@otherid"), ), expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, expectedBlobDeletions: []string{ @@ -563,193 +566,293 @@ func TestImagePruning(t *testing.T) { registryURL + "|" + layer5, }, }, + "referenced by rc - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - rcs: rcList(rc("foo", "rc1", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + rcs: rcList(rc("foo", "rc1", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by dc - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - dcs: dcList(dc("foo", "rc1", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + dcs: dcList(dc("foo", "rc1", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by bc - sti - ImageStreamImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), bcs: bcList(bc("foo", "bc1", "source", "ImageStreamImage", "foo", "bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by bc - docker - ImageStreamImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), bcs: bcList(bc("foo", "bc1", "docker", "ImageStreamImage", "foo", "bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by bc - custom - ImageStreamImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), bcs: bcList(bc("foo", "bc1", "custom", "ImageStreamImage", "foo", "bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by bc - sti - DockerImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - bcs: bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + bcs: bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by bc - docker - DockerImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - bcs: bcList(bc("foo", "bc1", "docker", "DockerImage", "foo", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + bcs: bcList(bc("foo", "bc1", "docker", "DockerImage", "foo", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by bc - custom - DockerImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - bcs: bcList(bc("foo", "bc1", "custom", "DockerImage", "foo", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + bcs: bcList(bc("foo", "bc1", "custom", "DockerImage", "foo", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by build - sti - ImageStreamImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), builds: buildList(build("foo", "build1", "source", "ImageStreamImage", "foo", "bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by build - docker - ImageStreamImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), builds: buildList(build("foo", "build1", "docker", "ImageStreamImage", "foo", "bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by build - custom - ImageStreamImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), builds: buildList(build("foo", "build1", "custom", "ImageStreamImage", "foo", "bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by build - sti - DockerImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - builds: buildList(build("foo", "build1", "source", "DockerImage", "foo", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + builds: buildList(build("foo", "build1", "source", "DockerImage", "foo", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by build - docker - DockerImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - builds: buildList(build("foo", "build1", "docker", "DockerImage", "foo", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + builds: buildList(build("foo", "build1", "docker", "DockerImage", "foo", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "referenced by build - custom - DockerImage - don't prune": { - images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - builds: buildList(build("foo", "build1", "custom", "DockerImage", "foo", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + images: imageList(image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + builds: buildList(build("foo", "build1", "custom", "DockerImage", "foo", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, }, + "image stream - keep most recent n images": { images: imageList( unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", false, "", ""), - image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - image("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), - image("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), ), )), ), expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000004"}, expectedStreamUpdates: []string{"foo/bar|sha256:0000000000000000000000000000000000000000000000000000000000000004"}, }, + "image stream - same manifest listed multiple times in tag history": { images: imageList( - image("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), - image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), ), )), ), }, + "image stream age less than min pruning age - don't prune": { images: imageList( - image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - image("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), - image("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), ), streams: streamList( - agedStream(registryURL, "foo", "bar", 5, tags( + agedStream(registryHost, "foo", "bar", 5, tags( tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), ), )), ), expectedImageDeletions: []string{}, expectedStreamUpdates: []string{}, }, + "multiple resources pointing to image - don't prune": { images: imageList( - image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + image("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), ), )), ), - rcs: rcList(rc("foo", "rc1", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002")), - pods: podList(pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002")), - dcs: dcList(dc("foo", "rc1", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), - bcs: bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + rcs: rcList(rc("foo", "rc1", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002")), + pods: podList(pod("foo", "pod1", kapi.PodRunning, registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002")), + dcs: dcList(dc("foo", "rc1", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), + bcs: bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), builds: buildList(build("foo", "build1", "custom", "ImageStreamImage", "foo", "bar@sha256:0000000000000000000000000000000000000000000000000000000000000000")), expectedImageDeletions: []string{}, expectedStreamUpdates: []string{}, }, + "image with nil annotations": { + images: imageList( + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", false, "", ""), + ), + expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + expectedStreamUpdates: []string{}, + }, + + "prune all-images=true image with nil annotations": { + allImages: newBool(true), + images: imageList( + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", false, "", ""), + ), + expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + expectedStreamUpdates: []string{}, + }, + + "prune all-images=false image with nil annotations": { + allImages: newBool(false), images: imageList( unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", false, "", ""), ), expectedImageDeletions: []string{}, expectedStreamUpdates: []string{}, }, + "image missing managed annotation": { images: imageList( unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, "foo", "bar"), ), - expectedImageDeletions: []string{}, + expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, expectedStreamUpdates: []string{}, }, + "image with managed annotation != true": { images: imageList( unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "false"), - unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "0"), - unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "1"), - unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "True"), - unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "yes"), - unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "Yes"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000001", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "0"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "1"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "True"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000004", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "yes"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000005", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "Yes"), + ), + expectedImageDeletions: []string{ + "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "sha256:0000000000000000000000000000000000000000000000000000000000000001", + "sha256:0000000000000000000000000000000000000000000000000000000000000002", + "sha256:0000000000000000000000000000000000000000000000000000000000000003", + "sha256:0000000000000000000000000000000000000000000000000000000000000004", + "sha256:0000000000000000000000000000000000000000000000000000000000000005", + }, + expectedStreamUpdates: []string{}, + }, + + "prune all-images=true with image missing managed annotation": { + allImages: newBool(true), + images: imageList( + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, "foo", "bar"), + ), + expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + expectedStreamUpdates: []string{}, + }, + + "prune all-images=true with image with managed annotation != true": { + allImages: newBool(true), + images: imageList( + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "false"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000001", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "0"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "1"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "True"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000004", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "yes"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000005", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "Yes"), + ), + expectedImageDeletions: []string{ + "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "sha256:0000000000000000000000000000000000000000000000000000000000000001", + "sha256:0000000000000000000000000000000000000000000000000000000000000002", + "sha256:0000000000000000000000000000000000000000000000000000000000000003", + "sha256:0000000000000000000000000000000000000000000000000000000000000004", + "sha256:0000000000000000000000000000000000000000000000000000000000000005", + }, + expectedStreamUpdates: []string{}, + }, + + "prune all-images=false with image missing managed annotation": { + allImages: newBool(false), + images: imageList( + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, "foo", "bar"), ), expectedImageDeletions: []string{}, expectedStreamUpdates: []string{}, }, + + "prune all-images=false with image with managed annotation != true": { + allImages: newBool(false), + images: imageList( + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "false"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000001", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "0"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "1"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "True"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000004", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "yes"), + unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000005", "someregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", true, imageapi.ManagedByOpenShiftAnnotation, "Yes"), + ), + expectedImageDeletions: []string{}, + expectedStreamUpdates: []string{}, + }, + "image with layers": { images: imageList( - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", &config1, "layer1", "layer2", "layer3", "layer4"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", &config2, "layer1", "layer2", "layer3", "layer4"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", nil, "layer1", "layer2", "layer3", "layer4"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004", nil, "layer5", "layer6", "layer7", "layer8"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", &config1, "layer1", "layer2", "layer3", "layer4"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", &config2, "layer1", "layer2", "layer3", "layer4"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", nil, "layer1", "layer2", "layer3", "layer4"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004", nil, "layer5", "layer6", "layer7", "layer8"), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), ), )), ), @@ -768,21 +871,22 @@ func TestImagePruning(t *testing.T) { registryURL + "|layer8", }, }, + "images with duplicate layers and configs": { images: imageList( - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", &config1, "layer1", "layer2", "layer3", "layer4"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", &config1, "layer1", "layer2", "layer3", "layer4"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", &config1, "layer1", "layer2", "layer3", "layer4"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004", &config2, "layer5", "layer6", "layer7", "layer8"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000005", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000005", &config2, "layer5", "layer6", "layer9", "layerX"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", &config1, "layer1", "layer2", "layer3", "layer4"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", &config1, "layer1", "layer2", "layer3", "layer4"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", &config1, "layer1", "layer2", "layer3", "layer4"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004", &config2, "layer5", "layer6", "layer7", "layer8"), + imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000005", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000005", &config2, "layer5", "layer6", "layer9", "layerX"), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000004"), ), )), ), @@ -805,19 +909,20 @@ func TestImagePruning(t *testing.T) { registryURL + "|layerX", }, }, + "image exceeding limits": { pruneOverSizeLimit: newBool(true), images: imageList( unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", false, "", ""), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 100, nil), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", 200, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 100, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", 200, nil), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), ), )), ), @@ -827,25 +932,26 @@ func TestImagePruning(t *testing.T) { expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000003"}, expectedStreamUpdates: []string{"foo/bar|sha256:0000000000000000000000000000000000000000000000000000000000000003"}, }, + "multiple images in different namespaces exceeding different limits": { pruneOverSizeLimit: newBool(true), images: imageList( - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", 100, nil), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 200, nil), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000003", 500, nil), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000004", 600, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", 100, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 200, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000003", 500, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000004", 600, nil), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), ), )), - stream(registryURL, "bar", "foo", tags( + stream(registryHost, "bar", "foo", tags( tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000003"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryURL+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000004"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000004", registryHost+"/bar/foo@sha256:0000000000000000000000000000000000000000000000000000000000000004"), ), )), ), @@ -856,19 +962,20 @@ func TestImagePruning(t *testing.T) { expectedImageDeletions: []string{"sha256:0000000000000000000000000000000000000000000000000000000000000002", "sha256:0000000000000000000000000000000000000000000000000000000000000004"}, expectedStreamUpdates: []string{"foo/bar|sha256:0000000000000000000000000000000000000000000000000000000000000002", "bar/foo|sha256:0000000000000000000000000000000000000000000000000000000000000004"}, }, + "image within allowed limits": { pruneOverSizeLimit: newBool(true), images: imageList( unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", false, "", ""), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 100, nil), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", 200, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 100, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", 200, nil), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), ), )), ), @@ -878,20 +985,21 @@ func TestImagePruning(t *testing.T) { expectedImageDeletions: []string{}, expectedStreamUpdates: []string{}, }, + "image exceeding limits with namespace specified": { pruneOverSizeLimit: newBool(true), namespace: "foo", images: imageList( unmanagedImage("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000", false, "", ""), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 100, nil), - sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", 200, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", 100, nil), + sizedImage("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003", 200, nil), ), streams: streamList( - stream(registryURL, "foo", "bar", tags( + stream(registryHost, "foo", "bar", tags( tag("latest", tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000000", "otherregistry/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryURL+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), + tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000003", registryHost+"/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000003"), ), )), ), @@ -910,6 +1018,7 @@ func TestImagePruning(t *testing.T) { options := PrunerOptions{ Namespace: test.namespace, + AllImages: test.allImages, Images: &test.images, Streams: &test.streams, Pods: &test.pods, @@ -918,6 +1027,7 @@ func TestImagePruning(t *testing.T) { Builds: &test.builds, DCs: &test.dcs, LimitRanges: test.limits, + RegistryURL: &url.URL{Scheme: "https", Host: registryHost}, } if test.pruneOverSizeLimit != nil { options.PruneOverSizeLimit = test.pruneOverSizeLimit @@ -928,7 +1038,6 @@ func TestImagePruning(t *testing.T) { options.KeepTagRevisions = &keepTagRevisions } p := NewPruner(options) - p.(*pruner).registryPinger = &fakeRegistryPinger{} imageDeleter := &fakeImageDeleter{invocations: sets.NewString()} streamDeleter := &fakeImageStreamDeleter{invocations: sets.NewString()} @@ -939,23 +1048,23 @@ func TestImagePruning(t *testing.T) { p.Prune(imageDeleter, streamDeleter, layerLinkDeleter, blobDeleter, manifestDeleter) expectedImageDeletions := sets.NewString(test.expectedImageDeletions...) - if !reflect.DeepEqual(expectedImageDeletions, imageDeleter.invocations) { - t.Errorf("%s: expected image deletions %q, got %q", name, expectedImageDeletions.List(), imageDeleter.invocations.List()) + if a, e := imageDeleter.invocations, expectedImageDeletions; !reflect.DeepEqual(a, e) { + t.Errorf("%s: unexpected image deletions: %s", name, diff.ObjectDiff(a, e)) } expectedStreamUpdates := sets.NewString(test.expectedStreamUpdates...) - if !reflect.DeepEqual(expectedStreamUpdates, streamDeleter.invocations) { - t.Errorf("%s: expected stream updates %q, got %q", name, expectedStreamUpdates.List(), streamDeleter.invocations.List()) + if a, e := streamDeleter.invocations, expectedStreamUpdates; !reflect.DeepEqual(a, e) { + t.Errorf("%s: unexpected stream updates: %s", name, diff.ObjectDiff(a, e)) } expectedLayerLinkDeletions := sets.NewString(test.expectedLayerLinkDeletions...) - if !reflect.DeepEqual(expectedLayerLinkDeletions, layerLinkDeleter.invocations) { - t.Errorf("%s: expected layer link deletions %q, got %q", name, expectedLayerLinkDeletions.List(), layerLinkDeleter.invocations.List()) + if a, e := layerLinkDeleter.invocations, expectedLayerLinkDeletions; !reflect.DeepEqual(a, e) { + t.Errorf("%s: unexpected layer link deletions: %s", name, diff.ObjectDiff(a, e)) } expectedBlobDeletions := sets.NewString(test.expectedBlobDeletions...) - if !reflect.DeepEqual(expectedBlobDeletions, blobDeleter.invocations) { - t.Errorf("%s: expected blob deletions %q, got %q", name, expectedBlobDeletions.List(), blobDeleter.invocations.List()) + if a, e := blobDeleter.invocations, expectedBlobDeletions; !reflect.DeepEqual(a, e) { + t.Errorf("%s: unexpected blob deletions: %s", name, diff.ObjectDiff(a, e)) } } } @@ -1006,11 +1115,10 @@ func TestLayerDeleter(t *testing.T) { return &http.Response{StatusCode: http.StatusServiceUnavailable, Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}, nil }) layerLinkDeleter := NewLayerLinkDeleter() - layerLinkDeleter.DeleteLayerLink(client, "registry1", "repo", "layer1") + layerLinkDeleter.DeleteLayerLink(client, &url.URL{Scheme: "http", Host: "registry1"}, "repo", "layer1") - if !reflect.DeepEqual(actions, []string{"DELETE:https://registry1/v2/repo/blobs/layer1", - "DELETE:http://registry1/v2/repo/blobs/layer1"}) { - t.Errorf("Unexpected actions %v", actions) + if e := []string{"DELETE:http://registry1/v2/repo/blobs/layer1"}; !reflect.DeepEqual(actions, e) { + t.Errorf("unexpected actions: %s", diff.ObjectDiff(actions, e)) } } @@ -1023,10 +1131,10 @@ func TestNotFoundLayerDeleter(t *testing.T) { return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}, nil }) layerLinkDeleter := NewLayerLinkDeleter() - layerLinkDeleter.DeleteLayerLink(client, "registry1", "repo", "layer1") + layerLinkDeleter.DeleteLayerLink(client, &url.URL{Scheme: "https", Host: "registry1"}, "repo", "layer1") - if !reflect.DeepEqual(actions, []string{"DELETE:https://registry1/v2/repo/blobs/layer1"}) { - t.Errorf("Unexpected actions %v", actions) + if e := []string{"DELETE:https://registry1/v2/repo/blobs/layer1"}; !reflect.DeepEqual(actions, e) { + t.Errorf("unexpected actions: %s", diff.ObjectDiff(actions, e)) } } @@ -1060,17 +1168,17 @@ func TestRegistryPruning(t *testing.T) { )), ), expectedLayerLinkDeletions: sets.NewString( - "registry1.io|foo/bar|"+config1, - "registry1.io|foo/bar|layer1", - "registry1.io|foo/bar|layer2", + "https://registry1.io|foo/bar|"+config1, + "https://registry1.io|foo/bar|layer1", + "https://registry1.io|foo/bar|layer2", ), expectedBlobDeletions: sets.NewString( - "registry1.io|"+config1, - "registry1.io|layer1", - "registry1.io|layer2", + "https://registry1.io|"+config1, + "https://registry1.io|layer1", + "https://registry1.io|layer2", ), expectedManifestDeletions: sets.NewString( - "registry1.io|foo/bar|sha256:0000000000000000000000000000000000000000000000000000000000000001", + "https://registry1.io|foo/bar|sha256:0000000000000000000000000000000000000000000000000000000000000001", ), }, "no pruning when no images are pruned": { @@ -1095,40 +1203,17 @@ func TestRegistryPruning(t *testing.T) { ), expectedLayerLinkDeletions: sets.NewString(), expectedBlobDeletions: sets.NewString( - "registry1.io|"+config1, - "registry1.io|"+config2, - "registry1.io|layer1", - "registry1.io|layer2", - "registry1.io|layer3", - "registry1.io|layer4", - "registry1.io|layer5", - "registry1.io|layer6", + "https://registry1.io|"+config1, + "https://registry1.io|"+config2, + "https://registry1.io|layer1", + "https://registry1.io|layer2", + "https://registry1.io|layer3", + "https://registry1.io|layer4", + "https://registry1.io|layer5", + "https://registry1.io|layer6", ), expectedManifestDeletions: sets.NewString(), }, - "ping error": { - images: imageList( - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000001", "registry1.io/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", &config1, "layer1", "layer2", "layer3", "layer4"), - imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000002", "registry1.io/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002", &config2, "layer3", "layer4", "layer5", "layer6"), - ), - streams: streamList( - stream("registry1.io", "foo", "bar", tags( - tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", "registry1.io/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000001", "registry1.io/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001"), - ), - )), - stream("registry1.io", "foo", "other", tags( - tag("latest", - tagEvent("sha256:0000000000000000000000000000000000000000000000000000000000000002", "registry1.io/foo/other@sha256:0000000000000000000000000000000000000000000000000000000000000002"), - ), - )), - ), - expectedLayerLinkDeletions: sets.NewString(), - expectedBlobDeletions: sets.NewString(), - expectedManifestDeletions: sets.NewString(), - pingErr: errors.New("foo"), - }, "config used as a layer": { images: imageList( imageWithLayers("sha256:0000000000000000000000000000000000000000000000000000000000000001", "registry1.io/foo/bar@sha256:0000000000000000000000000000000000000000000000000000000000000001", &config1, "layer1", "layer2", "layer3", config1), @@ -1150,16 +1235,16 @@ func TestRegistryPruning(t *testing.T) { )), ), expectedLayerLinkDeletions: sets.NewString( - "registry1.io|foo/bar|layer1", - "registry1.io|foo/bar|layer2", + "https://registry1.io|foo/bar|layer1", + "https://registry1.io|foo/bar|layer2", // TODO: ideally, pruner should remove layers of id2 from foo/bar as well ), expectedBlobDeletions: sets.NewString( - "registry1.io|layer1", - "registry1.io|layer2", + "https://registry1.io|layer1", + "https://registry1.io|layer2", ), expectedManifestDeletions: sets.NewString( - "registry1.io|foo/bar|sha256:0000000000000000000000000000000000000000000000000000000000000001", + "https://registry1.io|foo/bar|sha256:0000000000000000000000000000000000000000000000000000000000000001", ), }, } @@ -1184,9 +1269,9 @@ func TestRegistryPruning(t *testing.T) { BCs: &buildapi.BuildConfigList{}, Builds: &buildapi.BuildList{}, DCs: &deployapi.DeploymentConfigList{}, + RegistryURL: &url.URL{Scheme: "https", Host: "registry1.io"}, } p := NewPruner(options) - p.(*pruner).registryPinger = &fakeRegistryPinger{err: test.pingErr} imageDeleter := &fakeImageDeleter{invocations: sets.NewString()} streamDeleter := &fakeImageStreamDeleter{invocations: sets.NewString()} @@ -1196,14 +1281,14 @@ func TestRegistryPruning(t *testing.T) { p.Prune(imageDeleter, streamDeleter, layerLinkDeleter, blobDeleter, manifestDeleter) - if !reflect.DeepEqual(test.expectedLayerLinkDeletions, layerLinkDeleter.invocations) { - t.Errorf("%s: expected layer link deletions %#v, got %#v", name, test.expectedLayerLinkDeletions.List(), layerLinkDeleter.invocations.List()) + if a, e := layerLinkDeleter.invocations, test.expectedLayerLinkDeletions; !reflect.DeepEqual(a, e) { + t.Errorf("%s: unexpected layer link deletions: %s", name, diff.ObjectDiff(a, e)) } - if !reflect.DeepEqual(test.expectedBlobDeletions, blobDeleter.invocations) { - t.Errorf("%s: expected blob deletions %#v, got %#v", name, test.expectedBlobDeletions.List(), blobDeleter.invocations.List()) + if a, e := blobDeleter.invocations, test.expectedBlobDeletions; !reflect.DeepEqual(a, e) { + t.Errorf("%s: unexpected blob deletions: %s", name, diff.ObjectDiff(a, e)) } - if !reflect.DeepEqual(test.expectedManifestDeletions, manifestDeleter.invocations) { - t.Errorf("%s: expected manifest deletions %#v, got %#v", name, test.expectedManifestDeletions.List(), manifestDeleter.invocations.List()) + if a, e := manifestDeleter.invocations, test.expectedManifestDeletions; !reflect.DeepEqual(a, e) { + t.Errorf("%s: unexpected manifest deletions: %s", name, diff.ObjectDiff(a, e)) } } } @@ -1254,7 +1339,6 @@ func TestImageWithStrongAndWeakRefsIsNotPruned(t *testing.T) { options.KeepYoungerThan = &keepYoungerThan options.KeepTagRevisions = &keepTagRevisions p := NewPruner(options) - p.(*pruner).registryPinger = &fakeRegistryPinger{} imageDeleter := &fakeImageDeleter{invocations: sets.NewString()} streamDeleter := &fakeImageStreamDeleter{invocations: sets.NewString()} diff --git a/pkg/oc/admin/prune/images.go b/pkg/oc/admin/prune/images.go index f477213d3ffa..5929a1a2a410 100644 --- a/pkg/oc/admin/prune/images.go +++ b/pkg/oc/admin/prune/images.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "os" "strings" "text/tabwriter" @@ -33,6 +34,8 @@ import ( // PruneImagesRecommendedName is the recommended command name const PruneImagesRecommendedName = "images" +var errNoToken = errors.New("you must use a client config with a token") + var ( imagesLongDesc = templates.LongDesc(` Remove image stream tags, images, and image layers by age or usage @@ -52,11 +55,13 @@ var ( authority other than the one present in current user's config, you may need to specify it using --certificate-authority flag. - Insecure connection is allowed in following cases unless certificate-authority is specified: - 1. --force-insecure is given + Insecure connection is allowed in the following cases unless certificate-authority is + specified: + + 1. --force-insecure is given 2. user's config allows for insecure connection (the user logged in to the cluster with - --insecure-skip-tls-verify or allowed for insecure connection) - 3. registry url is not given or it's a private/link-local address`) + --insecure-skip-tls-verify or allowed for insecure connection) + 3. registry url is a private or link-local address`) imagesExample = templates.Examples(` # See, what the prune command would delete if only images more than an hour old and obsoleted @@ -92,11 +97,10 @@ type PruneImagesOptions struct { Namespace string ForceInsecure bool - OSClient client.Interface - KClient kclientset.Interface - RegistryClient *http.Client - Out io.Writer - Insecure bool + ClientConfig *restclient.Config + OSClient client.Interface + KubeClient kclientset.Interface + Out io.Writer } // NewCmdPruneImages implements the OpenShift cli prune images command. @@ -168,18 +172,14 @@ func (o *PruneImagesOptions) Complete(f *clientcmd.Factory, cmd *cobra.Command, if err != nil { return err } + o.ClientConfig = clientConfig - o.Insecure = o.ForceInsecure - if !o.Insecure && len(o.CABundle) == 0 { - o.Insecure = clientConfig.TLSClientConfig.Insecure || len(o.RegistryUrlOverride) == 0 || netutils.IsPrivateAddress(o.RegistryUrlOverride) - } - osClient, kClient, registryClient, err := getClients(f, o.CABundle, o.Insecure) + osClient, kubeClient, err := getClients(f) if err != nil { return err } o.OSClient = osClient - o.KClient = kClient - o.RegistryClient = registryClient + o.KubeClient = kubeClient return nil } @@ -218,12 +218,12 @@ func (o PruneImagesOptions) Run() error { return err } - allPods, err := o.KClient.Core().Pods(o.Namespace).List(metav1.ListOptions{}) + allPods, err := o.KubeClient.Core().Pods(o.Namespace).List(metav1.ListOptions{}) if err != nil { return err } - allRCs, err := o.KClient.Core().ReplicationControllers(o.Namespace).List(metav1.ListOptions{}) + allRCs, err := o.KubeClient.Core().ReplicationControllers(o.Namespace).List(metav1.ListOptions{}) if err != nil { return err } @@ -247,7 +247,7 @@ func (o PruneImagesOptions) Run() error { return err } - limitRangesList, err := o.KClient.Core().LimitRanges(o.Namespace).List(metav1.ListOptions{}) + limitRangesList, err := o.KubeClient.Core().LimitRanges(o.Namespace).List(metav1.ListOptions{}) if err != nil { return err } @@ -262,6 +262,42 @@ func (o PruneImagesOptions) Run() error { limitRangesMap[limit.Namespace] = limits } + var ( + registryHost = o.RegistryUrlOverride + registryClient *http.Client + registryPinger prune.RegistryPinger + ) + + if o.Confirm { + if len(registryHost) == 0 { + registryHost, err = prune.DetermineRegistryHost(allImages, allStreams) + if err != nil { + return fmt.Errorf("unable to determine registry: %v", err) + } + } + + insecure := o.ForceInsecure + if !insecure && len(o.CABundle) == 0 { + insecure = o.ClientConfig.TLSClientConfig.Insecure || netutils.IsPrivateAddress(registryHost) + } + + registryClient, err = getRegistryClient(o.ClientConfig, o.CABundle, insecure) + if err != nil { + return err + } + registryPinger = &prune.DefaultRegistryPinger{ + Client: registryClient, + Insecure: insecure, + } + } else { + registryPinger = &prune.DryRunRegistryPinger{} + } + + registryURL, err := registryPinger.Ping(registryHost) + if err != nil { + return fmt.Errorf("error communicating with registry %s: %v", registryHost, err) + } + options := prune.PrunerOptions{ KeepYoungerThan: o.KeepYoungerThan, KeepTagRevisions: o.KeepTagRevisions, @@ -276,9 +312,8 @@ func (o PruneImagesOptions) Run() error { DCs: allDCs, LimitRanges: limitRangesMap, DryRun: o.Confirm == false, - RegistryClient: o.RegistryClient, - RegistryURL: o.RegistryUrlOverride, - Insecure: o.Insecure, + RegistryClient: registryClient, + RegistryURL: registryURL, } if o.Namespace != metav1.NamespaceAll { options.Namespace = o.Namespace @@ -379,7 +414,7 @@ type describingLayerLinkDeleter struct { var _ prune.LayerLinkDeleter = &describingLayerLinkDeleter{} -func (p *describingLayerLinkDeleter) DeleteLayerLink(registryClient *http.Client, registryURL, repo, name string) error { +func (p *describingLayerLinkDeleter) DeleteLayerLink(registryClient *http.Client, registryURL *url.URL, repo, name string) error { if !p.headerPrinted { p.headerPrinted = true fmt.Fprintln(p.w, "\nDeleting registry repository layer links ...") @@ -410,7 +445,7 @@ type describingBlobDeleter struct { var _ prune.BlobDeleter = &describingBlobDeleter{} -func (p *describingBlobDeleter) DeleteBlob(registryClient *http.Client, registryURL, layer string) error { +func (p *describingBlobDeleter) DeleteBlob(registryClient *http.Client, registryURL *url.URL, layer string) error { if !p.headerPrinted { p.headerPrinted = true fmt.Fprintln(p.w, "\nDeleting registry layer blobs ...") @@ -442,7 +477,7 @@ type describingManifestDeleter struct { var _ prune.ManifestDeleter = &describingManifestDeleter{} -func (p *describingManifestDeleter) DeleteManifest(registryClient *http.Client, registryURL, repo, manifest string) error { +func (p *describingManifestDeleter) DeleteManifest(registryClient *http.Client, registryURL *url.URL, repo, manifest string) error { if !p.headerPrinted { p.headerPrinted = true fmt.Fprintln(p.w, "\nDeleting registry repository manifest data ...") @@ -457,46 +492,48 @@ func (p *describingManifestDeleter) DeleteManifest(registryClient *http.Client, err := p.delegate.DeleteManifest(registryClient, registryURL, repo, manifest) if err != nil { - fmt.Fprintf(os.Stderr, "error deleting data for repository %s image manifest %s from the registry: %v\n", repo, manifest, err) + fmt.Fprintf(os.Stderr, "error deleting manifest %s from repository %s: %v\n", manifest, repo, err) } return err } -// getClients returns a Kube client, OpenShift client, and registry client. Note that -// registryCABundle and registryInsecure=true are mutually exclusive. If registryInsecure=true is -// specified, the ca bundle is ignored. -func getClients(f *clientcmd.Factory, registryCABundle string, registryInsecure bool) (*client.Client, kclientset.Interface, *http.Client, error) { +// getClients returns a OpenShift client and Kube client. +func getClients(f *clientcmd.Factory) (*client.Client, kclientset.Interface, error) { clientConfig, err := f.ClientConfig() if err != nil { - return nil, nil, nil, err + return nil, nil, err + } + + if len(clientConfig.BearerToken) == 0 { + return nil, nil, errNoToken + } + + osClient, kubeClient, err := f.Clients() + if err != nil { + return nil, nil, err } + return osClient, kubeClient, err +} +// getRegistryClient returns a registry client. Note that registryCABundle and registryInsecure=true are +// mutually exclusive. If registryInsecure=true is specified, the ca bundle is ignored. +func getRegistryClient(clientConfig *restclient.Config, registryCABundle string, registryInsecure bool) (*http.Client, error) { var ( - token string - osClient *client.Client - kClient kclientset.Interface - registryClient *http.Client + err error + cadata []byte + registryCABundleIncluded = false + token = clientConfig.BearerToken ) - switch { - case len(clientConfig.BearerToken) > 0: - osClient, kClient, err = f.Clients() - if err != nil { - return nil, nil, nil, err - } - token = clientConfig.BearerToken - default: - err = errors.New("you must use a client config with a token") - return nil, nil, nil, err + if len(token) == 0 { + return nil, errNoToken } - cadata := []byte{} - registryCABundleIncluded := false if len(registryCABundle) > 0 { cadata, err = ioutil.ReadFile(registryCABundle) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to read registry ca bundle: %v", err) + return nil, fmt.Errorf("failed to read registry ca bundle: %v", err) } } @@ -532,7 +569,7 @@ func getClients(f *clientcmd.Factory, registryCABundle string, registryInsecure tlsConfig, err := restclient.TLSConfigFor(®istryClientConfig) if err != nil { - return nil, nil, nil, err + return nil, err } // Add the CA bundle to the client config's CA roots if provided and we haven't done that already. @@ -550,12 +587,10 @@ func getClients(f *clientcmd.Factory, registryCABundle string, registryInsecure wrappedTransport, err := restclient.HTTPWrappersForConfig(®istryClientConfig, transport) if err != nil { - return nil, nil, nil, err + return nil, err } - registryClient = &http.Client{ + return &http.Client{ Transport: wrappedTransport, - } - - return osClient, kClient, registryClient, nil + }, nil } diff --git a/pkg/oc/admin/prune/images_test.go b/pkg/oc/admin/prune/images_test.go index caf0f412921e..2b4266c3b2ff 100644 --- a/pkg/oc/admin/prune/images_test.go +++ b/pkg/oc/admin/prune/images_test.go @@ -15,9 +15,9 @@ func TestImagePruneNamespaced(t *testing.T) { opts := &PruneImagesOptions{ Namespace: "foo", - OSClient: osFake, - KClient: kFake, - Out: ioutil.Discard, + OSClient: osFake, + KubeClient: kFake, + Out: ioutil.Discard, } if err := opts.Run(); err != nil { From 7fd22bb9436d6628f671ecdde8297c8c2365955d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Min=C3=A1=C5=99?= Date: Mon, 31 Jul 2017 14:01:55 +0200 Subject: [PATCH 2/2] image-pruner: Reenable registry-url validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Michal Minář --- pkg/image/apis/image/helper.go | 42 +++++++++++++++ pkg/image/apis/image/helper_test.go | 82 +++++++++++++++++++++++++++++ pkg/oc/admin/prune/images.go | 31 +++++++---- 3 files changed, 144 insertions(+), 11 deletions(-) diff --git a/pkg/image/apis/image/helper.go b/pkg/image/apis/image/helper.go index 2100045d5229..b334a2b10d32 100644 --- a/pkg/image/apis/image/helper.go +++ b/pkg/image/apis/image/helper.go @@ -41,6 +41,10 @@ const ( ImportRegistryNotAllowed = "registry is not allowed for import" ) +var errNoRegistryURLPathAllowed = fmt.Errorf("no path after [:] is allowed") +var errNoRegistryURLQueryAllowed = fmt.Errorf("no query arguments are allowed after [:]") +var errRegistryURLHostEmpty = fmt.Errorf("no host name specified") + // DefaultRegistry returns the default Docker registry (host or host:port), or false if it is not available. type DefaultRegistry interface { DefaultRegistry() (string, bool) @@ -1161,3 +1165,41 @@ func (tagref TagReference) HasAnnotationTag(searchTag string) bool { } return false } + +// ValidateRegistryURL returns error if the given input is not a valid registry URL. The url may be prefixed +// with http:// or https:// schema. It may not contain any path or query after the host:[port]. +func ValidateRegistryURL(registryURL string) error { + var ( + u *url.URL + err error + parts = strings.SplitN(registryURL, "://", 2) + ) + + switch len(parts) { + case 2: + u, err = url.Parse(registryURL) + if err != nil { + return err + } + switch u.Scheme { + case "http", "https": + default: + return fmt.Errorf("unsupported scheme: %s", u.Scheme) + } + case 1: + u, err = url.Parse("https://" + registryURL) + if err != nil { + return err + } + } + if len(u.Path) > 0 && u.Path != "/" { + return errNoRegistryURLPathAllowed + } + if len(u.RawQuery) > 0 { + return errNoRegistryURLQueryAllowed + } + if len(u.Host) == 0 { + return errRegistryURLHostEmpty + } + return nil +} diff --git a/pkg/image/apis/image/helper_test.go b/pkg/image/apis/image/helper_test.go index e55e0e862bda..fadaec195bd5 100644 --- a/pkg/image/apis/image/helper_test.go +++ b/pkg/image/apis/image/helper_test.go @@ -1834,3 +1834,85 @@ func TestDockerImageReferenceForImage(t *testing.T) { t.Errorf("expected failure for unknown image") } } + +func TestValidateRegistryURL(t *testing.T) { + for _, tc := range []struct { + input string + expectedError bool + expectedErrorString string + }{ + {input: "172.30.30.30:5000"}, + {input: ":5000"}, + {input: "[fd12:3456:789a:1::1]:80/"}, + {input: "[fd12:3456:789a:1::1]:80"}, + {input: "http://172.30.30.30:5000"}, + {input: "http://[fd12:3456:789a:1::1]:5000/"}, + {input: "http://[fd12:3456:789a:1::1]:5000"}, + {input: "http://registry.org:5000"}, + {input: "https://172.30.30.30:5000"}, + {input: "https://:80/"}, + {input: "https://[fd12:3456:789a:1::1]/"}, + {input: "https://[fd12:3456:789a:1::1]"}, + {input: "https://[fd12:3456:789a:1::1]:5000/"}, + {input: "https://[fd12:3456:789a:1::1]:5000"}, + {input: "https://registry.org/"}, + {input: "https://registry.org"}, + {input: "localhost/"}, + {input: "localhost"}, + {input: "localhost:80"}, + {input: "registry.org/"}, + {input: "registry.org"}, + {input: "registry.org:5000"}, + + { + input: "httpss://registry.org", + expectedErrorString: "unsupported scheme: httpss", + }, + { + input: "ftp://registry.org", + expectedErrorString: "unsupported scheme: ftp", + }, + { + input: "http://registry.org://", + expectedErrorString: errNoRegistryURLPathAllowed.Error(), + }, + { + input: "http://registry.org/path", + expectedErrorString: errNoRegistryURLPathAllowed.Error(), + }, + { + input: "[fd12:3456:789a:1::1", + expectedError: true, + }, + { + input: "bad url", + expectedError: true, + }, + { + input: "/registry.org", + expectedErrorString: errNoRegistryURLPathAllowed.Error(), + }, + { + input: "https:///", + expectedErrorString: errRegistryURLHostEmpty.Error(), + }, + { + input: "http://registry.org?parm=arg", + expectedErrorString: errNoRegistryURLQueryAllowed.Error(), + }, + } { + + err := ValidateRegistryURL(tc.input) + if err != nil { + if len(tc.expectedErrorString) > 0 && err.Error() != tc.expectedErrorString { + t.Errorf("[%s] unexpected error string: %q != %q", tc.input, err.Error(), tc.expectedErrorString) + } else if len(tc.expectedErrorString) == 0 && !tc.expectedError { + t.Errorf("[%s] unexpected error: %q", tc.input, err.Error()) + } + } else if len(tc.expectedErrorString) > 0 { + t.Errorf("[%s] got non-error while expecting %q", tc.input, tc.expectedErrorString) + } else if tc.expectedError { + t.Errorf("[%s] got unexpected non-error", tc.input) + } + } +} diff --git a/pkg/oc/admin/prune/images.go b/pkg/oc/admin/prune/images.go index 5929a1a2a410..4b6c60d2383b 100644 --- a/pkg/oc/admin/prune/images.go +++ b/pkg/oc/admin/prune/images.go @@ -59,9 +59,10 @@ var ( specified: 1. --force-insecure is given - 2. user's config allows for insecure connection (the user logged in to the cluster with - --insecure-skip-tls-verify or allowed for insecure connection) - 3. registry url is a private or link-local address`) + 2. provided registry-url is prefixed with http:// + 3. registry url is a private or link-local address + 4. user's config allows for insecure connection (the user logged in to the cluster with + --insecure-skip-tls-verify or allowed for insecure connection)`) imagesExample = templates.Examples(` # See, what the prune command would delete if only images more than an hour old and obsoleted @@ -76,7 +77,13 @@ var ( %[1]s %[2]s --prune-over-size-limit # To actually perform the prune operation, the confirm flag must be appended - %[1]s %[2]s --prune-over-size-limit --confirm`) + %[1]s %[2]s --prune-over-size-limit --confirm + + # Force the insecure http protocol with the particular registry host name + %[1]s %[2]s --registry-url=http://registry.example.org --confirm + + # Force a secure connection with a custom certificate authority to the particular registry host name + %[1]s %[2]s --registry-url=registry.example.org --certificate-authority=/path/to/custom/ca.crt --confirm`) ) var ( @@ -134,7 +141,7 @@ func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Wri cmd.Flags().IntVar(opts.KeepTagRevisions, "keep-tag-revisions", *opts.KeepTagRevisions, "Specify the number of image revisions for a tag in an image stream that will be preserved.") cmd.Flags().BoolVar(opts.PruneOverSizeLimit, "prune-over-size-limit", *opts.PruneOverSizeLimit, "Specify if images which are exceeding LimitRanges (see 'openshift.io/Image'), specified in the same namespace, should be considered for pruning. This flag cannot be combined with --keep-younger-than nor --keep-tag-revisions.") cmd.Flags().StringVar(&opts.CABundle, "certificate-authority", opts.CABundle, "The path to a certificate authority bundle to use when communicating with the managed Docker registries. Defaults to the certificate authority data from the current user's config file. It cannot be used together with --force-insecure.") - cmd.Flags().StringVar(&opts.RegistryUrlOverride, "registry-url", opts.RegistryUrlOverride, "The address to use when contacting the registry, instead of using the default value. This is useful if you can't resolve or reach the registry (e.g.; the default is a cluster-internal URL) but you do have an alternative route that works.") + cmd.Flags().StringVar(&opts.RegistryUrlOverride, "registry-url", opts.RegistryUrlOverride, "The address to use when contacting the registry, instead of using the default value. This is useful if you can't resolve or reach the registry (e.g.; the default is a cluster-internal URL) but you do have an alternative route that works. Particular transport protocol can be enforced using '://' prefix.") cmd.Flags().BoolVar(&opts.ForceInsecure, "force-insecure", opts.ForceInsecure, "If true, allow an insecure connection to the docker registry that is hosted via HTTP or has an invalid HTTPS certificate. Whenever possible, use --certificate-authority instead of this dangerous option.") return cmd @@ -195,14 +202,15 @@ func (o PruneImagesOptions) Validate() error { if o.KeepTagRevisions != nil && *o.KeepTagRevisions < 0 { return fmt.Errorf("--keep-tag-revisions must be greater than or equal to 0") } - // golang validation tighten and our code actually expects this to be scheme-less - // TODO figure out how to validate - // if _, err := url.Parse(o.RegistryUrlOverride); err != nil { - // return fmt.Errorf("invalid --registry-url flag: %v", err) - // } + if err := imageapi.ValidateRegistryURL(o.RegistryUrlOverride); len(o.RegistryUrlOverride) > 0 && err != nil { + return fmt.Errorf("invalid --registry-url flag: %v", err) + } if o.ForceInsecure && len(o.CABundle) > 0 { return fmt.Errorf("--certificate-authority cannot be specified with --force-insecure") } + if len(o.CABundle) > 0 && strings.HasPrefix(o.RegistryUrlOverride, "http://") { + return fmt.Errorf("--cerificate-authority cannot be specified for insecure http protocol") + } return nil } @@ -278,7 +286,8 @@ func (o PruneImagesOptions) Run() error { insecure := o.ForceInsecure if !insecure && len(o.CABundle) == 0 { - insecure = o.ClientConfig.TLSClientConfig.Insecure || netutils.IsPrivateAddress(registryHost) + insecure = o.ClientConfig.TLSClientConfig.Insecure || netutils.IsPrivateAddress(registryHost) || + strings.HasPrefix(registryHost, "http://") } registryClient, err = getRegistryClient(o.ClientConfig, o.CABundle, insecure)