From 6b92879e969b2dd855829d093eb49385cfb659ea Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Thu, 31 Oct 2019 22:03:43 -0400 Subject: [PATCH 1/2] feat(resolver): rely on info from catalog to determine updates --- go.mod | 2 +- go.sum | 2 + pkg/controller/registry/resolver/evolver.go | 7 +- pkg/controller/registry/resolver/operators.go | 32 +- .../registry/resolver/operators_test.go | 22 +- pkg/controller/registry/resolver/querier.go | 18 +- .../registry/resolver/querier_test.go | 4 +- pkg/controller/registry/resolver/util_test.go | 1 + .../operator-registry/pkg/api/registry.pb.go | 105 +-- .../operator-registry/pkg/api/registry.proto | 2 + .../pkg/containertools/bundlereader.go | 186 ++++++ .../pkg/containertools/command.go | 66 ++ .../operator-registry/pkg/registry/bundle.go | 25 +- .../pkg/registry/interface.go | 3 + .../operator-registry/pkg/registry/types.go | 49 ++ .../operator-registry/pkg/sqlite/directory.go | 97 +-- .../operator-registry/pkg/sqlite/image.go | 239 +++++++ .../operator-registry/pkg/sqlite/load.go | 601 +++++++++++++++++- .../pkg/sqlite/migrations/000_init.go | 8 +- .../sqlite/migrations/001_related_images.go | 2 +- .../pkg/sqlite/migrations/002_bundle_path.go | 10 +- .../sqlite/migrations/003_required_apis.go | 2 +- .../sqlite/migrations/004_cascade_delete.go | 453 +++++++++++++ .../migrations/005_version_skiprange.go | 94 +++ .../pkg/sqlite/migrations/migrations.go | 11 + .../operator-registry/pkg/sqlite/migrator.go | 26 +- .../operator-registry/pkg/sqlite/query.go | 63 +- .../operator-registry/pkg/sqlite/remove.go | 67 ++ vendor/modules.txt | 3 +- 29 files changed, 1999 insertions(+), 201 deletions(-) create mode 100644 vendor/github.com/operator-framework/operator-registry/pkg/containertools/bundlereader.go create mode 100644 vendor/github.com/operator-framework/operator-registry/pkg/containertools/command.go create mode 100644 vendor/github.com/operator-framework/operator-registry/pkg/sqlite/image.go create mode 100644 vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/004_cascade_delete.go create mode 100644 vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/005_version_skiprange.go create mode 100644 vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go diff --git a/go.mod b/go.mod index f00f63084f..d48324d168 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20190414153302-2ae31c8b6b30 // indirect github.com/openshift/api v3.9.1-0.20190924102528-32369d4db2ad+incompatible github.com/openshift/client-go v0.0.0-20190923180330-3b6373338c9b - github.com/operator-framework/operator-registry v1.5.1 + github.com/operator-framework/operator-registry v1.5.3 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index aba4ad1866..6b16596369 100644 --- a/go.sum +++ b/go.sum @@ -438,6 +438,8 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/operator-framework/operator-registry v1.5.1 h1:8ruUOG6IBDVTAXYWKsv6hwr4yv/0SFPFPAYGCpcv97E= github.com/operator-framework/operator-registry v1.5.1/go.mod h1:agrQlkWOo1q8U1SAaLSS2WQ+Z9vswNT2M2HFib9iuLY= +github.com/operator-framework/operator-registry v1.5.3 h1:az83WDwgB+tHsmVn+tFq72yQBbaUAye8e4+KkDQmzLs= +github.com/operator-framework/operator-registry v1.5.3/go.mod h1:agrQlkWOo1q8U1SAaLSS2WQ+Z9vswNT2M2HFib9iuLY= github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA= github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= diff --git a/pkg/controller/registry/resolver/evolver.go b/pkg/controller/registry/resolver/evolver.go index f82487225b..8d630e4ddf 100644 --- a/pkg/controller/registry/resolver/evolver.go +++ b/pkg/controller/registry/resolver/evolver.go @@ -63,10 +63,11 @@ func (e *NamespaceGenerationEvolver) checkForUpdates() error { continue } - o, err := NewOperatorFromBundle(bundle, op.Identifier(), op.SourceInfo().StartingCSV, *key) + o, err := NewOperatorFromBundle(bundle, op.SourceInfo().StartingCSV, *key) if err != nil { return errors.Wrap(err, "error parsing bundle") } + o.SetReplaces(op.Identifier()) if err := e.gen.AddOperator(o); err != nil { return errors.Wrap(err, "error calculating generation changes due to new bundle") } @@ -90,7 +91,7 @@ func (e *NamespaceGenerationEvolver) addNewOperators(add map[OperatorSourceInfo] return errors.Wrapf(err, "%s not found", s) } - o, err := NewOperatorFromBundle(bundle, "", s.StartingCSV, *key) + o, err := NewOperatorFromBundle(bundle, s.StartingCSV, *key) if err != nil { return errors.Wrap(err, "error parsing bundle") } @@ -123,7 +124,7 @@ func (e *NamespaceGenerationEvolver) queryForRequiredAPIs() error { // attempt to find a bundle that provides that api if bundle, key, err := e.querier.FindProvider(*api, initialSource); err == nil { // add a bundle that provides the api to the generation - o, err := NewOperatorFromBundle(bundle, "", "", *key) + o, err := NewOperatorFromBundle(bundle, "", *key) if err != nil { return errors.Wrap(err, "error parsing bundle") } diff --git a/pkg/controller/registry/resolver/operators.go b/pkg/controller/registry/resolver/operators.go index ea9d0165b5..2b5e3eef7e 100644 --- a/pkg/controller/registry/resolver/operators.go +++ b/pkg/controller/registry/resolver/operators.go @@ -238,24 +238,11 @@ type Operator struct { var _ OperatorSurface = &Operator{} -func NewOperatorFromBundle(bundle *api.Bundle, replaces string, startingCSV string, sourceKey CatalogKey) (*Operator, error) { - if bundle.CsvJson == "" { - return nil, fmt.Errorf("no csv json found") - } - csv := ®istry.ClusterServiceVersion{} - if err := json.Unmarshal([]byte(bundle.CsvJson), csv); err != nil { - return nil, err - } - r := replaces - if r == "" { - r, _ = csv.GetReplaces() - } - - version, _ := csv.GetVersion() - parsedVersion, err := semver.ParseTolerant(version) - v := &parsedVersion +func NewOperatorFromBundle(bundle *api.Bundle, startingCSV string, sourceKey CatalogKey) (*Operator, error) { + parsedVersion, err := semver.ParseTolerant(bundle.Version) + version := &parsedVersion if err != nil { - v = nil + version = nil } provided := APISet{} for _, gvk := range bundle.ProvidedApis { @@ -290,14 +277,12 @@ func NewOperatorFromBundle(bundle *api.Bundle, replaces string, startingCSV stri } op.sourceInfo = sourceInfo op.bundle = bundle - op.replaces = r return op, nil } return &Operator{ - name: csv.GetName(), - replaces: r, - version: v, + name: bundle.CsvName, + version: version, providedAPIs: provided, requiredAPIs: required, bundle: bundle, @@ -333,7 +318,6 @@ func NewOperatorFromV1Alpha1CSV(csv *v1alpha1.ClusterServiceVersion) (*Operator, return &Operator{ name: csv.GetName(), version: &csv.Spec.Version.Version, - replaces: csv.Spec.Replaces, providedAPIs: providedAPIs, requiredAPIs: requiredAPIs, sourceInfo: &ExistingOperator, @@ -356,6 +340,10 @@ func (o *Operator) Replaces() string { return o.replaces } +func (o *Operator) SetReplaces(replacing string) { + o.replaces = replacing +} + func (o *Operator) Package() string { return o.bundle.PackageName } diff --git a/pkg/controller/registry/resolver/operators_test.go b/pkg/controller/registry/resolver/operators_test.go index e99c0946a6..a26242dae8 100644 --- a/pkg/controller/registry/resolver/operators_test.go +++ b/pkg/controller/registry/resolver/operators_test.go @@ -917,6 +917,7 @@ func TestNewOperatorFromBundle(t *testing.T) { CsvName: "testBundle", PackageName: "testPackage", ChannelName: "testChannel", + Version: version.String(), CsvJson: string(csvJson), Object: []string{string(csvJson)}, } @@ -987,6 +988,7 @@ func TestNewOperatorFromBundle(t *testing.T) { CsvName: "testBundle", PackageName: "testPackage", ChannelName: "testChannel", + Version: version.String(), CsvJson: string(csvJsonWithApis), Object: []string{string(csvJsonWithApis), string(crdJson)}, ProvidedApis: []*api.GroupVersionKind{ @@ -1043,12 +1045,11 @@ func TestNewOperatorFromBundle(t *testing.T) { args: args{ bundle: bundleNoAPIs, sourceKey: CatalogKey{Name: "source", Namespace: "testNamespace"}, - replaces: "", }, want: &Operator{ + // lack of full api response falls back to csv name name: "testCSV", version: &version.Version, - replaces: "v1", providedAPIs: EmptyAPISet(), requiredAPIs: EmptyAPISet(), bundle: bundleNoAPIs, @@ -1064,12 +1065,10 @@ func TestNewOperatorFromBundle(t *testing.T) { args: args{ bundle: bundleWithAPIs, sourceKey: CatalogKey{Name: "source", Namespace: "testNamespace"}, - replaces: "", }, want: &Operator{ - name: "testCSV", - version: &version.Version, - replaces: "v1", + name: "testBundle", + version: &version.Version, providedAPIs: APISet{ opregistry.APIKey{ Group: "crd.group.com", @@ -1111,14 +1110,13 @@ func TestNewOperatorFromBundle(t *testing.T) { args: args{ bundle: bundleNoAPIs, sourceKey: CatalogKey{Name: "source", Namespace: "testNamespace"}, - replaces: "replaced", }, want: &Operator{ + // lack of full api response falls back to csv name name: "testCSV", providedAPIs: EmptyAPISet(), requiredAPIs: EmptyAPISet(), bundle: bundleNoAPIs, - replaces: "replaced", version: &version.Version, sourceInfo: &OperatorSourceInfo{ Package: "testPackage", @@ -1132,7 +1130,6 @@ func TestNewOperatorFromBundle(t *testing.T) { args: args{ bundle: bundleWithAPIsUnextracted, sourceKey: CatalogKey{Name: "source", Namespace: "testNamespace"}, - replaces: "replaced", }, want: &Operator{ name: "testCSV", @@ -1164,9 +1161,8 @@ func TestNewOperatorFromBundle(t *testing.T) { Plural: "requiredapis", }: struct{}{}, }, - bundle: bundleWithAPIsUnextracted, - replaces: "replaced", - version: &version.Version, + bundle: bundleWithAPIsUnextracted, + version: &version.Version, sourceInfo: &OperatorSourceInfo{ Package: "testPackage", Channel: "testChannel", @@ -1177,7 +1173,7 @@ func TestNewOperatorFromBundle(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewOperatorFromBundle(tt.args.bundle, tt.args.replaces, "", tt.args.sourceKey) + got, err := NewOperatorFromBundle(tt.args.bundle, "", tt.args.sourceKey) require.Equal(t, tt.wantErr, err) require.Equal(t, tt.want, got) }) diff --git a/pkg/controller/registry/resolver/querier.go b/pkg/controller/registry/resolver/querier.go index 93402334ea..b85d29a4e3 100644 --- a/pkg/controller/registry/resolver/querier.go +++ b/pkg/controller/registry/resolver/querier.go @@ -4,13 +4,10 @@ package resolver import ( "context" "fmt" - "strings" "github.com/blang/semver" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/yaml" "github.com/operator-framework/operator-registry/pkg/api" "github.com/operator-framework/operator-registry/pkg/client" @@ -180,22 +177,11 @@ func (q *NamespaceSourceQuerier) findChannelHead(currentVersion *semver.Version, return nil, err } - if latest.CsvJson == "" { + if latest.SkipRange == "" { return nil, nil } - dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(latest.CsvJson), 10) - unst := &unstructured.Unstructured{} - if err := dec.Decode(unst); err != nil { - return nil, err - } - - skipRange, ok := unst.GetAnnotations()[SkipPackageAnnotationKey] - if !ok { - return nil, nil - } - - r, err := semver.ParseRange(skipRange) + r, err := semver.ParseRange(latest.SkipRange) if err != nil { return nil, err } diff --git a/pkg/controller/registry/resolver/querier_test.go b/pkg/controller/registry/resolver/querier_test.go index 201579f4f0..53afae5f48 100644 --- a/pkg/controller/registry/resolver/querier_test.go +++ b/pkg/controller/registry/resolver/querier_test.go @@ -360,7 +360,7 @@ func TestNamespaceSourceQuerier_FindReplacement(t *testing.T) { require.NoError(t, err) nextBundle := &api.Bundle{CsvName: "test.v1", PackageName: "testPkg", ChannelName: "testChannel"} - latestBundle := &api.Bundle{CsvName: "latest", PackageName: "testPkg", ChannelName: "testChannel", CsvJson: string(csvJson), Object: []string{string(csvJson)}} + latestBundle := &api.Bundle{CsvName: "latest", PackageName: "testPkg", ChannelName: "testChannel", CsvJson: string(csvJson), Object: []string{string(csvJson)}, SkipRange: ">= 1.0.0-0 < 1.0.0-1556661308", Version: latestVersion.String()} csv.SetAnnotations(map[string]string{}) csvUnstNoAnnotationJson, err := json.Marshal(csv) @@ -491,7 +491,7 @@ func TestNamespaceSourceQuerier_FindReplacement(t *testing.T) { if err != nil { t.Log(err.Error()) } - require.Equal(t, tt.out.err, err) + require.Equal(t, tt.out.err, err, "%v", err) require.Equal(t, tt.out.bundle, got) require.Equal(t, tt.out.key, key) }) diff --git a/pkg/controller/registry/resolver/util_test.go b/pkg/controller/registry/resolver/util_test.go index 66b1c38382..4c25bb4408 100644 --- a/pkg/controller/registry/resolver/util_test.go +++ b/pkg/controller/registry/resolver/util_test.go @@ -253,6 +253,7 @@ func bundle(name, pkg, channel, replaces string, providedCRDs, requiredCRDs, pro ChannelName: channel, CsvJson: string(csvJson), Object: objs, + Version: "0.0.0", ProvidedApis: apiSetToGVk(providedCRDs, providedAPIServices), RequiredApis: apiSetToGVk(requiredCRDs, requiredAPIServices), } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go index 06a9a93095..4430ca9586 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go @@ -168,6 +168,8 @@ type Bundle struct { BundlePath string `protobuf:"bytes,6,opt,name=bundlePath" json:"bundlePath,omitempty"` ProvidedApis []*GroupVersionKind `protobuf:"bytes,7,rep,name=providedApis" json:"providedApis,omitempty"` RequiredApis []*GroupVersionKind `protobuf:"bytes,8,rep,name=requiredApis" json:"requiredApis,omitempty"` + Version string `protobuf:"bytes,9,opt,name=version" json:"version,omitempty"` + SkipRange string `protobuf:"bytes,10,opt,name=skipRange" json:"skipRange,omitempty"` } func (m *Bundle) Reset() { *m = Bundle{} } @@ -231,6 +233,20 @@ func (m *Bundle) GetRequiredApis() []*GroupVersionKind { return nil } +func (m *Bundle) GetVersion() string { + if m != nil { + return m.Version + } + return "" +} + +func (m *Bundle) GetSkipRange() string { + if m != nil { + return m.SkipRange + } + return "" +} + type ChannelEntry struct { PackageName string `protobuf:"bytes,1,opt,name=packageName" json:"packageName,omitempty"` ChannelName string `protobuf:"bytes,2,opt,name=channelName" json:"channelName,omitempty"` @@ -985,48 +1001,49 @@ var _Registry_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("registry.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 680 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x56, 0x4f, 0x6f, 0xd3, 0x30, - 0x14, 0x5f, 0xda, 0xad, 0xed, 0x5e, 0x2b, 0xb4, 0x99, 0x6d, 0x84, 0x80, 0xa6, 0xe2, 0x0b, 0x3b, - 0x55, 0x30, 0x40, 0x88, 0x03, 0x87, 0x8d, 0x41, 0x05, 0x0c, 0x34, 0x45, 0xfc, 0x39, 0x70, 0xf2, - 0x12, 0xd3, 0x86, 0x65, 0x4e, 0x66, 0x3b, 0x9b, 0xf6, 0x25, 0xb8, 0xf0, 0x3d, 0xf8, 0x8c, 0x28, - 0xb6, 0x93, 0x3a, 0x69, 0xba, 0x21, 0x21, 0xe0, 0xd6, 0xf7, 0xfc, 0xfe, 0xfc, 0xde, 0xcb, 0xef, - 0x67, 0x17, 0x6e, 0x70, 0x3a, 0x89, 0x84, 0xe4, 0x97, 0xa3, 0x94, 0x27, 0x32, 0x41, 0x6d, 0x92, - 0x46, 0xf8, 0x29, 0x74, 0x5f, 0x4c, 0x09, 0x63, 0x34, 0x46, 0x08, 0x96, 0x19, 0x39, 0xa5, 0xae, - 0x33, 0x74, 0x76, 0x56, 0x7d, 0xf5, 0x1b, 0xb9, 0xd0, 0x0d, 0xc4, 0xf9, 0xfb, 0xdc, 0xdd, 0x52, - 0xee, 0xc2, 0xc4, 0xf7, 0xa0, 0x7f, 0x44, 0x82, 0x13, 0x32, 0xa1, 0xb9, 0xd9, 0x94, 0x8c, 0x2f, - 0xa0, 0x6b, 0x42, 0x1a, 0x6b, 0xef, 0x40, 0x2f, 0xd0, 0xad, 0x85, 0xdb, 0x1a, 0xb6, 0x77, 0xfa, - 0xbb, 0x83, 0x11, 0x49, 0xa3, 0x91, 0xc1, 0xe3, 0x97, 0xa7, 0x68, 0x04, 0x28, 0xa4, 0x5f, 0x49, - 0x16, 0x4b, 0x73, 0xa6, 0x00, 0xb5, 0x55, 0xad, 0x86, 0x13, 0xcc, 0x60, 0x6d, 0xcc, 0x93, 0x2c, - 0xfd, 0x44, 0xb9, 0x88, 0x12, 0xf6, 0x36, 0x62, 0x21, 0xda, 0x80, 0x95, 0x49, 0xee, 0x33, 0x10, - 0xb4, 0x91, 0xcf, 0x77, 0xae, 0x83, 0x8a, 0xf9, 0x8c, 0x99, 0x23, 0x3e, 0x89, 0x58, 0x68, 0xba, - 0xa8, 0xdf, 0x68, 0x0b, 0x3a, 0x69, 0x9c, 0x71, 0x12, 0xbb, 0xcb, 0xca, 0x6b, 0x2c, 0xfc, 0xb3, - 0x05, 0x9d, 0xfd, 0x8c, 0x85, 0x71, 0x65, 0x61, 0x4e, 0x65, 0x61, 0x68, 0x08, 0xfd, 0x74, 0xb6, - 0x30, 0xd3, 0xce, 0x76, 0xe5, 0x11, 0xc1, 0xdc, 0x7c, 0xb6, 0xcb, 0x54, 0x7f, 0x23, 0x12, 0x66, - 0x10, 0x14, 0x66, 0x0e, 0x2d, 0x39, 0xfe, 0x46, 0x03, 0xe9, 0xae, 0x0c, 0xdb, 0x39, 0x34, 0x6d, - 0xa1, 0x6d, 0x80, 0x63, 0x85, 0xec, 0x88, 0xc8, 0xa9, 0xdb, 0x51, 0x49, 0x96, 0x07, 0x3d, 0x83, - 0x41, 0xca, 0x93, 0xf3, 0x28, 0xa4, 0xe1, 0x5e, 0x1a, 0x09, 0xb7, 0xab, 0x3e, 0xc4, 0xa6, 0xfa, - 0x10, 0xf5, 0x1d, 0xfa, 0x95, 0xd0, 0x3c, 0x95, 0xd3, 0xb3, 0x2c, 0xe2, 0x26, 0xb5, 0x77, 0x65, - 0xaa, 0x1d, 0x8a, 0xbf, 0x3b, 0x30, 0x30, 0x1f, 0xec, 0x25, 0x93, 0xfc, 0xb2, 0xbe, 0x1c, 0xe7, - 0xda, 0xe5, 0xb4, 0xe6, 0x97, 0x53, 0x8e, 0x6a, 0x6d, 0xcf, 0xf2, 0x20, 0x0f, 0x7a, 0x9c, 0xa6, - 0x31, 0x09, 0xa8, 0x30, 0xdb, 0x2b, 0x6d, 0xbc, 0x01, 0xe8, 0x30, 0x12, 0xd2, 0xd0, 0xd5, 0xa7, - 0x67, 0x19, 0x15, 0x12, 0xdf, 0x87, 0xf5, 0x31, 0xad, 0x39, 0x1b, 0x99, 0x3e, 0x85, 0xb5, 0x31, - 0x95, 0x9a, 0x02, 0x45, 0x9c, 0x0b, 0xdd, 0xf4, 0x64, 0x62, 0x33, 0xc1, 0x98, 0xbf, 0x31, 0x8a, - 0xc5, 0xa2, 0x76, 0x55, 0x76, 0x9f, 0xe1, 0x76, 0xd9, 0xe9, 0x35, 0x2b, 0xa4, 0xf2, 0xe7, 0x2d, - 0xf1, 0x13, 0x55, 0x78, 0x2f, 0x8e, 0x7d, 0xbd, 0x93, 0x53, 0xca, 0xa4, 0xb0, 0x0a, 0x37, 0xb3, - 0x1a, 0x9f, 0xc2, 0xe6, 0x98, 0x4a, 0x2b, 0xe7, 0xda, 0x14, 0x1b, 0x65, 0xeb, 0x4a, 0x94, 0xf3, - 0x02, 0xc0, 0x12, 0xb6, 0x34, 0xca, 0x23, 0xcd, 0x44, 0x5e, 0x42, 0xfc, 0x9b, 0xfa, 0xbe, 0x50, - 0xbb, 0x39, 0x24, 0x92, 0x0a, 0xf9, 0x1f, 0x1a, 0x1f, 0xe8, 0x1b, 0xae, 0xe8, 0xfc, 0x0f, 0x1a, - 0xef, 0xfe, 0x58, 0x81, 0x9e, 0x6f, 0x9e, 0x0b, 0xf4, 0x1c, 0x06, 0x96, 0x38, 0x04, 0xba, 0xa5, - 0x24, 0x3e, 0xaf, 0x17, 0x6f, 0x4d, 0x1d, 0x58, 0xcf, 0x02, 0x5e, 0x7a, 0xe0, 0xa0, 0xc7, 0x00, - 0x33, 0x15, 0xa1, 0x2d, 0x7d, 0x3f, 0xd4, 0x65, 0xe5, 0x0d, 0xec, 0x5c, 0xbc, 0x84, 0x1e, 0xc2, - 0x6a, 0x49, 0x74, 0xb4, 0x59, 0x24, 0x55, 0x24, 0xe6, 0xf5, 0x95, 0x5b, 0xfb, 0xf0, 0x12, 0x3a, - 0x80, 0x9b, 0x65, 0xc8, 0xab, 0x84, 0x17, 0xef, 0xda, 0x76, 0x35, 0xb9, 0xae, 0x9a, 0x7a, 0x95, - 0x8f, 0x70, 0x77, 0x4c, 0xa5, 0x75, 0x3b, 0x45, 0x54, 0x7c, 0x98, 0x92, 0x82, 0xe3, 0xb3, 0x72, - 0xcd, 0x5a, 0xf1, 0xd6, 0xed, 0x47, 0x4c, 0xdd, 0x6e, 0x6a, 0x0b, 0xfb, 0x4a, 0x28, 0xba, 0x8b, - 0x55, 0x4e, 0x20, 0xaf, 0xa8, 0x37, 0x2f, 0xa2, 0x3a, 0x34, 0x7f, 0x01, 0x34, 0xc3, 0x0c, 0x74, - 0xc7, 0x82, 0x56, 0xe7, 0xe9, 0x22, 0x5c, 0x5f, 0x00, 0x97, 0xdc, 0x5e, 0x5c, 0xb9, 0x1c, 0xba, - 0x59, 0x04, 0x8b, 0x8a, 0xbf, 0x53, 0x80, 0x0d, 0x7f, 0x67, 0xb3, 0x9b, 0x74, 0x31, 0x2b, 0xdb, - 0x4c, 0xf1, 0xda, 0xfc, 0xc7, 0x1d, 0xf5, 0xc7, 0xe5, 0xd1, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xf7, 0x7c, 0x82, 0x4a, 0xca, 0x08, 0x00, 0x00, + // 701 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x56, 0x5d, 0x6f, 0xd3, 0x3c, + 0x14, 0x5e, 0xdb, 0xad, 0x1f, 0xa7, 0xd5, 0xab, 0xcd, 0xef, 0x36, 0x42, 0x99, 0xa6, 0xe2, 0x1b, + 0x76, 0x55, 0xc1, 0x00, 0x21, 0x2e, 0xb8, 0xd8, 0x18, 0x54, 0xc0, 0x40, 0x53, 0xc4, 0xc7, 0x05, + 0x57, 0x5e, 0x6b, 0x5a, 0xd3, 0xcc, 0xc9, 0x6c, 0x67, 0xd3, 0xfe, 0x04, 0x37, 0xfc, 0x47, 0x7e, + 0x07, 0x8a, 0xed, 0x24, 0x4e, 0x9a, 0x6e, 0x48, 0x08, 0xb8, 0xeb, 0x39, 0x3e, 0x1f, 0xcf, 0x79, + 0x72, 0x1e, 0xbb, 0xf0, 0x9f, 0xa0, 0x53, 0x26, 0x95, 0xb8, 0x1a, 0x46, 0x22, 0x54, 0x21, 0x6a, + 0x90, 0x88, 0xe1, 0x27, 0xd0, 0x7a, 0x3e, 0x23, 0x9c, 0xd3, 0x00, 0x21, 0x58, 0xe5, 0xe4, 0x8c, + 0x7a, 0xb5, 0x41, 0x6d, 0xaf, 0xe3, 0xeb, 0xdf, 0xc8, 0x83, 0xd6, 0x58, 0x5e, 0xbc, 0x4b, 0xdc, + 0x75, 0xed, 0x4e, 0x4d, 0x7c, 0x17, 0xba, 0x27, 0x64, 0x3c, 0x27, 0x53, 0x9a, 0x98, 0x55, 0xc9, + 0xf8, 0x12, 0x5a, 0x36, 0xa4, 0xb2, 0xf6, 0x1e, 0xb4, 0xc7, 0xa6, 0xb5, 0xf4, 0xea, 0x83, 0xc6, + 0x5e, 0x77, 0xbf, 0x37, 0x24, 0x11, 0x1b, 0x5a, 0x3c, 0x7e, 0x76, 0x8a, 0x86, 0x80, 0x26, 0xf4, + 0x0b, 0x89, 0x03, 0x65, 0xcf, 0x34, 0xa0, 0x86, 0xae, 0x55, 0x71, 0x82, 0x39, 0xac, 0x8f, 0x44, + 0x18, 0x47, 0x1f, 0xa9, 0x90, 0x2c, 0xe4, 0x6f, 0x18, 0x9f, 0xa0, 0x4d, 0x58, 0x9b, 0x26, 0x3e, + 0x0b, 0xc1, 0x18, 0xc9, 0x7c, 0x17, 0x26, 0x28, 0x9d, 0xcf, 0x9a, 0x09, 0xe2, 0x39, 0xe3, 0x13, + 0xdb, 0x45, 0xff, 0x46, 0xdb, 0xd0, 0x8c, 0x82, 0x58, 0x90, 0xc0, 0x5b, 0xd5, 0x5e, 0x6b, 0xe1, + 0x1f, 0x75, 0x68, 0x1e, 0xc6, 0x7c, 0x12, 0x14, 0x08, 0xab, 0x15, 0x08, 0x43, 0x03, 0xe8, 0x46, + 0x39, 0x61, 0xb6, 0x9d, 0xeb, 0x4a, 0x22, 0xc6, 0x0b, 0xf3, 0xb9, 0x2e, 0x5b, 0xfd, 0xb5, 0x0c, + 0xb9, 0x45, 0x90, 0x9a, 0x09, 0xb4, 0xf0, 0xf4, 0x2b, 0x1d, 0x2b, 0x6f, 0x6d, 0xd0, 0x48, 0xa0, + 0x19, 0x0b, 0xed, 0x02, 0x9c, 0x6a, 0x64, 0x27, 0x44, 0xcd, 0xbc, 0xa6, 0x4e, 0x72, 0x3c, 0xe8, + 0x29, 0xf4, 0x22, 0x11, 0x5e, 0xb0, 0x09, 0x9d, 0x1c, 0x44, 0x4c, 0x7a, 0x2d, 0xfd, 0x21, 0xb6, + 0xf4, 0x87, 0x28, 0x73, 0xe8, 0x17, 0x42, 0x93, 0x54, 0x41, 0xcf, 0x63, 0x26, 0x6c, 0x6a, 0xfb, + 0xda, 0x54, 0x37, 0xd4, 0xa5, 0xbd, 0x53, 0xa4, 0x7d, 0x07, 0x3a, 0x72, 0xce, 0x22, 0x9f, 0xf0, + 0x29, 0xf5, 0x40, 0x9f, 0xe5, 0x0e, 0xfc, 0xad, 0x06, 0x3d, 0xfb, 0xa1, 0x5f, 0x70, 0x25, 0xae, + 0xca, 0xa4, 0xd6, 0x6e, 0x24, 0xb5, 0xbe, 0x48, 0x6a, 0x46, 0x91, 0xc3, 0xba, 0xe3, 0x41, 0x7d, + 0x68, 0x0b, 0x1a, 0x05, 0x64, 0x4c, 0xa5, 0x65, 0x3d, 0xb3, 0xf1, 0x26, 0xa0, 0x63, 0x26, 0x95, + 0x5d, 0x73, 0x9f, 0x9e, 0xc7, 0x54, 0x2a, 0x7c, 0x0f, 0x36, 0x46, 0xb4, 0xe4, 0xac, 0x54, 0xc8, + 0x0c, 0xd6, 0x47, 0x54, 0x99, 0xd5, 0x49, 0xe3, 0x3c, 0x68, 0x45, 0xf3, 0xa9, 0xbb, 0x41, 0xd6, + 0xfc, 0x85, 0x51, 0x9c, 0xed, 0x6b, 0x14, 0xe5, 0xfa, 0x09, 0x6e, 0x67, 0x9d, 0x5e, 0xf1, 0x54, + 0x62, 0xbf, 0xdf, 0x12, 0x3f, 0xd6, 0x85, 0x0f, 0x82, 0xc0, 0x37, 0x9c, 0x9c, 0x51, 0xae, 0xa4, + 0x53, 0xb8, 0x5a, 0x0d, 0xf8, 0x0c, 0xb6, 0x46, 0x54, 0x39, 0x39, 0x37, 0xa6, 0xb8, 0x28, 0xeb, + 0xd7, 0xa2, 0x5c, 0x14, 0x0e, 0x56, 0xb0, 0x6d, 0x50, 0x9e, 0x98, 0x0d, 0x16, 0x19, 0xc4, 0x3f, + 0x79, 0x2f, 0x5c, 0x6a, 0x6e, 0x8e, 0x89, 0xa2, 0x52, 0xfd, 0x83, 0xc6, 0x47, 0xe6, 0x66, 0x4c, + 0x3b, 0xff, 0x85, 0xc6, 0xfb, 0xdf, 0xd7, 0xa0, 0xed, 0xdb, 0x67, 0x06, 0x3d, 0x83, 0x9e, 0x23, + 0x0e, 0x89, 0x6e, 0xe9, 0xab, 0x61, 0x51, 0x2f, 0xfd, 0x75, 0x7d, 0xe0, 0x3c, 0x27, 0x78, 0xe5, + 0x7e, 0x0d, 0x3d, 0x02, 0xc8, 0x55, 0x84, 0xb6, 0xcd, 0xbd, 0x52, 0x96, 0x55, 0xbf, 0xe7, 0xe6, + 0xe2, 0x15, 0xf4, 0x00, 0x3a, 0xd9, 0xa2, 0xa3, 0xad, 0x34, 0xa9, 0x20, 0xb1, 0x7e, 0x57, 0xbb, + 0x8d, 0x0f, 0xaf, 0xa0, 0x23, 0xf8, 0x3f, 0x0b, 0x79, 0x19, 0x8a, 0xf4, 0x3d, 0xdc, 0x2d, 0x26, + 0x97, 0x55, 0x53, 0xae, 0xf2, 0x01, 0x76, 0x46, 0x54, 0x39, 0xb7, 0x13, 0xa3, 0xf2, 0xfd, 0x8c, + 0xa4, 0x3b, 0x9e, 0x97, 0xab, 0xd6, 0x4a, 0x7f, 0xc3, 0x7d, 0xfc, 0xf4, 0xed, 0xa6, 0x59, 0x38, + 0xd4, 0x42, 0x31, 0x5d, 0x9c, 0x72, 0x12, 0xf5, 0xd3, 0x7a, 0x8b, 0x22, 0x2a, 0x43, 0xf3, 0x97, + 0x40, 0xb3, 0x9b, 0x81, 0xee, 0x38, 0xd0, 0xca, 0x7b, 0xba, 0x0c, 0xd7, 0x67, 0xc0, 0xd9, 0x6e, + 0x2f, 0xaf, 0x9c, 0x0d, 0x5d, 0x2d, 0x82, 0x65, 0xc5, 0xdf, 0x6a, 0xc0, 0x76, 0x7f, 0xf3, 0xd9, + 0x6d, 0xba, 0xcc, 0xcb, 0x56, 0xaf, 0x78, 0x69, 0xfe, 0xd3, 0xa6, 0xfe, 0xc3, 0xf3, 0xf0, 0x67, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xe1, 0x3c, 0x83, 0x36, 0x02, 0x09, 0x00, 0x00, } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.proto b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.proto index 08ca758419..84cf90e441 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.proto +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.proto @@ -46,6 +46,8 @@ message Bundle{ string bundlePath = 6; repeated GroupVersionKind providedApis = 7; repeated GroupVersionKind requiredApis = 8; + string version = 9; + string skipRange = 10; } message ChannelEntry{ diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/containertools/bundlereader.go b/vendor/github.com/operator-framework/operator-registry/pkg/containertools/bundlereader.go new file mode 100644 index 0000000000..92f418b3fc --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/containertools/bundlereader.go @@ -0,0 +1,186 @@ +package containertools + +import ( + "archive/tar" + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +const ( + imageManifestName = "manifest.json" +) + +// imageManifest is the object format of container image manifest files +// use this type to parse manifest.json files inside container image blobs +type imageManifest struct { + Layers []string `json:”Layers”` +} + +type BundleReader struct { +} + +func NewBundleReader() *BundleReader { + return &BundleReader{} +} + +func (b *BundleReader) GetBundle(image, outputDir string) error { + r := NewCommandRunner(Podman) + + // Create the output directory if it doesn't exist + if _, err := os.Stat(outputDir); os.IsNotExist(err) { + os.Mkdir(outputDir, 0777) + } + + err := r.Pull(image) + if err != nil { + return err + } + + workingDir, err := ioutil.TempDir("./", "bundle_staging_") + if err != nil { + return err + } + defer os.RemoveAll(workingDir) + + rootTarfile := filepath.Join(workingDir, "bundle.tar") + + err = r.Save(image, rootTarfile) + if err != nil { + return err + } + + f, err := os.Open(rootTarfile) + if err != nil { + return err + } + defer f.Close() + + // Read the manifest.json file to find the right embedded tarball + layerTarballs, err := getManifestLayers(tar.NewReader(f)) + if err != nil { + return err + } + + // Untar the image layer tarballs and push the bundle manifests to the output directory + for _, tarball := range layerTarballs { + f, err = os.Open(rootTarfile) + if err != nil { + return err + } + defer f.Close() + + err = extractBundleManifests(tarball, outputDir, tar.NewReader(f)) + if err != nil { + return err + } + } + + return nil +} + +func getManifestLayers(tarReader *tar.Reader) ([]string, error) { + for { + header, err := tarReader.Next() + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("invalid bundle image: unable to find manifest.json") + } + return nil, err + } + + if header.Name == imageManifestName { + buf := new(bytes.Buffer) + buf.ReadFrom(tarReader) + b := buf.Bytes() + + manifests := make([]imageManifest, 0) + err := json.Unmarshal(b, &manifests) + if err != nil { + return nil, err + } + + if len(manifests) == 0 { + return nil, fmt.Errorf("invalid bundle image: manifest.json missing manifest data") + } + + topManifest := manifests[0] + + if len(topManifest.Layers) == 0 { + return nil, fmt.Errorf("invalid bundle image: manifest has no layers") + } + + return topManifest.Layers, nil + } + } +} + +func extractBundleManifests(layerTarball, outputDir string, tarReader *tar.Reader) error { + for { + header, err := tarReader.Next() + if err != nil { + if err == io.EOF { + return fmt.Errorf("Manifest error: Layer tarball does not exist in bundle") + } + return err + } + + if header.Typeflag == tar.TypeReg { + if header.Name == layerTarball { + // Found the embedded top layer tarball + layerReader := tar.NewReader(tarReader) + + err = extractTarballToDir(outputDir, layerReader) + if err != nil { + return err + } + } + + continue + } else { + return nil + } + } +} + +func extractTarballToDir(outputDir string, tarReader *tar.Reader) error { + for { + header, err := tarReader.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + switch header.Typeflag { + case tar.TypeDir: + // Create the directory if it doesn't exist + directoryToWrite := filepath.Join(outputDir, header.Name) + if _, err := os.Stat(directoryToWrite); os.IsNotExist(err) { + os.Mkdir(directoryToWrite, 0777) + } + case tar.TypeReg: + buf := new(bytes.Buffer) + buf.ReadFrom(tarReader) + b := buf.Bytes() + + manifestToWrite := filepath.Join(outputDir, header.Name) + + m, err := os.Create(manifestToWrite) + if err != nil { + return err + } + defer m.Close() + + _, err = m.Write(b) + if err != nil { + return err + } + } + } +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/containertools/command.go b/vendor/github.com/operator-framework/operator-registry/pkg/containertools/command.go new file mode 100644 index 0000000000..2a9e6de3ac --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/containertools/command.go @@ -0,0 +1,66 @@ +package containertools + +import ( + "fmt" + "os/exec" +) + +const ( + // Podman cli tool + Podman = "podman" + // Docker cli tool + Docker = "docker" +) + +// CommandRunner is configured to select a container cli tool and execute commands with that +// tooling. +type CommandRunner struct { + containerTool string +} + +// NewCommandRunner takes the containerTool as an input string and returns a CommandRunner to +// run commands with that cli tool +func NewCommandRunner(containerTool string) *CommandRunner { + r := &CommandRunner{} + + switch containerTool { + case Podman: + r.containerTool = Podman + case Docker: + r.containerTool = Docker + default: + r.containerTool = Podman + } + + return r +} + +// Pull takes a container image path hosted on a container registry and runs the pull command to +// download it onto the local environment +func (r *CommandRunner) Pull(image string) error { + args := []string{"pull", image} + + command := exec.Command(r.containerTool, args...) + + out, err := command.Output() + if err != nil { + return fmt.Errorf("error with %s %s: %s", command.Args, string(out), err) + } + + return nil +} + +// Save takes a local container image and runs the save commmand to convert the image into a specified +// tarball and push it to the local directory +func (r *CommandRunner) Save(image, tarFile string) error { + args := []string{"save", image, "-o", tarFile} + + command := exec.Command(r.containerTool, args...) + + out, err := command.Output() + if err != nil { + return fmt.Errorf("error with %s %s: %s", command.Args, string(out), err) + } + + return nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go index b9ed042a6f..ec0198ca61 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go @@ -28,13 +28,14 @@ func init() { } type Bundle struct { - Name string - Objects []*unstructured.Unstructured - Package string - Channel string - csv *ClusterServiceVersion - crds []*v1beta1.CustomResourceDefinition - cacheStale bool + Name string + Objects []*unstructured.Unstructured + Package string + Channel string + BundleImage string + csv *ClusterServiceVersion + crds []*v1beta1.CustomResourceDefinition + cacheStale bool } func NewBundle(name, pkgName, channelName string, objs ...*unstructured.Unstructured) *Bundle { @@ -170,12 +171,12 @@ func (b *Bundle) AllProvidedAPIsInBundle() error { return nil } -func (b *Bundle) Serialize() (csvName string, csvBytes []byte, bundleBytes []byte, err error) { +func (b *Bundle) Serialize() (csvName, bundleImage string, csvBytes []byte, bundleBytes []byte, err error) { csvCount := 0 for _, obj := range b.Objects { objBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj) if err != nil { - return "", nil, nil, err + return "", "", nil, nil, err } bundleBytes = append(bundleBytes, objBytes...) @@ -183,16 +184,16 @@ func (b *Bundle) Serialize() (csvName string, csvBytes []byte, bundleBytes []byt csvName = obj.GetName() csvBytes, err = runtime.Encode(unstructured.UnstructuredJSONScheme, obj) if err != nil { - return "", nil, nil, err + return "", "", nil, nil, err } csvCount += 1 if csvCount > 1 { - return "", nil, nil, fmt.Errorf("two csvs found in one bundle") + return "", "", nil, nil, fmt.Errorf("two csvs found in one bundle") } } } - return csvName, csvBytes, bundleBytes, nil + return csvName, b.BundleImage, csvBytes, bundleBytes, nil } func (b *Bundle) Images() (map[string]struct{}, error) { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go index df2378088d..69a0102adf 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go @@ -8,7 +8,10 @@ import ( type Load interface { AddOperatorBundle(bundle *Bundle) error + AddBundlePackageChannels(manifest PackageManifest, bundle Bundle) error AddPackageChannels(manifest PackageManifest) error + RmPackageName(packageName string) error + ClearNonDefaultBundles(packageName string) error } type Query interface { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go index 4295d64e8a..7ca1c3e310 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go @@ -2,6 +2,7 @@ package registry import ( "fmt" + "strings" ) // APIKey stores GroupVersionKind for use as map keys @@ -74,3 +75,51 @@ type ChannelEntry struct { BundleName string Replaces string } + +// AnnotationsFile holds annotation information about a bundle +type AnnotationsFile struct { + // annotations is a list of annotations for a given bundle + Annotations Annotations `json:"annotations" yaml:"annotations"` +} + +// Annotations is a list of annotations for a given bundle +type Annotations struct { + // PackageName is the name of the overall package, ala `etcd`. + PackageName string `json:"operators.operatorframework.io.bundle.package.v1" yaml:"operators.operatorframework.io.bundle.package.v1"` + + // Channels are a comma separated list of the declared channels for the bundle, ala `stable` or `alpha`. + Channels string `json:"operators.operatorframework.io.bundle.channels.v1" yaml:"operators.operatorframework.io.bundle.channels.v1"` + + // DefaultChannelName is, if specified, the name of the default channel for the package. The + // default channel will be installed if no other channel is explicitly given. If the package + // has a single channel, then that channel is implicitly the default. + DefaultChannelName string `json:"operators.operatorframework.io.bundle.channel.default.v1" yaml:"operators.operatorframework.io.bundle.channel.default.v1"` +} + +// GetName returns the package name of the bundle +func (a *AnnotationsFile) GetName() string { + if a.Annotations.PackageName != "" { + return a.Annotations.PackageName + } + return "" +} + +// GetChannels returns the channels that this bundle should be added to +func (a *AnnotationsFile) GetChannels() []string { + if a.Annotations.Channels != "" { + return strings.Split(a.Annotations.Channels, ",") + } + return []string{} +} + +// GetDefaultChannelName returns the name of the default channel +func (a *AnnotationsFile) GetDefaultChannelName() string { + if a.Annotations.DefaultChannelName != "" { + return a.Annotations.DefaultChannelName + } + channels := a.GetChannels() + if len(channels) == 1 { + return channels[0] + } + return "" +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go index fedd8dfb96..fe708bf87d 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go @@ -107,7 +107,7 @@ func (d *DirectoryLoader) LoadBundleWalkFunc(path string, f os.FileInfo, err err log.Info("found csv, loading bundle") var errs []error - bundle, err := d.loadBundle(csv.GetName(), filepath.Dir(path)) + bundle, err := loadBundle(csv.GetName(), filepath.Dir(path)) if err != nil { errs = append(errs, fmt.Errorf("error loading objs in directory: %s", err)) } @@ -128,10 +128,56 @@ func (d *DirectoryLoader) LoadBundleWalkFunc(path string, f os.FileInfo, err err return utilerrors.NewAggregate(errs) } -// LoadBundle takes the directory that a CSV is in and assumes the rest of the objects in that directory +// LoadPackagesWalkFunc attempts to unmarshal the file at the given path into a PackageManifest resource. +// If unmarshaling is successful, the PackageManifest is added to the loader's store. +func (d *DirectoryLoader) LoadPackagesWalkFunc(path string, f os.FileInfo, err error) error { + if f == nil { + return fmt.Errorf("invalid file: %v", f) + } + + log := logrus.WithFields(logrus.Fields{"dir": d.directory, "file": f.Name(), "load": "package"}) + if f.IsDir() { + if strings.HasPrefix(f.Name(), ".") { + log.Info("skipping hidden directory") + return filepath.SkipDir + } + log.Info("directory") + return nil + } + + if strings.HasPrefix(f.Name(), ".") { + log.Info("skipping hidden file") + return nil + } + + fileReader, err := os.Open(path) + if err != nil { + return fmt.Errorf("unable to load package from file %s: %s", path, err) + } + + decoder := yaml.NewYAMLOrJSONDecoder(fileReader, 30) + manifest := registry.PackageManifest{} + if err = decoder.Decode(&manifest); err != nil { + if err != nil { + return fmt.Errorf("could not decode contents of file %s into package: %s", path, err) + } + + } + if manifest.PackageName == "" { + return nil + } + + if err := d.store.AddPackageChannels(manifest); err != nil { + return fmt.Errorf("error loading package into db: %s", err) + } + + return nil +} + +// loadBundle takes the directory that a CSV is in and assumes the rest of the objects in that directory // are part of the bundle. -func (d *DirectoryLoader) loadBundle(csvName string, dir string) (*registry.Bundle, error) { - log := logrus.WithFields(logrus.Fields{"dir": d.directory, "load": "bundle"}) +func loadBundle(csvName string, dir string) (*registry.Bundle, error) { + log := logrus.WithFields(logrus.Fields{"dir": dir, "load": "bundle"}) files, err := ioutil.ReadDir(dir) if err != nil { return nil, err @@ -178,46 +224,3 @@ func (d *DirectoryLoader) loadBundle(csvName string, dir string) (*registry.Bund return bundle, utilerrors.NewAggregate(errs) } - -// LoadPackagesWalkFunc attempts to unmarshal the file at the given path into a PackageManifest resource. -// If unmarshaling is successful, the PackageManifest is added to the loader's store. -func (d *DirectoryLoader) LoadPackagesWalkFunc(path string, f os.FileInfo, err error) error { - if f == nil { - return fmt.Errorf("invalid file: %v", f) - } - - log := logrus.WithFields(logrus.Fields{"dir": d.directory, "file": f.Name(), "load": "package"}) - if f.IsDir() { - if strings.HasPrefix(f.Name(), ".") { - log.Info("skipping hidden directory") - return filepath.SkipDir - } - log.Info("directory") - return nil - } - - if strings.HasPrefix(f.Name(), ".") { - log.Info("skipping hidden file") - return nil - } - - fileReader, err := os.Open(path) - if err != nil { - return fmt.Errorf("unable to load package from file %s: %s", path, err) - } - - decoder := yaml.NewYAMLOrJSONDecoder(fileReader, 30) - manifest := registry.PackageManifest{} - if err = decoder.Decode(&manifest); err != nil { - return fmt.Errorf("could not decode contents of file %s into package: %s", path, err) - } - if manifest.PackageName == "" { - return nil - } - - if err := d.store.AddPackageChannels(manifest); err != nil { - return fmt.Errorf("error loading package into db: %s", err) - } - - return nil -} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/image.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/image.go new file mode 100644 index 0000000000..6b8e257610 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/image.go @@ -0,0 +1,239 @@ +package sqlite + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/yaml" + + "github.com/operator-framework/operator-registry/pkg/containertools" + "github.com/operator-framework/operator-registry/pkg/registry" +) + +// ImageLoader loads a bundle image of resources into the database +type ImageLoader struct { + store registry.Load + image string + directory string +} + +func NewSQLLoaderForImage(store registry.Load, image string) *ImageLoader { + return &ImageLoader{ + store: store, + image: image, + directory: "", + } +} + +func (i *ImageLoader) Populate() error { + + log := logrus.WithField("img", i.image) + + workingDir, err := ioutil.TempDir("./", "bundle_tmp") + if err != nil { + return err + } + defer os.RemoveAll(workingDir) + + // Pull the image and get the manifests + reader := containertools.NewBundleReader() + + err = reader.GetBundle(i.image, workingDir) + if err != nil { + return err + } + + i.directory = workingDir + + log.Infof("loading Bundle %s", i.image) + errs := make([]error, 0) + if err := i.LoadBundleFunc(); err != nil { + errs = append(errs, err) + } + + return utilerrors.NewAggregate(errs) +} + +// LoadBundleFunc walks the bundle directory. Looks for the metadata and manifests +// sub-directories to find the annotations.yaml file that will inform how the +// manifests of the bundle should be loaded into the database. +func (i *ImageLoader) LoadBundleFunc() error { + path := i.directory + manifests := filepath.Join(path, "manifests") + metadata := filepath.Join(path, "metadata") + + // Get annotations file + log := logrus.WithFields(logrus.Fields{"dir": i.directory, "file": metadata, "load": "annotations"}) + files, err := ioutil.ReadDir(metadata) + if err != nil { + return fmt.Errorf("unable to read directory %s: %s", metadata, err) + } + + annotationsFile := ®istry.AnnotationsFile{} + for _, f := range files { + fileReader, err := os.Open(filepath.Join(metadata, f.Name())) + if err != nil { + return fmt.Errorf("unable to read file %s: %s", f.Name(), err) + } + decoder := yaml.NewYAMLOrJSONDecoder(fileReader, 30) + err = decoder.Decode(&annotationsFile) + if err != nil || *annotationsFile == (registry.AnnotationsFile{}) { + continue + } else { + log.Info("found annotations file searching for csv") + } + } + + if *annotationsFile == (registry.AnnotationsFile{}) { + return fmt.Errorf("Could not find annotations.yaml file") + } + + err = i.loadManifests(manifests, annotationsFile) + if err != nil { + return err + } + + return nil +} + +func (i *ImageLoader) loadManifests(manifests string, annotationsFile *registry.AnnotationsFile) error { + log := logrus.WithFields(logrus.Fields{"dir": i.directory, "file": manifests, "load": "bundle"}) + + csv, err := i.findCSV(manifests) + if err != nil { + return err + } + + if csv.Object == nil { + return fmt.Errorf("csv is empty: %s", err) + } + + log.Info("found csv, loading bundle") + + // TODO: Check channels against what's in the database vs in the bundle csv + + bundle, err := loadBundle(csv.GetName(), manifests) + if err != nil { + return fmt.Errorf("error loading objs in directory: %s", err) + } + + if bundle == nil || bundle.Size() == 0 { + return fmt.Errorf("no bundle objects found") + } + + // set the bundleimage on the bundle + bundle.BundleImage = i.image + + if err := bundle.AllProvidedAPIsInBundle(); err != nil { + return fmt.Errorf("error checking provided apis in bundle %s: %s", bundle.Name, err) + } + + bcsv, err := bundle.ClusterServiceVersion() + if err != nil { + return fmt.Errorf("error getting csv from bundle %s: %s", bundle.Name, err) + } + + packageManifest, err := translateAnnotationsIntoPackage(annotationsFile, bcsv) + if err != nil { + return fmt.Errorf("Could not translate annotations file into packageManifest %s", err) + } + + if err := i.loadOperatorBundle(packageManifest, *bundle); err != nil { + return fmt.Errorf("Error adding package %s", err) + } + + // Finally let's delete all the old bundles + if err = i.store.ClearNonDefaultBundles(packageManifest.PackageName); err != nil { + return fmt.Errorf("Error deleting previous bundles: %s", err) + } + + return nil +} + +// findCSV looks through the bundle directory to find a csv +func (i *ImageLoader) findCSV(manifests string) (*unstructured.Unstructured, error) { + log := logrus.WithFields(logrus.Fields{"dir": i.directory, "find": "csv"}) + + files, err := ioutil.ReadDir(manifests) + if err != nil { + return nil, fmt.Errorf("unable to read directory %s: %s", manifests, err) + } + + var errs []error + for _, f := range files { + log = log.WithField("file", f.Name()) + if f.IsDir() { + log.Info("skipping directory") + continue + } + + if strings.HasPrefix(f.Name(), ".") { + log.Info("skipping hidden file") + continue + } + + path := filepath.Join(manifests, f.Name()) + fileReader, err := os.Open(path) + if err != nil { + errs = append(errs, fmt.Errorf("unable to read file %s: %s", path, err)) + continue + } + + dec := yaml.NewYAMLOrJSONDecoder(fileReader, 30) + unst := &unstructured.Unstructured{} + if err := dec.Decode(unst); err != nil { + continue + } + + if unst.GetKind() != ClusterServiceVersionKind { + continue + } + + return unst, nil + + } + + errs = append(errs, fmt.Errorf("no csv found in bundle")) + return nil, utilerrors.NewAggregate(errs) +} + +// loadOperatorBundle adds the package information to the loader's store +func (i *ImageLoader) loadOperatorBundle(manifest registry.PackageManifest, bundle registry.Bundle) error { + if manifest.PackageName == "" { + return nil + } + + if err := i.store.AddBundlePackageChannels(manifest, bundle); err != nil { + return fmt.Errorf("error loading bundle into db: %s", err) + } + + return nil +} + +// translateAnnotationsIntoPackage attempts to translate the channels.yaml file at the given path into a package.yaml +func translateAnnotationsIntoPackage(annotations *registry.AnnotationsFile, csv *registry.ClusterServiceVersion) (registry.PackageManifest, error) { + manifest := registry.PackageManifest{} + + channels := []registry.PackageChannel{} + for _, ch := range annotations.GetChannels() { + channels = append(channels, + registry.PackageChannel{ + Name: ch, + CurrentCSVName: csv.GetName(), + }) + } + + manifest = registry.PackageManifest{ + PackageName: annotations.GetName(), + DefaultChannelName: annotations.GetDefaultChannelName(), + Channels: channels, + } + + return manifest, nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go index c379569014..5436faa9d4 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go @@ -68,7 +68,7 @@ func (s *SQLLoader) AddOperatorBundle(bundle *registry.Bundle) error { } defer addImage.Close() - csvName, csvBytes, bundleBytes, err := bundle.Serialize() + csvName, bundleImage, csvBytes, bundleBytes, err := bundle.Serialize() if err != nil { return err } @@ -77,7 +77,7 @@ func (s *SQLLoader) AddOperatorBundle(bundle *registry.Bundle) error { return fmt.Errorf("csv name not found") } - if _, err := stmt.Exec(csvName, csvBytes, bundleBytes, nil); err != nil { + if _, err := stmt.Exec(csvName, csvBytes, bundleBytes, bundleImage); err != nil { return err } @@ -134,13 +134,22 @@ func (s *SQLLoader) AddPackageChannels(manifest registry.PackageManifest) error } defer addReplaces.Close() + var errs []error + if _, err := addPackage.Exec(manifest.PackageName); err != nil { - // This should be terminal - return err + err = s.updatePackageChannels(tx, manifest) + if err != nil { + errs = append(errs, err) + } + + if err := tx.Commit(); err != nil { + errs = append(errs, err) + } + + return utilerrors.NewAggregate(errs) } hasDefault := false - var errs []error for _, c := range manifest.Channels { if _, err := addChannel.Exec(c.Name, manifest.PackageName, c.CurrentCSVName); err != nil { errs = append(errs, err) @@ -274,6 +283,106 @@ func (s *SQLLoader) AddPackageChannels(manifest registry.PackageManifest) error return utilerrors.NewAggregate(errs) } +func (s *SQLLoader) ClearNonDefaultBundles(packageName string) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { + tx.Rollback() + }() + + // First find the default channel for the package + getDefChan, err := tx.Prepare(fmt.Sprintf("select default_channel from package where name='%s'", packageName)) + if err != nil { + return err + } + defer getDefChan.Close() + + defaultChannelRows, err := getDefChan.Query() + if err != nil { + return err + } + defer defaultChannelRows.Close() + + if !defaultChannelRows.Next() { + return fmt.Errorf("no default channel found for package %s", packageName) + } + var defaultChannel sql.NullString + if err := defaultChannelRows.Scan(&defaultChannel); err != nil { + return err + } + + // Then get the head of the default channel + getChanHead, err := tx.Prepare(fmt.Sprintf("select head_operatorbundle_name from channel where name='%s'", defaultChannel.String)) + if err != nil { + return err + } + defer getChanHead.Close() + + chanHeadRows, err := getChanHead.Query() + if err != nil { + return err + } + defer chanHeadRows.Close() + + if !chanHeadRows.Next() { + return fmt.Errorf("no channel head found for default channel %s", defaultChannel.String) + } + var defChanHead sql.NullString + if err := chanHeadRows.Scan(&defChanHead); err != nil { + return err + } + + // Now get all the bundles that are not the head of the default channel + getChannelBundles, err := tx.Prepare(fmt.Sprintf("SELECT operatorbundle_name FROM channel_entry WHERE package_name='%s' AND operatorbundle_name!='%s'", packageName, defChanHead.String)) + if err != nil { + return err + } + defer getChanHead.Close() + + chanBundleRows, err := getChannelBundles.Query() + if err != nil { + return err + } + defer chanBundleRows.Close() + + bundles := make(map[string]struct{}, 0) + for chanBundleRows.Next() { + var bundleToUpdate sql.NullString + if err := chanBundleRows.Scan(&bundleToUpdate); err != nil { + return err + } + bundles[bundleToUpdate.String] = struct{}{} + } + + if len(bundles) > 0 { + bundlePredicates := []string{} + for bundle := range bundles { + bundlePredicates = append(bundlePredicates, fmt.Sprintf("name = '%s'", bundle)) + } + + var transactionPredicate string + if len(bundlePredicates) == 1 { + transactionPredicate = fmt.Sprintf("WHERE %s AND bundlepath != \"\"", bundlePredicates[0]) + } else { + transactionPredicate = fmt.Sprintf("WHERE (%s) AND bundlepath != \"\"", strings.Join(bundlePredicates, " OR ")) + } + + removeOldBundles, err := tx.Prepare(fmt.Sprintf("UPDATE operatorbundle SET bundle = null, csv = null %s", transactionPredicate)) + if err != nil { + return err + } + + _, err = removeOldBundles.Exec() + if err != nil { + return fmt.Errorf("Unable to remove previous bundles: %s", err) + } + } + + return tx.Commit() +} + func SplitCRDName(crdName string) (plural, group string, err error) { pluralGroup := strings.SplitN(crdName, ".", 2) if len(pluralGroup) != 2 { @@ -308,10 +417,14 @@ func (s *SQLLoader) getCSV(tx *sql.Tx, csvName string) (*registry.ClusterService return nil, err } + if !csvStringSQL.Valid { + return nil, fmt.Errorf("csv %s not stored for non-latest versions", csvName) + } + dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(csvStringSQL.String), 10) unst := &unstructured.Unstructured{} if err := dec.Decode(unst); err != nil { - return nil, err + return nil, fmt.Errorf("can't decode %s: %s", csvStringSQL.String, err) } csv := ®istry.ClusterServiceVersion{} @@ -342,6 +455,9 @@ func (s *SQLLoader) addAPIs(tx *sql.Tx, csv *registry.ClusterServiceVersion, cha defer addApiRequirer.Close() ownedCRDs, requiredCRDs, err := csv.GetCustomResourceDefintions() + if err != nil { + return err + } for _, crd := range ownedCRDs { plural, group, err := SplitCRDName(crd.Name) if err != nil { @@ -368,6 +484,9 @@ func (s *SQLLoader) addAPIs(tx *sql.Tx, csv *registry.ClusterServiceVersion, cha } ownedAPIs, requiredAPIs, err := csv.GetApiServiceDefinitions() + if err != nil { + return err + } for _, api := range ownedAPIs { if _, err := addAPI.Exec(api.Group, api.Version, api.Kind, api.Name); err != nil { return err @@ -380,9 +499,477 @@ func (s *SQLLoader) addAPIs(tx *sql.Tx, csv *registry.ClusterServiceVersion, cha if _, err := addAPI.Exec(api.Group, api.Version, api.Kind, api.Name); err != nil { return err } - if _, err := addAPIProvider.Exec(api.Group, api.Version, api.Kind, channelEntryId); err != nil { + if _, err := addApiRequirer.Exec(api.Group, api.Version, api.Kind, channelEntryId); err != nil { + return err + } + } + return nil +} +func (s *SQLLoader) getCSVNames(tx *sql.Tx, packageName string) ([]string, error) { + getID, err := tx.Prepare(` + SELECT DISTINCT channel_entry.operatorbundle_name + FROM channel_entry + WHERE channel_entry.package_name=?`) + + if err != nil { + return nil, err + } + defer getID.Close() + + rows, err := getID.Query(packageName) + if err != nil { + return nil, err + } + + var csvName string + csvNames := []string{} + for rows.Next() { + err := rows.Scan(&csvName) + if err != nil { + return nil, err + } + csvNames = append(csvNames, csvName) + } + + if err := rows.Close(); err != nil { + return nil, err + } + + return csvNames, nil +} + +func (s *SQLLoader) rmAPIs(tx *sql.Tx, csv *registry.ClusterServiceVersion) error { + rmAPI, err := tx.Prepare("delete from api where group_name=? AND version=? AND kind=?") + if err != nil { + return err + } + defer rmAPI.Close() + + ownedCRDs, _, err := csv.GetCustomResourceDefintions() + for _, crd := range ownedCRDs { + _, group, err := SplitCRDName(crd.Name) + if err != nil { + return err + } + if _, err := rmAPI.Exec(group, crd.Version, crd.Kind); err != nil { + return err + } + } + + return nil +} + +func (s *SQLLoader) RmPackageName(packageName string) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { + tx.Rollback() + }() + + csvNames, err := s.getCSVNames(tx, packageName) + if err != nil { + return err + } + for _, csvName := range csvNames { + csv, err := s.getCSV(tx, csvName) + if csv != nil { + err = s.rmBundle(tx, csvName) + if err != nil { + return err + } + err = s.rmAPIs(tx, csv) + if err != nil { + return err + } + } else { + err = s.rmBundle(tx, csvName) + if err != nil { + return err + } + } + } + + return tx.Commit() +} + +func (s *SQLLoader) rmBundle(tx *sql.Tx, csvName string) error { + stmt, err := tx.Prepare("DELETE FROM operatorbundle WHERE operatorbundle.name=?") + if err != nil { + return err + } + defer stmt.Close() + + if _, err := stmt.Exec(csvName); err != nil { + return err + } + + return nil +} + +func (s *SQLLoader) AddBundlePackageChannels(manifest registry.PackageManifest, bundle registry.Bundle) error { + var errs []error + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { + tx.Rollback() + }() + + stmt, err := tx.Prepare("insert into operatorbundle(name, csv, bundle, bundlepath) values(?, ?, ?, ?)") + if err != nil { + return err + } + defer stmt.Close() + + addImage, err := tx.Prepare("insert into related_image(image, operatorbundle_name) values(?,?)") + if err != nil { + return err + } + defer addImage.Close() + + csvName, bundleImage, csvBytes, bundleBytes, err := bundle.Serialize() + if err != nil { + return err + } + + if csvName == "" { + return fmt.Errorf("csv name not found") + } + + if _, err := stmt.Exec(csvName, csvBytes, bundleBytes, bundleImage); err != nil { + return err + } + + imgs, err := bundle.Images() + if err != nil { + return err + } + // TODO: bulk insert + for img := range imgs { + if _, err := addImage.Exec(img, csvName); err != nil { return err } } + + if err := tx.Commit(); err != nil { + return err + } + + if err := s.AddPackageChannels(manifest); err != nil { + errs = append(errs, err) + tx, err := s.db.Begin() + if err != nil { + errs = append(errs, err) + return utilerrors.NewAggregate(errs) + } + defer func() { + tx.Rollback() + }() + + if err := s.rmBundle(tx, csvName); err != nil { + errs = append(errs, err) + return utilerrors.NewAggregate(errs) + } + + if err := tx.Commit(); err != nil { + errs = append(errs, err) + } + + return utilerrors.NewAggregate(errs) + } + + return nil +} + +func (s *SQLLoader) updatePackageChannels(tx *sql.Tx, manifest registry.PackageManifest) error { + updateDefaultChannel, err := tx.Prepare("update package set default_channel = ? where name = ?") + if err != nil { + return err + } + defer updateDefaultChannel.Close() + + getDefaultChannel, err := tx.Prepare(`SELECT default_channel FROM package WHERE name = ? LIMIT 1`) + if err != nil { + return err + } + defer getDefaultChannel.Close() + + updateChannel, err := tx.Prepare("update channel set head_operatorbundle_name = ? where name = ? and package_name = ?") + if err != nil { + return err + } + defer updateChannel.Close() + + addChannelEntry, err := tx.Prepare("insert into channel_entry(channel_name, package_name, operatorbundle_name, depth) values(?, ?, ?, ?)") + if err != nil { + return err + } + defer addChannelEntry.Close() + + updateChannelEntry, err := tx.Prepare("update channel_entry set depth = ? where channel_name = ? and package_name = ? and operatorbundle_name = ?") + if err != nil { + return err + } + defer updateChannelEntry.Close() + + addReplaces, err := tx.Prepare("update channel_entry set replaces = ? where entry_id = ?") + if err != nil { + return err + } + defer addReplaces.Close() + + getDepth, err := tx.Prepare(` + SELECT channel_entry.depth, channel_entry.entry_id + FROM channel_entry + WHERE channel_name = ? and package_name = ? and operatorbundle_name =? + LIMIT 1`) + if err != nil { + return err + } + defer getDepth.Close() + + getChannelEntryID, err := tx.Prepare(` + SELECT channel_entry.entry_id + FROM channel_entry + WHERE channel_name = ? and package_name = ? and operatorbundle_name =? + LIMIT 1`) + if err != nil { + return err + } + defer getChannelEntryID.Close() + + updateDepth, err := tx.Prepare("update channel_entry set depth = depth + 1 where channel_name = ? and package_name = ? and operatorbundle_name = ?") + if err != nil { + return err + } + defer updateDepth.Close() + + removeSkipped, err := tx.Prepare("delete from channel_entry where channel_name = ? and package_name = ? and operatorbundle_name = ?") + if err != nil { + return err + } + defer removeSkipped.Close() + + getBundleIDNameFromDepthToHead, err := tx.Prepare(` + SELECT entry_id, operatorbundle_name + FROM channel_entry + WHERE depth < ? and channel_name = ? and package_name = ?`) + if err != nil { + return err + } + defer getBundleIDNameFromDepthToHead.Close() + + var errs []error + + // update head bundle name in channel table + for _, c := range manifest.Channels { + if _, err := updateChannel.Exec(c.CurrentCSVName, c.Name, manifest.PackageName); err != nil { + errs = append(errs, err) + continue + } + } + + // insert/replace default channel + defaultChannelName := manifest.GetDefaultChannel() + if defaultChannelName != "" { + if _, err := updateDefaultChannel.Exec(defaultChannelName, manifest.PackageName); err != nil { + errs = append(errs, err) + } + } // else assume default channel is already in db and need not be changed + + // For each channel, check where in update graph + // the bundle is attempted to be inserted. + // If not at the head of the channel then error + for _, c := range manifest.Channels { + // don't need to check if version has been inserted for a given channel + // because this is caught by primary key of operatorbundle table + + channelEntryCSV, err := s.getCSV(tx, c.CurrentCSVName) + + // check replaces + replaces, err := channelEntryCSV.GetReplaces() + if err != nil { + errs = append(errs, err) + break + } + + // where does the replaces fall in the update graph + rows, err := getDepth.Query(c.Name, manifest.PackageName, replaces) + if err != nil { + errs = append(errs, err) + continue + } + + var depth int64 + var currentID int64 + var replacedIDs []int64 + skips, err := channelEntryCSV.GetSkips() + if err != nil { + errs = append(errs, err) + continue + } + + if rows.Next() { + err := rows.Scan(&depth, ¤tID) + if err != nil { + errs = append(errs, err) + continue + } + // check if replaces not at the head of the channel + if depth != 0 { + // if not at the head of the channel, need to specify appropriate skips + if len(skips) != int(depth) { + errs = append(errs, fmt.Errorf("%s attempts to replace %s that is already replaced by another version", c.CurrentCSVName, replaces)) + return utilerrors.NewAggregate(errs) + } + skipmap := make(map[string]struct{}, 0) + for _, sk := range skips { + skipmap[sk] = struct{}{} + } + // get csv from depth to head for channel + skipped, err := getBundleIDNameFromDepthToHead.Query(depth, c.Name, manifest.PackageName) + if err != nil { + errs = append(errs, err) + continue + } + defer skipped.Close() + + // see if csvs match skips + var skip string + var replacedID int64 + for skipped.Next() { + err := skipped.Scan(&replacedID, &skip) + if err != nil { + errs = append(errs, err) + return utilerrors.NewAggregate(errs) + } + replacedIDs = append(replacedIDs, replacedID) + if _, ok := skipmap[skip]; !ok { + errs = append(errs, fmt.Errorf("%s attempts to replace %s that is already replaced by %s without specifying a skip", c.CurrentCSVName, replaces, skip)) + } + } + // aggregate all the errors instead of returning on first error + if len(errs) > 0 { + return utilerrors.NewAggregate(errs) + } + } + } else { + // specifies a replacement that is not in db + errs = append(errs, fmt.Errorf("%s specifies a replacement %s that cannot be found", c.CurrentCSVName, replaces)) + return utilerrors.NewAggregate(errs) + } + + if err := rows.Close(); err != nil { + errs = append(errs, err) + continue + } + + // insert version into head of channel + res, err := addChannelEntry.Exec(c.Name, manifest.PackageName, c.CurrentCSVName, 0) + if err != nil { + errs = append(errs, err) + continue + } + + currentID, err = res.LastInsertId() + if err != nil { + errs = append(errs, err) + continue + } + + // update replacement to point to new head of channel + var replacedID int64 + rows, err = getChannelEntryID.Query(c.Name, manifest.PackageName, replaces) + if err != nil { + errs = append(errs, err) + continue + } + if rows.Next() { + err := rows.Scan(&replacedID) + if err != nil { + errs = append(errs, err) + } + } // else is not possible by previous SELECT statement on replaces + + if err := rows.Close(); err != nil { + errs = append(errs, err) + continue + } + + if _, err = addReplaces.Exec(replacedID, currentID); err != nil { + errs = append(errs, err) + continue + } + + // remove skips from graph + for _, skip := range skips { + if _, err := removeSkipped.Exec(c.Name, manifest.PackageName, skip); err != nil { + errs = append(errs, err) + continue + } + } + + // add APIs + if err := s.addAPIs(tx, channelEntryCSV, currentID); err != nil { + errs = append(errs, err) + continue + } + + // update depth to depth + 1 for replaced entry + _, err = updateDepth.Exec(c.Name, manifest.PackageName, replaces) + if err != nil { + errs = append(errs, err) + continue + } + + // insert dummy skips entries if needed or update the graph based on skips + depth = 1 + for _, skip := range skips { + // add dummy channel entry for the skipped version + skippedChannelEntry, err := addChannelEntry.Exec(c.Name, manifest.PackageName, skip, depth) + if err != nil { + errs = append(errs, err) + continue + } + + skippedID, err := skippedChannelEntry.LastInsertId() + if err != nil { + errs = append(errs, err) + continue + } + + // add another channel entry for the parent, which replaces the skipped + synthesizedChannelEntry, err := addChannelEntry.Exec(c.Name, manifest.PackageName, c.CurrentCSVName, depth) + if err != nil { + errs = append(errs, err) + continue + } + + synthesizedID, err := synthesizedChannelEntry.LastInsertId() + if err != nil { + errs = append(errs, err) + continue + } + + if _, err = addReplaces.Exec(skippedID, synthesizedID); err != nil { + errs = append(errs, err) + continue + } + + if err := s.addAPIs(tx, channelEntryCSV, synthesizedID); err != nil { + errs = append(errs, err) + continue + } + + depth++ + } + } + + if errs != nil { + return utilerrors.NewAggregate(errs) + } return nil } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/000_init.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/000_init.go index 67df6476cb..00986acae8 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/000_init.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/000_init.go @@ -7,6 +7,10 @@ import ( var InitMigrationKey = 0 +func init() { + registerMigration(InitMigrationKey, initMigration) +} + var initMigration = &Migration{ Id: InitMigrationKey, Up: func(ctx context.Context, tx *sql.Tx) error { @@ -72,7 +76,3 @@ var initMigration = &Migration{ return err }, } - -func init() { - migrations[InitMigrationKey] = initMigration -} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/001_related_images.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/001_related_images.go index 392f722526..3b3c8c36b9 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/001_related_images.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/001_related_images.go @@ -14,7 +14,7 @@ import ( const RelatedImagesMigrationKey = 1 func init() { - migrations[RelatedImagesMigrationKey] = relatedImagesMigration + registerMigration(RelatedImagesMigrationKey, relatedImagesMigration) } // listBundles returns a list of operatorbundles as strings diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/002_bundle_path.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/002_bundle_path.go index 0ba0f09f06..b16c13d76b 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/002_bundle_path.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/002_bundle_path.go @@ -9,7 +9,7 @@ const BundlePathMigrationKey = 2 // Register this migration func init() { - migrations[BundlePathMigrationKey] = bundlePathMigration + registerMigration(BundlePathMigrationKey, bundlePathMigration) } var bundlePathMigration = &Migration{ @@ -23,13 +23,13 @@ var bundlePathMigration = &Migration{ return err }, Down: func(ctx context.Context, tx *sql.Tx) error { - foreingKeyOff := `PRAGMA foreign_keys = 0` + foreignKeyOff := `PRAGMA foreign_keys = 0` createTempTable := `CREATE TABLE operatorbundle_backup (name TEXT,csv TEXT,bundle TEXT)` backupTargetTable := `INSERT INTO operatorbundle_backup SELECT name,csv,bundle FROM operatorbundle` dropTargetTable := `DROP TABLE operatorbundle` renameBackUpTable := `ALTER TABLE operatorbundle_backup RENAME TO operatorbundle;` - foreingKeyOn := `PRAGMA foreign_keys = 1` - _, err := tx.ExecContext(ctx, foreingKeyOff) + foreignKeyOn := `PRAGMA foreign_keys = 1` + _, err := tx.ExecContext(ctx, foreignKeyOff) if err != nil { return err } @@ -49,7 +49,7 @@ var bundlePathMigration = &Migration{ if err != nil { return err } - _, err = tx.ExecContext(ctx, foreingKeyOn) + _, err = tx.ExecContext(ctx, foreignKeyOn) return err }, } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/003_required_apis.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/003_required_apis.go index dc1b451d74..08006a0300 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/003_required_apis.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/003_required_apis.go @@ -13,7 +13,7 @@ const RequiredApiMigrationKey = 3 // Register this migration func init() { - migrations[RequiredApiMigrationKey] = requiredApiMigration + registerMigration(RequiredApiMigrationKey, requiredApiMigration) } var requiredApiMigration = &Migration{ diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/004_cascade_delete.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/004_cascade_delete.go new file mode 100644 index 0000000000..6cdd9bed21 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/004_cascade_delete.go @@ -0,0 +1,453 @@ +package migrations + +import ( + "context" + "database/sql" +) + +var CascadeDeleteMigrationKey = 4 + +// Register this migration +func init() { + registerMigration(CascadeDeleteMigrationKey, cascadeDeleteMigration) +} + +var cascadeDeleteMigration = &Migration{ + Id: CascadeDeleteMigrationKey, + Up: func(ctx context.Context, tx *sql.Tx) error { + foreingKeyOff := `PRAGMA foreign_keys = 0` + renameTable := func(table string) string { + return `ALTER TABLE ` + table + ` RENAME TO ` + table + `_old;` + } + createNewOperatorBundleTable := ` + CREATE TABLE operatorbundle ( + name TEXT PRIMARY KEY, + csv TEXT, + bundle TEXT, + bundlepath TEXT);` + createNewPackageTable := ` + CREATE TABLE package ( + name TEXT PRIMARY KEY, + default_channel TEXT, + FOREIGN KEY(name, default_channel) REFERENCES channel(package_name,name) ON DELETE CASCADE + );` + createNewChannelTable := ` + CREATE TABLE channel ( + name TEXT, + package_name TEXT, + head_operatorbundle_name TEXT, + PRIMARY KEY(name, package_name), + FOREIGN KEY(head_operatorbundle_name) REFERENCES operatorbundle(name) ON DELETE CASCADE + );` + createNewChannelEntryTable := ` + CREATE TABLE channel_entry ( + entry_id INTEGER PRIMARY KEY, + channel_name TEXT, + package_name TEXT, + operatorbundle_name TEXT, + replaces INTEGER, + depth INTEGER, + FOREIGN KEY(replaces) REFERENCES channel_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + FOREIGN KEY(channel_name, package_name) REFERENCES channel(name, package_name) ON DELETE CASCADE + );` + createNewAPIProviderTable := ` + CREATE TABLE api_provider ( + group_name TEXT, + version TEXT, + kind TEXT, + channel_entry_id INTEGER, + PRIMARY KEY(group_name, version, kind, channel_entry_id), + FOREIGN KEY(channel_entry_id) REFERENCES channel_entry(entry_id) ON DELETE CASCADE, + FOREIGN KEY(group_name, version, kind) REFERENCES api(group_name, version, kind) + );` + createNewRelatedImageTable := ` + CREATE TABLE related_image ( + image TEXT, + operatorbundle_name TEXT, + FOREIGN KEY(operatorbundle_name) REFERENCES operatorbundle(name) ON DELETE CASCADE + );` + createNewAPIRequirerTable := ` + CREATE TABLE api_requirer ( + group_name TEXT, + version TEXT, + kind TEXT, + channel_entry_id INTEGER, + PRIMARY KEY(group_name, version, kind, channel_entry_id), + FOREIGN KEY(channel_entry_id) REFERENCES channel_entry(entry_id) ON DELETE CASCADE, + FOREIGN KEY(group_name, version, kind) REFERENCES api(group_name, version, kind) + );` + newTableTransfer := func(table string) string { + return `INSERT INTO ` + table + ` SELECT * FROM "` + table + `_old"` + } + dropTable := func(table string) string { + return `DROP TABLE "` + table + `_old"` + } + foreingKeyOn := `PRAGMA foreign_keys = 1` + + _, err := tx.ExecContext(ctx, foreingKeyOff) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`operatorbundle`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`package`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`channel`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`channel_entry`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`api_provider`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`related_image`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`api_requirer`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createNewOperatorBundleTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createNewPackageTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createNewChannelTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createNewChannelEntryTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createNewAPIProviderTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createNewRelatedImageTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createNewAPIRequirerTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, newTableTransfer(`operatorbundle`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`operatorbundle`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, newTableTransfer(`package`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`package`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, newTableTransfer(`channel`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`channel`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, newTableTransfer(`channel_entry`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`channel_entry`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, newTableTransfer(`api_provider`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`api_provider`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, newTableTransfer(`related_image`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`related_image`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, newTableTransfer(`api_requirer`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`api_requirer`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, foreingKeyOn) + return err + }, + Down: func(ctx context.Context, tx *sql.Tx) error { + foreingKeyOff := `PRAGMA foreign_keys = 0` + renameTable := func(table string) string { + return `ALTER TABLE ` + table + ` RENAME TO ` + table + `_old;` + } + createBackupOperatorBundleTable := ` + CREATE TABLE operatorbundle ( + name TEXT PRIMARY KEY, + csv TEXT UNIQUE, + bundle TEXT, + bundlepath TEXT);` + createBackupPackageTable := ` + CREATE TABLE IF NOT EXISTS package ( + name TEXT PRIMARY KEY, + default_channel TEXT, + FOREIGN KEY(name, default_channel) REFERENCES channel(package_name,name) + );` + createBackupChannelTable := ` + CREATE TABLE IF NOT EXISTS channel ( + name TEXT, + package_name TEXT, + head_operatorbundle_name TEXT, + PRIMARY KEY(name, package_name), + FOREIGN KEY(package_name) REFERENCES package(name), + FOREIGN KEY(head_operatorbundle_name) REFERENCES operatorbundle(name) + );` + createBackupChannelEntryTable := ` + CREATE TABLE IF NOT EXISTS channel_entry ( + entry_id INTEGER PRIMARY KEY, + channel_name TEXT, + package_name TEXT, + operatorbundle_name TEXT, + replaces INTEGER, + depth INTEGER, + FOREIGN KEY(replaces) REFERENCES channel_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + FOREIGN KEY(channel_name, package_name) REFERENCES channel(name, package_name) + );` + createBackupAPIProviderTable := ` + CREATE TABLE IF NOT EXISTS api_provider ( + group_name TEXT, + version TEXT, + kind TEXT, + channel_entry_id INTEGER, + FOREIGN KEY(channel_entry_id) REFERENCES channel_entry(entry_id), + FOREIGN KEY(group_name, version, kind) REFERENCES api(group_name, version, kind) + );` + createBackupRelatedImageTable := ` + CREATE TABLE IF NOT EXISTS related_image ( + image TEXT, + operatorbundle_name TEXT, + FOREIGN KEY(operatorbundle_name) REFERENCES operatorbundle(name) + );` + createBackupAPIRequirerTable := ` + CREATE TABLE IF NOT EXISTS api_requirer ( + group_name TEXT, + version TEXT, + kind TEXT, + channel_entry_id INTEGER, + FOREIGN KEY(channel_entry_id) REFERENCES channel_entry(entry_id), + FOREIGN KEY(group_name, version, kind) REFERENCES api(group_name, version, kind) + );` + backupTableTransfer := func(table string) string { + return `INSERT INTO ` + table + ` SELECT * FROM "` + table + `_old"` + } + dropTable := func(table string) string { + return `DROP TABLE "` + table + `_old"` + } + + foreingKeyOn := `PRAGMA foreign_keys = 1` + + _, err := tx.ExecContext(ctx, foreingKeyOff) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`operatorbundle`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`package`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`channel`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`channel_entry`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`api_provider`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`related_image`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, renameTable(`api_requirer`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createBackupOperatorBundleTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createBackupPackageTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createBackupChannelTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createBackupChannelEntryTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createBackupAPIProviderTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createBackupRelatedImageTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, createBackupAPIRequirerTable) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, backupTableTransfer(`operatorbundle`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`operatorbundle`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, backupTableTransfer(`package`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`package`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, backupTableTransfer(`channel`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`channel`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, backupTableTransfer(`channel_entry`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`channel_entry`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, backupTableTransfer(`api_provider`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`api_provider`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, backupTableTransfer(`related_image`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`related_image`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, backupTableTransfer(`api_requirer`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, dropTable(`api_requirer`)) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, foreingKeyOn) + return err + }, +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/005_version_skiprange.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/005_version_skiprange.go new file mode 100644 index 0000000000..7c56236f64 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/005_version_skiprange.go @@ -0,0 +1,94 @@ +package migrations + +import ( + "context" + "database/sql" + + "github.com/sirupsen/logrus" +) + +const VersionSkipRangeMigrationKey = 5 +const SkipRangeAnnotationKey = "olm.skipRange" + +// Register this migration +func init() { + registerMigration(VersionSkipRangeMigrationKey, versionSkipRangeMigration) +} + +var versionSkipRangeMigration = &Migration{ + Id: VersionSkipRangeMigrationKey, + Up: func(ctx context.Context, tx *sql.Tx) error { + sql := ` + ALTER TABLE operatorbundle + ADD COLUMN skiprange TEXT; + + ALTER TABLE operatorbundle + ADD COLUMN version TEXT; + ` + _, err := tx.ExecContext(ctx, sql) + if err != nil { + return err + } + + bundles, err := listBundles(ctx, tx) + if err != nil { + return err + } + for _, bundle := range bundles { + if err := extractVersioning(ctx, tx, bundle); err != nil { + logrus.Warnf("error backfilling related images: %v", err) + continue + } + } + return err + }, + Down: func(ctx context.Context, tx *sql.Tx) error { + foreignKeyOff := `PRAGMA foreign_keys = 0` + createTempTable := `CREATE TABLE operatorbundle_backup (name TEXT, csv TEXT, bundle TEXT, bundlepath TEXT)` + backupTargetTable := `INSERT INTO operatorbundle_backup SELECT name, csv, bundle, bundlepath FROM operatorbundle` + dropTargetTable := `DROP TABLE operatorbundle` + renameBackUpTable := `ALTER TABLE operatorbundle_backup RENAME TO operatorbundle;` + foreignKeyOn := `PRAGMA foreign_keys = 1` + _, err := tx.ExecContext(ctx, foreignKeyOff) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, createTempTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, backupTargetTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, dropTargetTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, renameBackUpTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, foreignKeyOn) + return err + }, +} + +func extractVersioning(ctx context.Context, tx *sql.Tx, name string) error { + addSql := `insert into operatorbundle(version, skiprange) values(?,?)` + csv, err := getCSV(ctx, tx, name) + if err != nil { + logrus.Warnf("error backfilling versioning: %v", err) + return err + } + skiprange, ok := csv.Annotations[SkipRangeAnnotationKey] + if !ok { + skiprange = "" + } + version, err := csv.GetVersion() + if err != nil { + version = "" + } + _, err = tx.ExecContext(ctx, addSql, version, skiprange) + return err +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/migrations.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/migrations.go index d2802c2ee0..b9bb60fbaf 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/migrations.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/migrations.go @@ -3,6 +3,7 @@ package migrations import ( "context" "database/sql" + "fmt" "sort" ) @@ -80,3 +81,13 @@ func Only(key int) Migrations { func All() MigrationSet { return migrations } + +func registerMigration(key int, m *Migration) { + if _, ok := migrations[key]; ok { + panic(fmt.Sprintf("already have a migration registered with id %d", key)) + } + if m.Id != key { + panic(fmt.Sprintf("migration has wrong id for key. key: %d, id: %d", key, m.Id)) + } + migrations[key] = m +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrator.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrator.go index f1c349200b..da86bbc202 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrator.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrator.go @@ -69,9 +69,14 @@ func (m *SQLLiteMigrator) Up(ctx context.Context, migrations migrations.Migratio if err != nil { return err } + var commitErr error defer func() { + if commitErr == nil { + return + } + logrus.WithError(commitErr).Warningf("tx commit failed") if err := tx.Rollback(); err != nil { - logrus.WithError(err).Debugf("couldn't rollback - this is expected if the transaction committed") + logrus.WithError(err).Warningf("couldn't rollback after failed commit") } }() @@ -97,10 +102,8 @@ func (m *SQLLiteMigrator) Up(ctx context.Context, migrations migrations.Migratio return err } } - if err := tx.Commit(); err != nil { - return err - } - return nil + commitErr = tx.Commit() + return commitErr } func (m *SQLLiteMigrator) Down(ctx context.Context, migrations migrations.Migrations) error { @@ -108,9 +111,14 @@ func (m *SQLLiteMigrator) Down(ctx context.Context, migrations migrations.Migrat if err != nil { return err } + var commitErr error defer func() { + if commitErr == nil { + return + } + logrus.WithError(commitErr).Warningf("tx commit failed") if err := tx.Rollback(); err != nil { - logrus.WithError(err).Debugf("couldn't rollback - this is expected if the transaction committed") + logrus.WithError(err).Warningf("couldn't rollback after failed commit") } }() if err := m.ensureMigrationTable(ctx, tx); err != nil { @@ -135,10 +143,8 @@ func (m *SQLLiteMigrator) Down(ctx context.Context, migrations migrations.Migrat return err } } - if err := tx.Commit(); err != nil { - return err - } - return nil + commitErr = tx.Commit() + return commitErr } func (m *SQLLiteMigrator) ensureMigrationTable(ctx context.Context, tx *sql.Tx) error { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go index 54d3d95e53..0ebc4f6954 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go @@ -36,6 +36,8 @@ func (s *SQLQuerier) ListTables(ctx context.Context) ([]string, error) { if err != nil { return nil, err } + defer rows.Close() + tables := []string{} for rows.Next() { var tableName sql.NullString @@ -56,6 +58,8 @@ func (s *SQLQuerier) ListPackages(ctx context.Context) ([]string, error) { if err != nil { return nil, err } + defer rows.Close() + packages := []string{} for rows.Next() { var pkgName sql.NullString @@ -77,6 +81,7 @@ func (s *SQLQuerier) GetPackage(ctx context.Context, name string) (*registry.Pac if err != nil { return nil, err } + defer rows.Close() var pkgName sql.NullString var defaultChannel sql.NullString @@ -109,13 +114,14 @@ func (s *SQLQuerier) GetPackage(ctx context.Context, name string) (*registry.Pac } func (s *SQLQuerier) GetBundle(ctx context.Context, pkgName, channelName, csvName string) (*api.Bundle, error) { - query := `SELECT DISTINCT channel_entry.entry_id, operatorbundle.name, operatorbundle.bundle, operatorbundle.bundlepath + query := `SELECT DISTINCT channel_entry.entry_id, operatorbundle.name, operatorbundle.bundle, operatorbundle.bundlepath, operatorbundle.version, operatorbundle.skiprange FROM operatorbundle INNER JOIN channel_entry ON operatorbundle.name=channel_entry.operatorbundle_name WHERE channel_entry.package_name=? AND channel_entry.channel_name=? AND operatorbundle_name=? LIMIT 1` rows, err := s.db.QueryContext(ctx, query, pkgName, channelName, csvName) if err != nil { return nil, err } + defer rows.Close() if !rows.Next() { return nil, fmt.Errorf("no entry found for %s %s %s", pkgName, channelName, csvName) @@ -124,7 +130,9 @@ func (s *SQLQuerier) GetBundle(ctx context.Context, pkgName, channelName, csvNam var name sql.NullString var bundle sql.NullString var bundlePath sql.NullString - if err := rows.Scan(&entryId, &name, &bundle, &bundlePath); err != nil { + var version sql.NullString + var skipRange sql.NullString + if err := rows.Scan(&entryId, &name, &bundle, &bundlePath, &version, &skipRange); err != nil { return nil, err } @@ -139,6 +147,8 @@ func (s *SQLQuerier) GetBundle(ctx context.Context, pkgName, channelName, csvNam out.PackageName = pkgName out.ChannelName = channelName out.BundlePath = bundlePath.String + out.Version = version.String + out.SkipRange = skipRange.String provided, required, err := s.GetApisForEntry(ctx, entryId.Int64) if err != nil { @@ -151,7 +161,7 @@ func (s *SQLQuerier) GetBundle(ctx context.Context, pkgName, channelName, csvNam } func (s *SQLQuerier) GetBundleForChannel(ctx context.Context, pkgName string, channelName string) (*api.Bundle, error) { - query := `SELECT DISTINCT channel_entry.entry_id, operatorbundle.name, operatorbundle.bundle, operatorbundle.bundlepath FROM channel + query := `SELECT DISTINCT channel_entry.entry_id, operatorbundle.name, operatorbundle.bundle, operatorbundle.bundlepath, operatorbundle.version, operatorbundle.skiprange FROM channel INNER JOIN operatorbundle ON channel.head_operatorbundle_name=operatorbundle.name INNER JOIN channel_entry ON (channel_entry.channel_name = channel.name and channel_entry.package_name=channel.package_name and channel_entry.operatorbundle_name=operatorbundle.name) WHERE channel.package_name=? AND channel.name=? LIMIT 1` @@ -159,6 +169,7 @@ func (s *SQLQuerier) GetBundleForChannel(ctx context.Context, pkgName string, ch if err != nil { return nil, err } + defer rows.Close() if !rows.Next() { return nil, fmt.Errorf("no entry found for %s %s", pkgName, channelName) @@ -167,7 +178,9 @@ func (s *SQLQuerier) GetBundleForChannel(ctx context.Context, pkgName string, ch var name sql.NullString var bundle sql.NullString var bundlePath sql.NullString - if err := rows.Scan(&entryId, &name, &bundle, &bundlePath); err != nil { + var version sql.NullString + var skipRange sql.NullString + if err := rows.Scan(&entryId, &name, &bundle, &bundlePath, &version, &skipRange); err != nil { return nil, err } @@ -182,6 +195,8 @@ func (s *SQLQuerier) GetBundleForChannel(ctx context.Context, pkgName string, ch out.PackageName = pkgName out.ChannelName = channelName out.BundlePath = bundlePath.String + out.Version = version.String + out.SkipRange = skipRange.String provided, required, err := s.GetApisForEntry(ctx, entryId.Int64) if err != nil { @@ -202,6 +217,7 @@ func (s *SQLQuerier) GetChannelEntriesThatReplace(ctx context.Context, name stri if err != nil { return } + defer rows.Close() entries = []*registry.ChannelEntry{} @@ -228,7 +244,7 @@ func (s *SQLQuerier) GetChannelEntriesThatReplace(ctx context.Context, name stri } func (s *SQLQuerier) GetBundleThatReplaces(ctx context.Context, name, pkgName, channelName string) (*api.Bundle, error) { - query := `SELECT DISTINCT replaces.entry_id, operatorbundle.name, operatorbundle.bundle, operatorbundle.bundlepath + query := `SELECT DISTINCT replaces.entry_id, operatorbundle.name, operatorbundle.bundle, operatorbundle.bundlepath, operatorbundle.version, operatorbundle.skiprange FROM channel_entry LEFT OUTER JOIN channel_entry replaces ON replaces.replaces = channel_entry.entry_id INNER JOIN operatorbundle ON replaces.operatorbundle_name = operatorbundle.name @@ -237,6 +253,8 @@ func (s *SQLQuerier) GetBundleThatReplaces(ctx context.Context, name, pkgName, c if err != nil { return nil, err } + defer rows.Close() + if !rows.Next() { return nil, fmt.Errorf("no entry found for %s %s", pkgName, channelName) @@ -245,7 +263,9 @@ func (s *SQLQuerier) GetBundleThatReplaces(ctx context.Context, name, pkgName, c var outName sql.NullString var bundle sql.NullString var bundlePath sql.NullString - if err := rows.Scan(&entryId, &outName, &bundle, &bundlePath); err != nil { + var version sql.NullString + var skipRange sql.NullString + if err := rows.Scan(&entryId, &outName, &bundle, &bundlePath, &version, &skipRange); err != nil { return nil, err } @@ -260,6 +280,8 @@ func (s *SQLQuerier) GetBundleThatReplaces(ctx context.Context, name, pkgName, c out.PackageName = pkgName out.ChannelName = channelName out.BundlePath = bundlePath.String + out.Version = version.String + out.SkipRange = skipRange.String provided, required, err := s.GetApisForEntry(ctx, entryId.Int64) if err != nil { @@ -282,6 +304,7 @@ func (s *SQLQuerier) GetChannelEntriesThatProvide(ctx context.Context, group, ve if err != nil { return } + defer rows.Close() entries = []*registry.ChannelEntry{} @@ -320,6 +343,7 @@ func (s *SQLQuerier) GetLatestChannelEntriesThatProvide(ctx context.Context, gro if err != nil { return nil, err } + defer rows.Close() entries = []*registry.ChannelEntry{} @@ -348,8 +372,8 @@ func (s *SQLQuerier) GetLatestChannelEntriesThatProvide(ctx context.Context, gro } // Get the the latest bundle that provides the API in a default channel, error unless there is ONLY one -func (s *SQLQuerier) GetBundleThatProvides(ctx context.Context, group, version, kind string) (*api.Bundle, error) { - query := `SELECT DISTINCT channel_entry.entry_id, operatorbundle.bundle, operatorbundle.bundlepath, MIN(channel_entry.depth), channel_entry.operatorbundle_name, channel_entry.package_name, channel_entry.channel_name, channel_entry.replaces +func (s *SQLQuerier) GetBundleThatProvides(ctx context.Context, group, apiVersion, kind string) (*api.Bundle, error) { + query := `SELECT DISTINCT channel_entry.entry_id, operatorbundle.bundle, operatorbundle.bundlepath, MIN(channel_entry.depth), channel_entry.operatorbundle_name, channel_entry.package_name, channel_entry.channel_name, channel_entry.replaces, operatorbundle.version, operatorbundle.skiprange FROM channel_entry INNER JOIN api_provider ON channel_entry.entry_id = api_provider.channel_entry_id INNER JOIN operatorbundle ON operatorbundle.name = channel_entry.operatorbundle_name @@ -357,13 +381,14 @@ func (s *SQLQuerier) GetBundleThatProvides(ctx context.Context, group, version, WHERE api_provider.group_name = ? AND api_provider.version = ? AND api_provider.kind = ? AND package.default_channel = channel_entry.channel_name GROUP BY channel_entry.package_name, channel_entry.channel_name` - rows, err := s.db.QueryContext(ctx, query, group, version, kind) + rows, err := s.db.QueryContext(ctx, query, group, apiVersion, kind) if err != nil { return nil, err } + defer rows.Close() if !rows.Next() { - return nil, fmt.Errorf("no entry found that provides %s %s %s", group, version, kind) + return nil, fmt.Errorf("no entry found that provides %s %s %s", group, apiVersion, kind) } var entryId sql.NullInt64 var bundle sql.NullString @@ -373,12 +398,14 @@ func (s *SQLQuerier) GetBundleThatProvides(ctx context.Context, group, version, var pkgName sql.NullString var channelName sql.NullString var replaces sql.NullString - if err := rows.Scan(&entryId, &bundle, &bundlePath, &min_depth, &bundleName, &pkgName, &channelName, &replaces); err != nil { + var version sql.NullString + var skipRange sql.NullString + if err := rows.Scan(&entryId, &bundle, &bundlePath, &min_depth, &bundleName, &pkgName, &channelName, &replaces, &version, &skipRange); err != nil { return nil, err } if !bundle.Valid { - return nil, fmt.Errorf("no entry found that provides %s %s %s", group, version, kind) + return nil, fmt.Errorf("no entry found that provides %s %s %s", group, apiVersion, kind) } out := &api.Bundle{} @@ -392,6 +419,8 @@ func (s *SQLQuerier) GetBundleThatProvides(ctx context.Context, group, version, out.PackageName = pkgName.String out.ChannelName = channelName.String out.BundlePath = bundlePath.String + out.Version = version.String + out.SkipRange = skipRange.String provided, required, err := s.GetApisForEntry(ctx, entryId.Int64) if err != nil { @@ -409,6 +438,8 @@ func (s *SQLQuerier) ListImages(ctx context.Context) ([]string, error) { if err != nil { return nil, err } + defer rows.Close() + images := []string{} for rows.Next() { var imgName sql.NullString @@ -428,6 +459,7 @@ func (s *SQLQuerier) GetImagesForBundle(ctx context.Context, csvName string) ([] if err != nil { return nil, err } + defer rows.Close() images := []string{} for rows.Next() { var imgName sql.NullString @@ -450,6 +482,7 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provid if err != nil { return nil,nil, err } + provided = []*api.GroupVersionKind{} for providedRows.Next() { var groupName sql.NullString @@ -470,6 +503,9 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provid Plural: pluralName.String, }) } + if err := providedRows.Close(); err != nil { + return nil, nil, err + } requiredQuery := `SELECT DISTINCT api.group_name, api.version, api.kind, api.plural FROM api INNER JOIN api_requirer ON (api.group_name=api_requirer.group_name AND api.version=api_requirer.version AND api.kind=api_requirer.kind) @@ -499,6 +535,9 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provid Plural: pluralName.String, }) } + if err := requiredRows.Close(); err != nil { + return nil, nil, err + } return } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go new file mode 100644 index 0000000000..644950adeb --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go @@ -0,0 +1,67 @@ +package sqlite + +import ( + "fmt" + "strings" + + "github.com/sirupsen/logrus" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "github.com/operator-framework/operator-registry/pkg/registry" +) + +type SQLRemover interface { + Remove() error +} + +// PackageRemover removes a package from the database +type PackageRemover struct { + store registry.Load + packages string +} + +var _ SQLRemover = &PackageRemover{} + +func NewSQLRemoverForPackages(store registry.Load, packages string) *PackageRemover { + return &PackageRemover{ + store: store, + packages: packages, + } +} + +func (d *PackageRemover) Remove() error { + log := logrus.WithField("pkg", d.packages) + + log.Info("deleting packages") + + var errs []error + packages := sanitizePackageList(strings.Split(d.packages, ",")) + log.Info("input has been sanitized") + log.Infof("packages: %s", packages) + + for _, pkg := range packages { + if err := d.store.RmPackageName(pkg); err != nil { + errs = append(errs, fmt.Errorf("error removing operator package %s: %s", pkg, err)) + } + } + + return utilerrors.NewAggregate(errs) +} + +// sanitizePackageList sanitizes the set of package(s) specified. It removes +// duplicates and ignores empty string. +func sanitizePackageList(in []string) []string { + out := make([]string, 0) + + inMap := map[string]bool{} + for _, item := range in { + if _, ok := inMap[item]; ok || item == "" { + continue + } + + inMap[item] = true + out = append(out, item) + } + + return out +} diff --git a/vendor/modules.txt b/vendor/modules.txt index bdd871bad7..ee26b26c4f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -168,10 +168,11 @@ github.com/openshift/client-go/config/informers/externalversions/config github.com/openshift/client-go/config/informers/externalversions/config/v1 github.com/openshift/client-go/config/informers/externalversions/internalinterfaces github.com/openshift/client-go/config/listers/config/v1 -# github.com/operator-framework/operator-registry v1.5.1 +# github.com/operator-framework/operator-registry v1.5.3 github.com/operator-framework/operator-registry/pkg/api github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1 github.com/operator-framework/operator-registry/pkg/client +github.com/operator-framework/operator-registry/pkg/containertools github.com/operator-framework/operator-registry/pkg/registry github.com/operator-framework/operator-registry/pkg/server github.com/operator-framework/operator-registry/pkg/sqlite From 2b343b57c844bfc0f7f29ee36accc377b5ba864d Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Wed, 6 Nov 2019 18:37:39 -0500 Subject: [PATCH 2/2] test(e2e): testing CRD upgrades shouldn't create a new subscription instead, updates should be simulated - in this case, by switching to a channel that contains an update. --- test/e2e/installplan_e2e_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/e2e/installplan_e2e_test.go b/test/e2e/installplan_e2e_test.go index 4ba345033e..a8c9a0075f 100644 --- a/test/e2e/installplan_e2e_test.go +++ b/test/e2e/installplan_e2e_test.go @@ -962,8 +962,8 @@ func TestInstallPlanWithCRDSchemaChange(t *testing.T) { require.NoError(t, err) subscriptionName := genName("sub-nginx-alpha-") - // this subscription will be cleaned up below without the clean up function - createSubscriptionForCatalog(t, crc, testNamespace, subscriptionName, mainCatalogSourceName, mainPackageName, stableChannel, "", v1alpha1.ApprovalAutomatic) + cleanupSubscription := createSubscriptionForCatalog(t, crc, testNamespace, subscriptionName, mainCatalogSourceName, mainPackageName, stableChannel, "", v1alpha1.ApprovalAutomatic) + defer cleanupSubscription() subscription, err := fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanChecker) require.NoError(t, err) @@ -1010,15 +1010,18 @@ func TestInstallPlanWithCRDSchemaChange(t *testing.T) { // Attempt to get the catalog source before creating install plan(s) _, err = fetchCatalogSource(t, crc, mainCatalogSourceName, testNamespace, catalogSourceRegistryPodSynced) require.NoError(t, err) + // Update the subscription resource to point to the beta CSV - err = crc.OperatorsV1alpha1().Subscriptions(testNamespace).DeleteCollection(metav1.NewDeleteOptions(0), metav1.ListOptions{}) + subscription, err = fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanChecker) require.NoError(t, err) + require.NotNil(t, subscription) - // existing cleanup should remove this - subscriptionName = genName("sub-nginx-beta") - createSubscriptionForCatalog(t, crc, testNamespace, subscriptionName, mainCatalogSourceName, mainPackageName, betaChannel, "", v1alpha1.ApprovalAutomatic) + subscription.Spec.Channel = betaChannel + subscription, err = crc.OperatorsV1alpha1().Subscriptions(testNamespace).Update(subscription) + require.NoError(t, err) - subscription, err = fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanChecker) + // Wait for subscription to have a new installplan + subscription, err = fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanDifferentChecker(fetchedInstallPlan.GetName())) require.NoError(t, err) require.NotNil(t, subscription)