From d261f8d33ed07beee86ed76a09c05ef5b8fe947e Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 21 May 2026 14:43:11 -0700 Subject: [PATCH 1/3] Support pre-built function runtime images in projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crossplane projects today discover embedded functions by convention: every subdirectory of paths.functions is treated as a function, and the CLI auto-detects the language and builds the runtime image. This works well for simple projects but blocks projects that have outgrown the built-in builders or that need to coordinate function builds with an existing build system (make, nix, Bazel, CI pipelines). Per https://github.com/crossplane/cli/issues/21, users want to supply pre-built OCI runtime images alongside source-based functions, so the CLI handles packaging while the user owns the build. This commit adds an optional functions list to ProjectSpec. When the list is present it disables auto-discovery and is the sole source of truth for which functions to build. Each entry uses a Source discriminator (Directory or Tarball) and a corresponding sub-field: spec: architectures: [amd64, arm64] functions: - source: Directory directory: name: function-a - source: Tarball tarball: name: function-b pathPrefix: build/function-b Directory-source functions follow the existing build path. Tarball- source functions skip language detection and load one pre-built single-platform OCI image tarball per target architecture, following the naming convention `-.tar`. So the example above loads `build/function-b-amd64.tar` and `build/function-b-arm64.tar`. Per-architecture tarballs match what build tools naturally produce without bundling: `docker save`, Nix's dockerTools.buildImage, Bazel's oci_tarball, `ko build --tarball`, etc. all emit one single-platform tarball at a time. Packaging is inherently per- architecture too — each runtime image gets its own crossplane.yaml layer before they're tied together into a multi-arch package index — so the CLI would have to split a multi-arch input apart anyway. The CLI verifies that each tarball's image config records the architecture its filename promises, and adds the package metadata layer (crossplane.yaml) before assembling the multi-arch package index. The on-disk output is identical to a CLI-built function. When the functions list is omitted, the existing auto-discovery behaviour is preserved unchanged. Fixes https://github.com/crossplane/cli/issues/21. Signed-off-by: Nic Cope --- apis/dev/v1alpha1/project_types.go | 86 ++++++ apis/dev/v1alpha1/validate.go | 95 +++++++ apis/dev/v1alpha1/validate_test.go | 163 +++++++++++ apis/dev/v1alpha1/zz_generated.deepcopy.go | 62 +++++ internal/project/build.go | 210 ++++++++++---- internal/project/build_test.go | 301 +++++++++++++++++++++ 6 files changed, 863 insertions(+), 54 deletions(-) diff --git a/apis/dev/v1alpha1/project_types.go b/apis/dev/v1alpha1/project_types.go index dc9dad5..255bb52 100644 --- a/apis/dev/v1alpha1/project_types.go +++ b/apis/dev/v1alpha1/project_types.go @@ -33,6 +33,18 @@ const ( DependencyTypeXpkg = "xpkg" ) +// Function source constants. +const ( + // FunctionSourceDirectory indicates a function whose source code lives in + // a directory under the project's functions path. The CLI builds the + // runtime image from this source. + FunctionSourceDirectory = "Directory" + // FunctionSourceTarball indicates a function whose runtime image is + // supplied as a pre-built OCI image tarball. The CLI skips building and + // uses the tarball as the runtime image. + FunctionSourceTarball = "Tarball" +) + // Project defines a Crossplane Project, which can be built into a Crossplane // Configuration package. // @@ -64,6 +76,12 @@ type ProjectSpec struct { Crossplane *pkgmetav1.CrossplaneConstraints `json:"crossplane,omitempty"` // Dependencies are built-time and runtime dependencies of the project. Dependencies []Dependency `json:"dependencies,omitempty"` + // Functions explicitly declares the embedded functions in this project. + // If specified, automatic discovery of functions under the functions path + // is disabled and only the functions listed here will be built and + // packaged. If omitted, all subdirectories of the functions path are + // treated as Directory-source functions and built automatically. + Functions []Function `json:"functions,omitempty"` // Paths defines the relative paths to various parts of the project. Paths *ProjectPaths `json:"paths,omitempty"` // Architectures indicates for which architectures embedded functions should @@ -186,3 +204,71 @@ type K8sDependency struct { // Version is the Kubernetes API version (e.g., "v1.33.0"). Version string `json:"version"` } + +// Function explicitly declares an embedded function in a Crossplane project. +// The Source field is the discriminator that determines which sub-field is +// relevant. +type Function struct { + // Source defines how the function's runtime image is supplied. + // +kubebuilder:validation:Enum=Directory;Tarball + Source string `json:"source"` + + // Directory describes a function whose source code lives in a directory + // under the project's functions path. The CLI builds the runtime image + // from this source. Only used when Source is "Directory". + // +optional + Directory *FunctionDirectory `json:"directory,omitempty"` + + // Tarball describes a function whose runtime image is supplied as a + // pre-built OCI image tarball. Only used when Source is "Tarball". + // +optional + Tarball *FunctionTarball `json:"tarball,omitempty"` +} + +// Name returns the name of the function, derived from the source-specific +// fields. +func (f *Function) Name() string { + switch f.Source { + case FunctionSourceDirectory: + if f.Directory == nil { + return "" + } + return f.Directory.Name + case FunctionSourceTarball: + if f.Tarball == nil { + return "" + } + return f.Tarball.Name + } + return "" +} + +// FunctionDirectory describes a function whose source code lives in a +// directory under the project's functions path. +type FunctionDirectory struct { + // Name is the name of the function. It must match the name of a + // subdirectory under the project's functions path. + Name string `json:"name"` +} + +// FunctionTarball describes a function whose runtime images are supplied as +// pre-built single-platform OCI image tarballs (as produced by `docker save`, +// Nix's dockerTools.buildImage, Bazel's oci_tarball, ko --tarball, etc.). +// +// The CLI expects one tarball per target architecture, named according to the +// convention `-.tar` and resolved relative to the project +// root. For example, with PathPrefix "build/function-b" and project +// architectures [amd64, arm64], the CLI loads: +// +// build/function-b-amd64.tar +// build/function-b-arm64.tar +type FunctionTarball struct { + // Name is the name of the function. It is used to derive the OCI + // repository for the function's package as `_`. + Name string `json:"name"` + + // PathPrefix is the prefix of the per-architecture runtime image + // tarballs, relative to the project root. For each target architecture + // the CLI loads the file at `-.tar`. + PathPrefix string `json:"pathPrefix"` +} diff --git a/apis/dev/v1alpha1/validate.go b/apis/dev/v1alpha1/validate.go index 7bb63b2..d358f29 100644 --- a/apis/dev/v1alpha1/validate.go +++ b/apis/dev/v1alpha1/validate.go @@ -20,6 +20,8 @@ import ( "fmt" "path/filepath" + "k8s.io/apimachinery/pkg/util/validation" + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" ) @@ -75,6 +77,23 @@ func (s *ProjectSpec) Validate() error { } } + // Validate functions. Names must be unique across the list, regardless of + // source, since the function name is used to derive both the package + // metadata name and the OCI repository. + seen := make(map[string]int, len(s.Functions)) + for i, fn := range s.Functions { + if err := fn.Validate(); err != nil { + errs = append(errs, errors.Wrapf(err, "function %d", i)) + continue + } + name := fn.Name() + if first, ok := seen[name]; ok { + errs = append(errs, errors.Errorf("function %d: name %q is already used by function %d", i, name, first)) + continue + } + seen[name] = i + } + return errors.Join(errs...) } @@ -172,3 +191,79 @@ func (k *K8sDependency) Validate() error { return errors.Join(errs...) } + +// Validate validates a Function declaration. +func (f *Function) Validate() error { + var errs []error + + // Count non-nil sources to enforce that exactly one matches the + // discriminator. + sourceCount := 0 + if f.Directory != nil { + sourceCount++ + } + if f.Tarball != nil { + sourceCount++ + } + if sourceCount != 1 { + errs = append(errs, errors.New("exactly one source (directory or tarball) must be specified")) + } + + switch f.Source { + case FunctionSourceDirectory: + if err := f.Directory.Validate(); err != nil { + errs = append(errs, fmt.Errorf("directory: %w", err)) + } + case FunctionSourceTarball: + if err := f.Tarball.Validate(); err != nil { + errs = append(errs, fmt.Errorf("tarball: %w", err)) + } + case "": + errs = append(errs, errors.New("source must not be empty")) + default: + errs = append(errs, errors.Errorf("source %q is not supported, must be one of %q or %q", f.Source, FunctionSourceDirectory, FunctionSourceTarball)) + } + + return errors.Join(errs...) +} + +// Validate validates a FunctionDirectory. A nil receiver is invalid; this is +// the failure mode when a function is declared with source Directory but no +// directory field set. +func (d *FunctionDirectory) Validate() error { + if d == nil { + return errors.Errorf("source %q requires the directory field to be set", FunctionSourceDirectory) + } + + var errs []error + if d.Name == "" { + errs = append(errs, errors.New("name must not be empty")) + } else if msgs := validation.IsDNS1123Subdomain(d.Name); len(msgs) > 0 { + errs = append(errs, errors.Errorf("name %q is not a valid DNS-1123 subdomain: %v", d.Name, msgs)) + } + + return errors.Join(errs...) +} + +// Validate validates a FunctionTarball. A nil receiver is invalid; this is +// the failure mode when a function is declared with source Tarball but no +// tarball field set. +func (t *FunctionTarball) Validate() error { + if t == nil { + return errors.Errorf("source %q requires the tarball field to be set", FunctionSourceTarball) + } + + var errs []error + if t.Name == "" { + errs = append(errs, errors.New("name must not be empty")) + } else if msgs := validation.IsDNS1123Subdomain(t.Name); len(msgs) > 0 { + errs = append(errs, errors.Errorf("name %q is not a valid DNS-1123 subdomain: %v", t.Name, msgs)) + } + if t.PathPrefix == "" { + errs = append(errs, errors.New("pathPrefix must not be empty")) + } else if filepath.IsAbs(t.PathPrefix) { + errs = append(errs, errors.New("pathPrefix must be relative")) + } + + return errors.Join(errs...) +} diff --git a/apis/dev/v1alpha1/validate_test.go b/apis/dev/v1alpha1/validate_test.go index 66c4ab1..6e01653 100644 --- a/apis/dev/v1alpha1/validate_test.go +++ b/apis/dev/v1alpha1/validate_test.go @@ -285,6 +285,169 @@ func TestValidate(t *testing.T) { "dependency 0: k8s: version must not be empty", }, }, + "ValidDirectoryFunction": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + }, + "ValidTarballFunction": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Tarball: &FunctionTarball{Name: "fn-two", PathPrefix: "build/fn-two"}, + }}, + }, + }, + }, + "FunctionMissingSource": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: source must not be empty", + }, + }, + "FunctionUnknownSource": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: "Mystery", + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + `function 0: source "Mystery" is not supported`, + }, + }, + "FunctionDirectorySourceMissingDirectory": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Tarball: &FunctionTarball{Name: "fn-one", PathPrefix: "build/fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + `function 0: directory: source "Directory" requires the directory field to be set`, + }, + }, + "FunctionTarballSourceMissingTarball": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + `function 0: tarball: source "Tarball" requires the tarball field to be set`, + }, + }, + "FunctionMultipleSources": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Directory: &FunctionDirectory{Name: "fn-one"}, + Tarball: &FunctionTarball{Name: "fn-one", PathPrefix: "build/fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: exactly one source (directory or tarball) must be specified", + }, + }, + "FunctionDirectoryEmptyName": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Directory: &FunctionDirectory{}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: directory: name must not be empty", + }, + }, + "FunctionTarballEmptyPathPrefix": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Tarball: &FunctionTarball{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: tarball: pathPrefix must not be empty", + }, + }, + "FunctionTarballAbsolutePathPrefix": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Tarball: &FunctionTarball{Name: "fn-one", PathPrefix: "/abs/prefix"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: tarball: pathPrefix must be relative", + }, + }, + "FunctionDuplicateName": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{ + {Source: FunctionSourceDirectory, Directory: &FunctionDirectory{Name: "shared"}}, + {Source: FunctionSourceTarball, Tarball: &FunctionTarball{Name: "shared", PathPrefix: "build/shared"}}, + }, + }, + }, + expectedErrors: []string{ + `function 1: name "shared" is already used by function 0`, + }, + }, } for name, tc := range tcs { diff --git a/apis/dev/v1alpha1/zz_generated.deepcopy.go b/apis/dev/v1alpha1/zz_generated.deepcopy.go index 19714fc..533c804 100644 --- a/apis/dev/v1alpha1/zz_generated.deepcopy.go +++ b/apis/dev/v1alpha1/zz_generated.deepcopy.go @@ -61,6 +61,61 @@ func (in *Dependency) DeepCopy() *Dependency { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Function) DeepCopyInto(out *Function) { + *out = *in + if in.Directory != nil { + in, out := &in.Directory, &out.Directory + *out = new(FunctionDirectory) + **out = **in + } + if in.Tarball != nil { + in, out := &in.Tarball, &out.Tarball + *out = new(FunctionTarball) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Function. +func (in *Function) DeepCopy() *Function { + if in == nil { + return nil + } + out := new(Function) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FunctionDirectory) DeepCopyInto(out *FunctionDirectory) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FunctionDirectory. +func (in *FunctionDirectory) DeepCopy() *FunctionDirectory { + if in == nil { + return nil + } + out := new(FunctionDirectory) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FunctionTarball) DeepCopyInto(out *FunctionTarball) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FunctionTarball. +func (in *FunctionTarball) DeepCopy() *FunctionTarball { + if in == nil { + return nil + } + out := new(FunctionTarball) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitDependency) DeepCopyInto(out *GitDependency) { *out = *in @@ -178,6 +233,13 @@ func (in *ProjectSpec) DeepCopyInto(out *ProjectSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Functions != nil { + in, out := &in.Functions, &out.Functions + *out = make([]Function, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Paths != nil { in, out := &in.Paths, &out.Paths *out = new(ProjectPaths) diff --git a/internal/project/build.go b/internal/project/build.go index 92628a9..6322794 100644 --- a/internal/project/build.go +++ b/internal/project/build.go @@ -29,6 +29,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/spf13/afero" "golang.org/x/sync/errgroup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -181,6 +182,15 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p } functionsSource := afero.NewBasePathFs(projectFS, project.Spec.Paths.Functions) + + // Determine the set of functions to build. If the project explicitly + // declares a Functions list we use it verbatim. Otherwise we auto-discover + // by listing subdirectories of the functions path. + fns, err := resolveFunctions(project, functionsSource) + if err != nil { + return nil, errors.Wrap(err, "failed to resolve functions") + } + apisSource := projectFS apiExcludes := []string{ project.Spec.Paths.Examples, @@ -248,9 +258,9 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p o.eventCh.SendEvent("Generating schemas", async.EventStatusSuccess) } - // Find and build embedded functions. + // Build the resolved functions. o.log.Debug("Building functions") - imgMap, deps, err := b.buildFunctions(ctx, projectFS, functionsSource, project, o.projectBasePath, o.eventCh) + imgMap, deps, err := b.buildFunctions(ctx, projectFS, functionsSource, project, fns, o.projectBasePath, o.eventCh) if err != nil { return nil, err } @@ -301,50 +311,60 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p return imgMap, nil } -// buildFunctions builds the embedded functions found in directories at the top -// level of the provided filesystem. -func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afero.Fs, project *devv1alpha1.Project, basePath string, eventCh async.EventChannel) (ImageTagMap, []xpmetav1.Dependency, error) { - var ( - imgMap = make(map[name.Tag]v1.Image) - imgMu sync.Mutex - ) +// resolveFunctions returns the list of functions to build for the project. If +// the project explicitly declares functions, that list is returned verbatim. +// Otherwise it auto-discovers Directory-source functions by listing +// subdirectories of the project's functions path. +func resolveFunctions(project *devv1alpha1.Project, functionsSource afero.Fs) ([]devv1alpha1.Function, error) { + if len(project.Spec.Functions) > 0 { + return project.Spec.Functions, nil + } - infos, err := afero.ReadDir(fromFS, "/") + infos, err := afero.ReadDir(functionsSource, "/") switch { case os.IsNotExist(err): - return imgMap, nil, nil + return nil, nil case err != nil: - return nil, nil, errors.Wrap(err, "failed to list functions directory") + return nil, errors.Wrap(err, "failed to list functions directory") } - fnDirs := make([]string, 0, len(infos)) + fns := make([]devv1alpha1.Function, 0, len(infos)) for _, info := range infos { - if info.IsDir() { - fnDirs = append(fnDirs, info.Name()) + if !info.IsDir() { + continue } + fns = append(fns, devv1alpha1.Function{ + Source: devv1alpha1.FunctionSourceDirectory, + Directory: &devv1alpha1.FunctionDirectory{Name: info.Name()}, + }) } + return fns, nil +} + +// buildFunctions builds the given list of embedded functions. +func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afero.Fs, project *devv1alpha1.Project, fns []devv1alpha1.Function, basePath string, eventCh async.EventChannel) (ImageTagMap, []xpmetav1.Dependency, error) { + var ( + imgMap = make(map[name.Tag]v1.Image) + imgMu sync.Mutex + ) - deps := make([]xpmetav1.Dependency, len(fnDirs)) + deps := make([]xpmetav1.Dependency, len(fns)) eg, ctx := errgroup.WithContext(ctx) sem := make(chan struct{}, b.maxConcurrency) - for i, fnName := range fnDirs { + for i, fn := range fns { eg.Go(func() error { sem <- struct{}{} defer func() { <-sem }() + fnName := fn.Name() eventText := fmt.Sprintf("Building function %s", fnName) eventCh.SendEvent(eventText, async.EventStatusStarted) fnRepo := fmt.Sprintf("%s_%s", project.Spec.Repository, fnName) - fnFS := afero.NewBasePathFs(fromFS, fnName) - fnBasePath := "" - if basePath != "" { - fnBasePath = filepath.Join(basePath, project.Spec.Paths.Functions, fnName) - } - imgs, err := b.buildFunction(ctx, projectFS, fnFS, project, fnName, fnBasePath) + imgs, err := b.buildFunction(ctx, projectFS, fromFS, project, fn, basePath) if err != nil { eventCh.SendEvent(eventText, async.EventStatusFailure) return errors.Wrapf(err, "failed to build function %q", fnName) @@ -387,18 +407,19 @@ func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afer }) } - err = eg.Wait() - if err != nil { + if err := eg.Wait(); err != nil { return nil, nil, err } return imgMap, deps, nil } -// buildFunction builds images for a single function whose source resides in the -// given filesystem. -func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero.Fs, project *devv1alpha1.Project, fnName string, basePath string) ([]v1.Image, error) { - fn := &xpmetav1.Function{ +// buildFunction builds the package images for a single function. It resolves +// the function's runtime images (either by building from source or by loading +// a pre-built tarball) and then wraps each one with the package metadata. +func (b *realBuilder) buildFunction(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { + fnName := fn.Name() + meta := &xpmetav1.Function{ TypeMeta: metav1.TypeMeta{ APIVersion: xpmetav1.SchemeGroupVersion.String(), Kind: xpmetav1.FunctionKind, @@ -414,7 +435,7 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero }, } metaFS := afero.NewMemMapFs() - y, err := yaml.Marshal(fn) + y, err := yaml.Marshal(meta) if err != nil { return nil, errors.Wrap(err, "failed to marshal function metadata") } @@ -423,18 +444,24 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero return nil, errors.Wrap(err, "failed to write function metadata") } + // Source the examples from the function's own directory if it's a + // Directory-source function. Tarball-source functions don't have a source + // directory under functions/, so they have no examples to ship. examplesParser := parser.NewEchoBackend("") - examplesExist, err := afero.IsDir(fromFS, "/examples") - switch { - case err == nil, os.IsNotExist(err): - default: - return nil, errors.Wrap(err, "failed to check for examples") - } - if examplesExist { - examplesParser = parser.NewFsBackend(fromFS, - parser.FsDir("/examples"), - parser.FsFilters(parser.SkipNotYAML()), - ) + if fn.Source == devv1alpha1.FunctionSourceDirectory { + fnFS := afero.NewBasePathFs(functionsFS, fn.Directory.Name) + examplesExist, err := afero.IsDir(fnFS, "/examples") + switch { + case err == nil, os.IsNotExist(err): + default: + return nil, errors.Wrap(err, "failed to check for examples") + } + if examplesExist { + examplesParser = parser.NewFsBackend(fnFS, + parser.FsDir("/examples"), + parser.FsFilters(parser.SkipNotYAML()), + ) + } } pp, err := pyaml.New() @@ -448,37 +475,112 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero examples.New(), ) - fnBuilder, err := b.functionIdentifier.Identify(fromFS, project.Spec.ImageConfigs) + runtimeImages, err := b.runtimeImages(ctx, projectFS, functionsFS, project, fn, basePath) if err != nil { - return nil, errors.Wrap(err, "failed to find a builder") + return nil, err } - if bfs, ok := fromFS.(*afero.BasePathFs); ok && basePath == "" { - basePath = afero.FullBaseFsPath(bfs, ".") + pkgImages := make([]v1.Image, 0, len(runtimeImages)) + for _, img := range runtimeImages { + pkgImage, _, err := builder.Build(ctx, xpkg.WithBase(img)) + if err != nil { + return nil, errors.Wrap(err, "failed to build function package") + } + pkgImages = append(pkgImages, pkgImage) + } + + return pkgImages, nil +} + +// runtimeImages returns the per-architecture runtime images for a function. For +// Directory-source functions this dispatches to the appropriate builder. For +// Tarball-source functions it loads the supplied OCI tarball. +func (b *realBuilder) runtimeImages(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { + switch fn.Source { + case devv1alpha1.FunctionSourceDirectory: + return b.buildDirectoryRuntime(ctx, projectFS, functionsFS, project, fn.Directory, basePath) + case devv1alpha1.FunctionSourceTarball: + return loadTarballRuntime(projectFS, fn.Tarball, project.Spec.Architectures) + default: + // Should be caught at validation time, but be defensive. + return nil, errors.Errorf("unsupported function source %q", fn.Source) + } +} + +// buildDirectoryRuntime invokes the appropriate language builder to produce +// runtime images from a function's source directory. +func (b *realBuilder) buildDirectoryRuntime(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, dir *devv1alpha1.FunctionDirectory, basePath string) ([]v1.Image, error) { + fnFS := afero.NewBasePathFs(functionsFS, dir.Name) + + fnBasePath := "" + if basePath != "" { + fnBasePath = filepath.Join(basePath, project.Spec.Paths.Functions, dir.Name) + } + if bfs, ok := fnFS.(*afero.BasePathFs); ok && fnBasePath == "" { + fnBasePath = afero.FullBaseFsPath(bfs, ".") } - runtimeImages, err := fnBuilder.Build(ctx, functions.BuildContext{ + fnBuilder, err := b.functionIdentifier.Identify(fnFS, project.Spec.ImageConfigs) + if err != nil { + return nil, errors.Wrap(err, "failed to find a builder") + } + + imgs, err := fnBuilder.Build(ctx, functions.BuildContext{ ProjectFS: projectFS, - FunctionPath: filepath.Join(project.Spec.Paths.Functions, fnName), + FunctionPath: filepath.Join(project.Spec.Paths.Functions, dir.Name), SchemasPath: project.Spec.Paths.Schemas, Architectures: project.Spec.Architectures, - OSBasePath: basePath, + OSBasePath: fnBasePath, }) if err != nil { return nil, errors.Wrap(err, "failed to build runtime images") } + return imgs, nil +} - pkgImages := make([]v1.Image, 0, len(runtimeImages)) +// loadTarballRuntime reads one pre-built single-platform OCI image tarball per +// target architecture, following the naming convention +// `-.tar`. The tarball format is the Docker-style image +// tarball produced by `docker save`, Nix's dockerTools.buildImage, Bazel's +// oci_tarball, `ko build --tarball`, etc. +func loadTarballRuntime(projectFS afero.Fs, tb *devv1alpha1.FunctionTarball, architectures []string) ([]v1.Image, error) { + images := make([]v1.Image, 0, len(architectures)) + for _, arch := range architectures { + rel := fmt.Sprintf("%s-%s.tar", tb.PathPrefix, arch) + path := resolveProjectPath(projectFS, rel) + + img, err := tarball.ImageFromPath(path, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to load runtime image for architecture %q from %q", arch, rel) + } - for _, img := range runtimeImages { - pkgImage, _, err := builder.Build(ctx, xpkg.WithBase(img)) + // The image's own config records the platform it was built for. If + // it doesn't match the architecture we expected based on the file + // name, the user has almost certainly made a packaging mistake; + // fail loudly rather than producing a multi-arch index that lies. + cfg, err := img.ConfigFile() if err != nil { - return nil, errors.Wrap(err, "failed to build function package") + return nil, errors.Wrapf(err, "failed to read config for runtime image %q", rel) } - pkgImages = append(pkgImages, pkgImage) + if cfg.Architecture != arch { + return nil, errors.Errorf("runtime image %q reports architecture %q but was expected to be %q", rel, cfg.Architecture, arch) + } + + images = append(images, img) } + return images, nil +} - return pkgImages, nil +// resolveProjectPath returns the real on-disk path for a path that is +// expressed relative to the project root. go-containerregistry reads tarballs +// directly from the filesystem, so we need to translate any afero.BasePathFs +// indirection back to a real path. +func resolveProjectPath(projectFS afero.Fs, rel string) string { + path := rel + if bfs, ok := projectFS.(*afero.BasePathFs); ok { + path = filepath.Join(afero.FullBaseFsPath(bfs, ""), rel) + } + return filepath.Clean(path) } func collectResources(toFS afero.Fs, fromFS afero.Fs, gvks []string, exclude []string) error { diff --git a/internal/project/build_test.go b/internal/project/build_test.go index 8f74c35..2b7a0ef 100644 --- a/internal/project/build_test.go +++ b/internal/project/build_test.go @@ -18,10 +18,16 @@ package project import ( "fmt" + "path/filepath" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -296,3 +302,298 @@ func tagsOf(m ImageTagMap) []string { } return out } + +func TestResolveFunctions(t *testing.T) { + t.Parallel() + + tcs := map[string]struct { + spec devv1alpha1.ProjectSpec + fnDirs []string + fnFiles []string // files (not dirs) under the functions path; should be ignored. + want []devv1alpha1.Function + }{ + "ExplicitListWins": { + // When the project declares functions explicitly, + // auto-discovery is disabled and the list is returned verbatim. + spec: devv1alpha1.ProjectSpec{ + Functions: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "explicit"}}, + }, + }, + fnDirs: []string{"would-be-discovered"}, + want: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "explicit"}}, + }, + }, + "AutoDiscoverDirectories": { + // Every subdirectory of the functions path becomes a + // Directory-source function. + fnDirs: []string{"fn-a", "fn-b"}, + want: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "fn-a"}}, + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "fn-b"}}, + }, + }, + "AutoDiscoverIgnoresFiles": { + // Files directly under the functions path are not treated as + // functions; only subdirectories are. + fnDirs: []string{"fn-real"}, + fnFiles: []string{"README.md", "stray.tar"}, + want: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "fn-real"}}, + }, + }, + "AutoDiscoverNoFunctionsDir": { + // A missing functions path is not an error; it just yields no + // functions. + }, + } + + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + for _, d := range tc.fnDirs { + if err := projFS.MkdirAll(filepath.Join("functions", d), 0o755); err != nil { + t.Fatal(err) + } + } + for _, f := range tc.fnFiles { + if err := projFS.MkdirAll("functions", 0o755); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, filepath.Join("functions", f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "p"}, + Spec: tc.spec, + } + proj.Spec.Repository = "xpkg.crossplane.io/example/test" + proj.Default() + + fnsSource := afero.NewBasePathFs(projFS, proj.Spec.Paths.Functions) + got, err := resolveFunctions(proj, fnsSource) + if err != nil { + t.Fatalf("resolveFunctions: %v", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("resolveFunctions(...): -want, +got:\n%s", diff) + } + }) + } +} + +func TestBuilderBuildExplicitFunctions(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + writeProject(t, projFS, + map[string]string{ + "db.yaml": xrdYAML("acme.example.com", "xdatabases", "xdatabase", "XDatabase"), + "db-comp.yaml": compositionYAML("xdb", "acme.example.com", "XDatabase"), + }, + // Auto-discovery would find fn-auto; explicit functions should + // override and only build fn-explicit. + []string{"fn-auto", "fn-explicit"}, + ) + + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: devv1alpha1.ProjectSpec{ + Repository: "xpkg.crossplane.io/example/test", + Functions: []devv1alpha1.Function{{ + Source: devv1alpha1.FunctionSourceDirectory, + Directory: &devv1alpha1.FunctionDirectory{Name: "fn-explicit"}, + }}, + }, + } + proj.Default() + + imgMap, err := NewBuilder(BuildWithFunctionIdentifier(functions.FakeIdentifier)).Build(t.Context(), proj, projFS) + if err != nil { + t.Fatalf("Build: %v", err) + } + + // The configuration image plus per-arch images for fn-explicit. fn-auto + // must not appear because the explicit list disables auto-discovery. + want := map[string]bool{ + proj.Spec.Repository: true, + proj.Spec.Repository + "_fn-explicit": true, + } + got := map[string]bool{} + for tag := range imgMap { + got[tag.Repository.Name()] = true + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Build(...) function repos: -want, +got:\n%s", diff) + } +} + +func TestBuilderBuildTarballFunction(t *testing.T) { + t.Parallel() + + // Build one single-platform Docker-style tarball per architecture, named + // using the -.tar convention. + tmp := t.TempDir() + for _, arch := range []string{"amd64", "arm64"} { + writeRuntimeTar(t, filepath.Join(tmp, "fn-prebuilt-"+arch+".tar"), arch) + } + + projFS := afero.NewBasePathFs(afero.NewOsFs(), tmp) + if err := projFS.MkdirAll("apis", 0o755); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, "apis/db.yaml", []byte(xrdYAML("acme.example.com", "xdatabases", "xdatabase", "XDatabase")), 0o644); err != nil { + t.Fatal(err) + } + + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: devv1alpha1.ProjectSpec{ + Repository: "xpkg.crossplane.io/example/test", + Architectures: []string{"amd64", "arm64"}, + Functions: []devv1alpha1.Function{{ + Source: devv1alpha1.FunctionSourceTarball, + Tarball: &devv1alpha1.FunctionTarball{Name: "fn-prebuilt", PathPrefix: "fn-prebuilt"}, + }}, + }, + } + proj.Default() + + imgMap, err := NewBuilder(BuildWithFunctionIdentifier(functions.FakeIdentifier)).Build(t.Context(), proj, projFS) + if err != nil { + t.Fatalf("Build: %v", err) + } + + // The pre-built tarballs should produce one package image per target + // architecture under the function's derived repo. + wantRepo := proj.Spec.Repository + "_fn-prebuilt" + want := map[string]int{wantRepo: 2} + got := map[string]int{} + for tag := range imgMap { + got[tag.Repository.Name()]++ + } + if diff := cmp.Diff(want, got, cmpopts.IgnoreMapEntries(func(k string, _ int) bool { + return k != wantRepo + })); diff != "" { + t.Errorf("Build(...) tarball function images: -want, +got:\n%s", diff) + } +} + +func TestLoadTarballRuntime(t *testing.T) { + t.Parallel() + + type args struct { + // files maps relative file names under the project root to the + // architecture the runtime image they contain should report. + files map[string]string + archs []string + } + type want struct { + archs []string + err error + } + + tcs := map[string]struct { + args args + want want + }{ + "AllArchitecturesPresent": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + "fn-arm64.tar": "arm64", + }, + archs: []string{"amd64", "arm64"}, + }, + want: want{archs: []string{"amd64", "arm64"}}, + }, + "MissingArchitectureFile": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + }, + archs: []string{"amd64", "arm64"}, + }, + want: want{err: cmpopts.AnyError}, + }, + "ArchitectureMismatch": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "arm64", + }, + archs: []string{"amd64"}, + }, + want: want{err: cmpopts.AnyError}, + }, + "SingleArchitecture": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + }, + archs: []string{"amd64"}, + }, + want: want{archs: []string{"amd64"}}, + }, + } + + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + for fname, arch := range tc.args.files { + writeRuntimeTar(t, filepath.Join(tmp, fname), arch) + } + + projFS := afero.NewBasePathFs(afero.NewOsFs(), tmp) + tb := &devv1alpha1.FunctionTarball{Name: "fn", PathPrefix: "fn"} + got, err := loadTarballRuntime(projFS, tb, tc.args.archs) + + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("loadTarballRuntime(...): -want error, +got error:\n%s", diff) + } + if diff := cmp.Diff(tc.want.archs, archsOf(t, got), cmpopts.EquateEmpty()); diff != "" { + t.Errorf("loadTarballRuntime(...) architectures: -want, +got:\n%s", diff) + } + }) + } +} + +// archsOf returns the architecture each image reports, in order. +func archsOf(t *testing.T, imgs []v1.Image) []string { + t.Helper() + + archs := make([]string, 0, len(imgs)) + for _, img := range imgs { + cfg, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + archs = append(archs, cfg.Architecture) + } + return archs +} + +// writeRuntimeTar writes a single-platform Docker-style image tarball at the +// given path containing an empty image whose config records the given +// architecture. +func writeRuntimeTar(t *testing.T, path, arch string) { + t.Helper() + + img, err := mutate.ConfigFile(empty.Image, &v1.ConfigFile{OS: "linux", Architecture: arch}) + if err != nil { + t.Fatal(err) + } + tag, err := name.NewTag("crossplane.io/test:" + arch) + if err != nil { + t.Fatal(err) + } + if err := tarball.WriteToFile(path, tag, img); err != nil { + t.Fatal(err) + } +} From 4ba2fdd1f733f29a4421e081630d5075309ba80b Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 21 May 2026 20:52:21 -0700 Subject: [PATCH 2/3] Support gzipped function runtime image tarballs Nix's dockerTools.buildImage produces gzipped tarballs by default. Some other build tools (Bazel rules_oci's oci_load, certain ko invocations) do the same. With only plain .tar accepted, users of these tools had to add a decompress step to their build pipeline just to feed images to the Crossplane CLI. This commit teaches the function tarball loader to fall back to `-.tar.gz` when `-.tar` is not present, preferring the plain tar when both exist. The gzipped tarball is streamed through gzip.NewReader into go-containerregistry's tarball.Image; no temporary files are written. Signed-off-by: Nic Cope --- apis/dev/v1alpha1/project_types.go | 17 +++--- internal/project/build.go | 88 +++++++++++++++++++++++++++--- internal/project/build_test.go | 54 +++++++++++++++++- 3 files changed, 141 insertions(+), 18 deletions(-) diff --git a/apis/dev/v1alpha1/project_types.go b/apis/dev/v1alpha1/project_types.go index 255bb52..a7c43be 100644 --- a/apis/dev/v1alpha1/project_types.go +++ b/apis/dev/v1alpha1/project_types.go @@ -255,13 +255,14 @@ type FunctionDirectory struct { // pre-built single-platform OCI image tarballs (as produced by `docker save`, // Nix's dockerTools.buildImage, Bazel's oci_tarball, ko --tarball, etc.). // -// The CLI expects one tarball per target architecture, named according to the -// convention `-.tar` and resolved relative to the project -// root. For example, with PathPrefix "build/function-b" and project -// architectures [amd64, arm64], the CLI loads: +// The CLI expects one tarball per target architecture, named according to +// the convention `-.tar` or `-.tar.gz`, +// resolved relative to the project root. The CLI prefers the plain `.tar` +// when both are present. For example, with PathPrefix "build/function-b" and +// project architectures [amd64, arm64], the CLI looks for: // -// build/function-b-amd64.tar -// build/function-b-arm64.tar +// build/function-b-amd64.tar (or .tar.gz) +// build/function-b-arm64.tar (or .tar.gz) type FunctionTarball struct { // Name is the name of the function. It is used to derive the OCI // repository for the function's package as `_`. @@ -269,6 +270,8 @@ type FunctionTarball struct { // PathPrefix is the prefix of the per-architecture runtime image // tarballs, relative to the project root. For each target architecture - // the CLI loads the file at `-.tar`. + // the CLI loads either `-.tar` or + // `-.tar.gz`, preferring the former when both are + // present. PathPrefix string `json:"pathPrefix"` } diff --git a/internal/project/build.go b/internal/project/build.go index 6322794..90b6909 100644 --- a/internal/project/build.go +++ b/internal/project/build.go @@ -18,8 +18,10 @@ limitations under the License. package project import ( + "compress/gzip" "context" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -539,19 +541,21 @@ func (b *realBuilder) buildDirectoryRuntime(ctx context.Context, projectFS, func } // loadTarballRuntime reads one pre-built single-platform OCI image tarball per -// target architecture, following the naming convention -// `-.tar`. The tarball format is the Docker-style image -// tarball produced by `docker save`, Nix's dockerTools.buildImage, Bazel's -// oci_tarball, `ko build --tarball`, etc. +// target architecture. For each architecture it looks for, in order: +// +// - -.tar +// - -.tar.gz +// +// The tarball format is the Docker-style image tarball produced by +// `docker save`, Nix's dockerTools.buildImage, Bazel's oci_tarball, +// `ko build --tarball`, etc. The gzipped variant is what most Nix image +// builders emit by default. func loadTarballRuntime(projectFS afero.Fs, tb *devv1alpha1.FunctionTarball, architectures []string) ([]v1.Image, error) { images := make([]v1.Image, 0, len(architectures)) for _, arch := range architectures { - rel := fmt.Sprintf("%s-%s.tar", tb.PathPrefix, arch) - path := resolveProjectPath(projectFS, rel) - - img, err := tarball.ImageFromPath(path, nil) + img, rel, err := loadRuntimeImage(projectFS, tb.PathPrefix, arch) if err != nil { - return nil, errors.Wrapf(err, "failed to load runtime image for architecture %q from %q", arch, rel) + return nil, err } // The image's own config records the platform it was built for. If @@ -571,6 +575,72 @@ func loadTarballRuntime(projectFS afero.Fs, tb *devv1alpha1.FunctionTarball, arc return images, nil } +// loadRuntimeImage loads the runtime image for a single architecture, trying +// the plain .tar file first and falling back to a gzipped .tar.gz file. It +// returns the loaded image and the relative path (for error messages). +func loadRuntimeImage(projectFS afero.Fs, prefix, arch string) (v1.Image, string, error) { + plain := fmt.Sprintf("%s-%s.tar", prefix, arch) + if exists, err := afero.Exists(projectFS, plain); err != nil { + return nil, plain, errors.Wrapf(err, "failed to stat runtime image %q", plain) + } else if exists { + img, err := tarball.ImageFromPath(resolveProjectPath(projectFS, plain), nil) + if err != nil { + return nil, plain, errors.Wrapf(err, "failed to load runtime image for architecture %q from %q", arch, plain) + } + return img, plain, nil + } + + gzipped := fmt.Sprintf("%s-%s.tar.gz", prefix, arch) + if exists, err := afero.Exists(projectFS, gzipped); err != nil { + return nil, gzipped, errors.Wrapf(err, "failed to stat runtime image %q", gzipped) + } else if exists { + img, err := tarball.Image(gzipOpener(resolveProjectPath(projectFS, gzipped)), nil) + if err != nil { + return nil, gzipped, errors.Wrapf(err, "failed to load runtime image for architecture %q from %q", arch, gzipped) + } + return img, gzipped, nil + } + + return nil, plain, errors.Errorf("no runtime image found for architecture %q: looked for %q and %q", arch, plain, gzipped) +} + +// gzipOpener returns a tarball.Opener that reads a gzipped tar file. It can +// be called repeatedly; each call returns a fresh decompressing reader that +// reads the file from the beginning. tarball.Image calls its opener multiple +// times - once for the manifest and once per layer - so re-decompression is +// required. +func gzipOpener(path string) tarball.Opener { + return func() (io.ReadCloser, error) { + f, err := os.Open(path) //nolint:gosec // Path is provided by the project author. + if err != nil { + return nil, err + } + gz, err := gzip.NewReader(f) + if err != nil { + _ = f.Close() + return nil, err + } + return gzipReadCloser{Reader: gz, file: f}, nil + } +} + +// gzipReadCloser ties together a gzip.Reader and the underlying file so that +// closing the gzip reader also closes the file. +type gzipReadCloser struct { + *gzip.Reader + + file *os.File +} + +func (g gzipReadCloser) Close() error { + gerr := g.Reader.Close() + ferr := g.file.Close() + if gerr != nil { + return gerr + } + return ferr +} + // resolveProjectPath returns the real on-disk path for a path that is // expressed relative to the project root. go-containerregistry reads tarballs // directly from the filesystem, so we need to translate any afero.BasePathFs diff --git a/internal/project/build_test.go b/internal/project/build_test.go index 2b7a0ef..f39e6fd 100644 --- a/internal/project/build_test.go +++ b/internal/project/build_test.go @@ -17,8 +17,11 @@ limitations under the License. package project import ( + "compress/gzip" "fmt" + "os" "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -539,6 +542,37 @@ func TestLoadTarballRuntime(t *testing.T) { }, want: want{archs: []string{"amd64"}}, }, + "GzippedTarball": { + args: args{ + files: map[string]string{ + "fn-amd64.tar.gz": "amd64", + }, + archs: []string{"amd64"}, + }, + want: want{archs: []string{"amd64"}}, + }, + "MixedPlainAndGzipped": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + "fn-arm64.tar.gz": "arm64", + }, + archs: []string{"amd64", "arm64"}, + }, + want: want{archs: []string{"amd64", "arm64"}}, + }, + "PlainPreferredOverGzipped": { + // When both .tar and .tar.gz exist for the same architecture, + // the plain .tar is used. + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + "fn-amd64.tar.gz": "arm64", // mismatched on purpose to prove it isn't read. + }, + archs: []string{"amd64"}, + }, + want: want{archs: []string{"amd64"}}, + }, } for name, tc := range tcs { @@ -581,7 +615,7 @@ func archsOf(t *testing.T, imgs []v1.Image) []string { // writeRuntimeTar writes a single-platform Docker-style image tarball at the // given path containing an empty image whose config records the given -// architecture. +// architecture. If the path ends with ".tar.gz" the tarball is gzipped. func writeRuntimeTar(t *testing.T, path, arch string) { t.Helper() @@ -593,7 +627,23 @@ func writeRuntimeTar(t *testing.T, path, arch string) { if err != nil { t.Fatal(err) } - if err := tarball.WriteToFile(path, tag, img); err != nil { + if !strings.HasSuffix(path, ".gz") { + if err := tarball.WriteToFile(path, tag, img); err != nil { + t.Fatal(err) + } + return + } + + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz := gzip.NewWriter(f) + if err := tarball.Write(tag, img, gz); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { t.Fatal(err) } } From 5208019668b92f9e912442583f100e78514d030a Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Fri, 22 May 2026 12:12:35 -0700 Subject: [PATCH 3/3] Support generating schemas for specific languages By default crossplane project build and crossplane dependency update-cache generate schemas for all four supported languages (Go, JSON, KCL, Python). Per https://github.com/crossplane/cli/issues/29 this is wasteful for projects that only consume some of them: every build generates language bindings the project never imports. This commit adds an optional schemas block to ProjectSpec: spec: schemas: languages: [python] When languages is set, schema generation is restricted to the listed languages. The filter applies both to the project's own XRD schemas and to its declared dependencies, and flows through project build/run and dependency update-cache/clean-cache. When schemas is omitted (the default), all languages are generated as before. The schemas block is nested rather than flat to leave room for future schema-related knobs (output paths, generator-specific options) without scattering schema config across ProjectSpec. The supported language identifiers are defined as constants (SchemaLanguageGo, SchemaLanguageJSON, SchemaLanguageKCL, SchemaLanguagePython) in the API package, with SupportedSchemaLanguages returning the canonical set. The schema generator package consumes these constants directly so the two cannot drift, and a test in the generator package asserts that AllLanguages covers exactly the API's declared set. Fixes https://github.com/crossplane/cli/issues/29. Signed-off-by: Nic Cope --- apis/dev/v1alpha1/project_types.go | 42 +++++++++ apis/dev/v1alpha1/validate.go | 27 ++++++ apis/dev/v1alpha1/validate_test.go | 45 ++++++++++ apis/dev/v1alpha1/zz_generated.deepcopy.go | 25 ++++++ cmd/crossplane/dependency/cache.go | 2 + cmd/crossplane/project/build.go | 2 +- cmd/crossplane/project/run.go | 2 +- internal/schemas/generator/go.go | 3 +- internal/schemas/generator/interface.go | 21 ++++- internal/schemas/generator/interface_test.go | 90 ++++++++++++++++++++ internal/schemas/generator/json.go | 3 +- internal/schemas/generator/kcl.go | 3 +- internal/schemas/generator/python.go | 3 +- internal/schemas/manager/manager.go | 3 +- 14 files changed, 263 insertions(+), 8 deletions(-) create mode 100644 internal/schemas/generator/interface_test.go diff --git a/apis/dev/v1alpha1/project_types.go b/apis/dev/v1alpha1/project_types.go index a7c43be..39514d2 100644 --- a/apis/dev/v1alpha1/project_types.go +++ b/apis/dev/v1alpha1/project_types.go @@ -45,6 +45,27 @@ const ( FunctionSourceTarball = "Tarball" ) +// Schema language constants. These are the values accepted in +// ProjectSchemas.Languages. Each corresponds to a schema generator in +// internal/schemas/generator. +const ( + SchemaLanguageGo = "go" + SchemaLanguageJSON = "json" + SchemaLanguageKCL = "kcl" + SchemaLanguagePython = "python" +) + +// SupportedSchemaLanguages returns the set of language identifiers accepted +// in ProjectSchemas.Languages. +func SupportedSchemaLanguages() []string { + return []string{ + SchemaLanguageGo, + SchemaLanguageJSON, + SchemaLanguageKCL, + SchemaLanguagePython, + } +} + // Project defines a Crossplane Project, which can be built into a Crossplane // Configuration package. // @@ -87,6 +108,9 @@ type ProjectSpec struct { // Architectures indicates for which architectures embedded functions should // be built. If not specified, it defaults to [amd64, arm64]. Architectures []string `json:"architectures,omitempty"` + // Schemas configures language-specific schema generation for the + // project's XRDs and declared dependencies. + Schemas *ProjectSchemas `json:"schemas,omitempty"` // ImageConfigs configure how images are fetched during // development. Currently, only rewriting is supported; other options will // be silently ignored. Note that these configs are for development only; @@ -105,6 +129,24 @@ type ProjectPackageMetadata struct { Readme string `json:"readme,omitempty"` } +// ProjectSchemas configures language-specific schema generation. Schemas are +// produced both for the project's own XRDs and for its declared dependencies. +type ProjectSchemas struct { + // Languages restricts schema generation to the listed languages. + // Supported values are "go", "json", "kcl", and "python". If not + // specified, schemas are generated for all supported languages. + Languages []string `json:"languages,omitempty"` +} + +// GetLanguages returns the configured schema languages, or nil if no Schemas +// config is set. It is safe to call on a nil receiver. +func (s *ProjectSchemas) GetLanguages() []string { + if s == nil { + return nil + } + return s.Languages +} + // ProjectPaths configures the locations of various parts of the project, for // use at build time. All paths must be relative to the project root. type ProjectPaths struct { diff --git a/apis/dev/v1alpha1/validate.go b/apis/dev/v1alpha1/validate.go index d358f29..af33dd1 100644 --- a/apis/dev/v1alpha1/validate.go +++ b/apis/dev/v1alpha1/validate.go @@ -19,6 +19,7 @@ package v1alpha1 import ( "fmt" "path/filepath" + "slices" "k8s.io/apimachinery/pkg/util/validation" @@ -70,6 +71,8 @@ func (s *ProjectSpec) Validate() error { errs = append(errs, errors.New("architectures must not be empty")) } + errs = append(errs, s.Schemas.Validate()...) + // Validate dependencies for i, dep := range s.Dependencies { if err := dep.Validate(); err != nil { @@ -97,6 +100,30 @@ func (s *ProjectSpec) Validate() error { return errors.Join(errs...) } +// Validate returns errors for an invalid ProjectSchemas. A nil receiver is +// valid (it means "generate schemas for all languages"); an explicitly empty +// Languages list is rejected because it would disable all schema generation, +// which is almost certainly a mistake. +func (s *ProjectSchemas) Validate() []error { + if s == nil { + return nil + } + if s.Languages == nil { + return nil + } + if len(s.Languages) == 0 { + return []error{errors.New("schemas.languages must not be empty when specified")} + } + supported := SupportedSchemaLanguages() + var errs []error + for i, lang := range s.Languages { + if !slices.Contains(supported, lang) { + errs = append(errs, errors.Errorf("schemas.languages[%d]: %q is not a supported schema language, must be one of %v", i, lang, supported)) + } + } + return errs +} + // Validate validates a dependency. func (d *Dependency) Validate() error { var errs []error diff --git a/apis/dev/v1alpha1/validate_test.go b/apis/dev/v1alpha1/validate_test.go index 6e01653..d7457c5 100644 --- a/apis/dev/v1alpha1/validate_test.go +++ b/apis/dev/v1alpha1/validate_test.go @@ -140,6 +140,51 @@ func TestValidate(t *testing.T) { "architectures must not be empty", }, }, + "ValidSchemaLanguages": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + Languages: []string{"python"}, + }, + }, + }, + }, + "EmptySchemaLanguages": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + Languages: []string{}, + }, + }, + }, + expectedErrors: []string{ + "schemas.languages must not be empty when specified", + }, + }, + "UnsupportedSchemaLanguage": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + Languages: []string{"python", "fortran"}, + }, + }, + }, + expectedErrors: []string{ + `schemas.languages[1]: "fortran" is not a supported schema language`, + }, + }, "ValidAPIDependency": { input: &Project{ ObjectMeta: metav1.ObjectMeta{ diff --git a/apis/dev/v1alpha1/zz_generated.deepcopy.go b/apis/dev/v1alpha1/zz_generated.deepcopy.go index 533c804..dc5d553 100644 --- a/apis/dev/v1alpha1/zz_generated.deepcopy.go +++ b/apis/dev/v1alpha1/zz_generated.deepcopy.go @@ -217,6 +217,26 @@ func (in *ProjectPaths) DeepCopy() *ProjectPaths { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProjectSchemas) DeepCopyInto(out *ProjectSchemas) { + *out = *in + if in.Languages != nil { + in, out := &in.Languages, &out.Languages + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProjectSchemas. +func (in *ProjectSchemas) DeepCopy() *ProjectSchemas { + if in == nil { + return nil + } + out := new(ProjectSchemas) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProjectSpec) DeepCopyInto(out *ProjectSpec) { *out = *in @@ -250,6 +270,11 @@ func (in *ProjectSpec) DeepCopyInto(out *ProjectSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Schemas != nil { + in, out := &in.Schemas, &out.Schemas + *out = new(ProjectSchemas) + (*in).DeepCopyInto(*out) + } if in.ImageConfigs != nil { in, out := &in.ImageConfigs, &out.ImageConfigs *out = make([]v1beta1.ImageConfig, len(*in)) diff --git a/cmd/crossplane/dependency/cache.go b/cmd/crossplane/dependency/cache.go index 81d3804..16a2bd4 100644 --- a/cmd/crossplane/dependency/cache.go +++ b/cmd/crossplane/dependency/cache.go @@ -29,6 +29,7 @@ import ( "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/dependency" "github.com/crossplane/cli/v2/internal/project/projectfile" + "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/terminal" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" @@ -83,6 +84,7 @@ func (c *updateCacheCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) opts := []dependency.ManagerOption{ dependency.WithProjectFile(c.ProjectFile), + dependency.WithSchemaGenerators(generator.Filter(generator.AllLanguages(), proj.Spec.Schemas.GetLanguages())), dependency.WithXpkgClient(client), dependency.WithResolver(resolver), } diff --git a/cmd/crossplane/project/build.go b/cmd/crossplane/project/build.go index aa400a1..ccb4fd1 100644 --- a/cmd/crossplane/project/build.go +++ b/cmd/crossplane/project/build.go @@ -96,7 +96,7 @@ func (c *buildCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) error concurrency := max(1, c.MaxConcurrency) schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas) - generators := generator.AllLanguages() + generators := generator.Filter(generator.AllLanguages(), c.proj.Spec.Schemas.GetLanguages()) schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)) schemaMgr := manager.New(schemasFS, generators, schemaRunner) cacheDir := c.CacheDir diff --git a/cmd/crossplane/project/run.go b/cmd/crossplane/project/run.go index e005ac6..f1f7603 100644 --- a/cmd/crossplane/project/run.go +++ b/cmd/crossplane/project/run.go @@ -149,7 +149,7 @@ func (c *runCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) error { concurrency := max(1, c.MaxConcurrency) schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas) - generators := generator.AllLanguages() + generators := generator.Filter(generator.AllLanguages(), c.proj.Spec.Schemas.GetLanguages()) schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)) schemaMgr := manager.New(schemasFS, generators, schemaRunner) cacheDir := c.CacheDir diff --git a/internal/schemas/generator/go.go b/internal/schemas/generator/go.go index 9e58f90..b83f58c 100644 --- a/internal/schemas/generator/go.go +++ b/internal/schemas/generator/go.go @@ -47,6 +47,7 @@ import ( xpv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/crd" "github.com/crossplane/cli/v2/internal/schemas/runner" ) @@ -140,7 +141,7 @@ var ( type goGenerator struct{} func (goGenerator) Language() string { - return "go" + return devv1alpha1.SchemaLanguageGo } // GenerateFromCRD generates Go schemas for the CRDs in the given filesystem. diff --git a/internal/schemas/generator/interface.go b/internal/schemas/generator/interface.go index 06ef60f..d41300b 100644 --- a/internal/schemas/generator/interface.go +++ b/internal/schemas/generator/interface.go @@ -20,6 +20,7 @@ package generator import ( "context" + "slices" "github.com/spf13/afero" @@ -33,7 +34,9 @@ type Interface interface { GenerateFromOpenAPI(ctx context.Context, fs afero.Fs, runner runner.SchemaRunner) (afero.Fs, error) } -// AllLanguages returns generators for all supported languages. +// AllLanguages returns generators for all supported languages. The set of +// supported language identifiers is defined by +// devv1alpha1.SupportedSchemaLanguages. func AllLanguages() []Interface { return []Interface{ &goGenerator{}, @@ -42,3 +45,19 @@ func AllLanguages() []Interface { &pythonGenerator{}, } } + +// Filter returns the subset of generators whose language identifier appears +// in langs. The order of generators in the result matches the order of all. +// If langs is empty, all generators are returned unchanged. +func Filter(all []Interface, langs []string) []Interface { + if len(langs) == 0 { + return all + } + out := make([]Interface, 0, len(all)) + for _, g := range all { + if slices.Contains(langs, g.Language()) { + out = append(out, g) + } + } + return out +} diff --git a/internal/schemas/generator/interface_test.go b/internal/schemas/generator/interface_test.go new file mode 100644 index 0000000..adf864f --- /dev/null +++ b/internal/schemas/generator/interface_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" +) + +func TestAllLanguagesMatchesAPI(t *testing.T) { + t.Parallel() + + // The generators returned by AllLanguages must cover exactly the set + // of language identifiers declared in the API package. If this test + // fails the two are out of sync; update one to match the other. + got := make([]string, 0, len(AllLanguages())) + for _, g := range AllLanguages() { + got = append(got, g.Language()) + } + if diff := cmp.Diff(devv1alpha1.SupportedSchemaLanguages(), got); diff != "" { + t.Errorf("AllLanguages() languages: -want (from API), +got (from generators):\n%s", diff) + } +} + +func TestFilter(t *testing.T) { + t.Parallel() + + all := AllLanguages() + + tcs := map[string]struct { + langs []string + want []string + }{ + "Empty": { + // An empty filter returns all languages unchanged. + want: devv1alpha1.SupportedSchemaLanguages(), + }, + "SingleLanguage": { + langs: []string{devv1alpha1.SchemaLanguagePython}, + want: []string{devv1alpha1.SchemaLanguagePython}, + }, + "PreservesAllLanguagesOrder": { + // Filter preserves the order of AllLanguages, not the order + // of the input list. + langs: []string{devv1alpha1.SchemaLanguagePython, devv1alpha1.SchemaLanguageGo}, + want: []string{devv1alpha1.SchemaLanguageGo, devv1alpha1.SchemaLanguagePython}, + }, + "UnknownLanguageIgnored": { + // Filter is permissive; validation happens elsewhere. + langs: []string{devv1alpha1.SchemaLanguagePython, "fortran"}, + want: []string{devv1alpha1.SchemaLanguagePython}, + }, + "AllLanguages": { + langs: devv1alpha1.SupportedSchemaLanguages(), + want: devv1alpha1.SupportedSchemaLanguages(), + }, + } + + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got := Filter(all, tc.langs) + gotLangs := make([]string, len(got)) + for i, g := range got { + gotLangs[i] = g.Language() + } + if diff := cmp.Diff(tc.want, gotLangs); diff != "" { + t.Errorf("Filter(...): -want, +got:\n%s", diff) + } + }) + } +} diff --git a/internal/schemas/generator/json.go b/internal/schemas/generator/json.go index cee0d07..8ddc45a 100644 --- a/internal/schemas/generator/json.go +++ b/internal/schemas/generator/json.go @@ -31,13 +31,14 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/schemas/runner" ) type jsonGenerator struct{} func (jsonGenerator) Language() string { - return "json" + return devv1alpha1.SchemaLanguageJSON } // GenerateFromCRD generates jsonschemas for the CRDs in the given filesystem. diff --git a/internal/schemas/generator/kcl.go b/internal/schemas/generator/kcl.go index 4aa69db..0a3e9cf 100644 --- a/internal/schemas/generator/kcl.go +++ b/internal/schemas/generator/kcl.go @@ -42,6 +42,7 @@ import ( xpv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" xcrd "github.com/crossplane/cli/v2/internal/crd" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/schemas/runner" @@ -56,7 +57,7 @@ const ( type kclGenerator struct{} func (kclGenerator) Language() string { - return "kcl" + return devv1alpha1.SchemaLanguageKCL } // GenerateFromCRD generates KCL schema files from the XRDs and CRDs fromFS. diff --git a/internal/schemas/generator/python.go b/internal/schemas/generator/python.go index c2698be..5f61b4f 100644 --- a/internal/schemas/generator/python.go +++ b/internal/schemas/generator/python.go @@ -37,6 +37,7 @@ import ( xpv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/crd" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/schemas/runner" @@ -55,7 +56,7 @@ var importRE = regexp.MustCompile(`^(from\s+)(\.*)([^\s]+)(.*)`) type pythonGenerator struct{} func (pythonGenerator) Language() string { - return "python" + return devv1alpha1.SchemaLanguagePython } // GenerateFromCRD generates Python schema files from the XRDs and CRDs fromFS. diff --git a/internal/schemas/manager/manager.go b/internal/schemas/manager/manager.go index ef975e7..fd35fe2 100644 --- a/internal/schemas/manager/manager.go +++ b/internal/schemas/manager/manager.go @@ -30,6 +30,7 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/schemas/runner" @@ -143,7 +144,7 @@ func (m *Manager) Generate(ctx context.Context, source Source) (map[string]afero func postProcessForLanguage(language string, langFS afero.Fs) error { switch language { - case "json": + case devv1alpha1.SchemaLanguageJSON: if err := jsonBuildIndexSchema(langFS); err != nil { return errors.Wrap(err, "failed to build index schema for JSON") }