From e5efaeb888cede7857c44a2304d2663eea21230b Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 30 May 2022 16:34:44 -0400 Subject: [PATCH 01/19] offline starter project downloading feature added. Signed-off-by: Michael Valdron --- index/server/pkg/server/endpoint.go | 37 ++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index 3017e5d98..f767d1178 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -131,8 +131,13 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { version := c.Param("version") starterProjectName := c.Param("starterProjectName") downloadTmpLoc := path.Join("/tmp", starterProjectName) + offlineLoc := path.Join("/stacks", starterProjectName) devfileBytes, _ := fetchDevfile(c, devfileName, version) // TODO: add devfileIndex when telemetry is migrated + if version != "default" { + offlineLoc = path.Join(offlineLoc, version) + } + if len(devfileBytes) == 0 { // fetchDevfile was unsuccessful (error or not found) return @@ -194,14 +199,30 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { return } } else if starterProject.Zip != nil { - downloadBytes, err = libutil.DownloadStackFromZipUrl(starterProject.Zip.Location, starterProject.SubDir, downloadTmpLoc) - if err != nil { - log.Print(err.Error()) - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - "status": fmt.Sprintf("Problem with downloading starter project %s", starterProjectName), - }) - return + if _, err = url.ParseRequestURI(starterProject.Zip.Location); err != nil { + localLoc := path.Join(offlineLoc, starterProject.Zip.Location) + log.Printf("zip location is not a valid http url: %v\nTrying local path %s..", err, localLoc) + + downloadBytes, err = ioutil.ReadFile(localLoc) + if err != nil { + log.Print(err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": fmt.Sprintf("Problem with reading starter project %s at %s", starterProjectName, + localLoc), + }) + return + } + } else { + downloadBytes, err = libutil.DownloadStackFromZipUrl(starterProject.Zip.Location, starterProject.SubDir, downloadTmpLoc) + if err != nil { + log.Print(err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": fmt.Sprintf("Problem with downloading starter project %s", starterProjectName), + }) + return + } } } else { c.JSON(http.StatusBadRequest, gin.H{ From 4238b74340d08da02bc99e1e165b225b3b68185f Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 2 May 2022 11:02:20 -0400 Subject: [PATCH 02/19] Endpoint fixups: - Removed duplicate import of indexSchema and fixed line effecting it - Error pulling from OCI Registry now returns error 500 and gets reported Signed-off-by: Michael Valdron --- index/server/pkg/server/endpoint.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index f767d1178..0626126ce 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -16,7 +16,6 @@ import ( "github.com/devfile/library/pkg/devfile/parser" "github.com/devfile/library/pkg/devfile/parser/data/v2/common" libutil "github.com/devfile/registry-support/index/generator/library" - "github.com/devfile/registry-support/index/generator/schema" indexSchema "github.com/devfile/registry-support/index/generator/schema" "github.com/devfile/registry-support/index/server/pkg/util" "github.com/gin-gonic/gin" @@ -174,7 +173,7 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { } if starterProject := starterProjects[0]; starterProject.Git != nil { - gitScheme := schema.Git{ + gitScheme := indexSchema.Git{ Remotes: starterProject.Git.Remotes, RemoteName: "origin", SubDir: starterProject.SubDir, @@ -489,6 +488,13 @@ func fetchDevfile(c *gin.Context, name string, version string) ([]byte, indexSch if foundVersion, ok := versionMap[version]; ok { if devfileIndex.Type == indexSchema.StackDevfileType { bytes, err = pullStackFromRegistry(foundVersion) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": fmt.Sprintf("Problem pulling version %s from OCI Registry", foundVersion.Version), + }) + return []byte{}, indexSchema.Schema{} + } } else { // Retrieve the sample devfile stored under /registry/samples/ sampleDevfilePath = path.Join(samplesPath, devfileIndex.Name, foundVersion.Version, devfileName) From 457c4528d246c61c779cef4ee98897763f004457 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 1 Jun 2022 17:38:03 -0400 Subject: [PATCH 03/19] offline starter project root path fixed. Signed-off-by: Michael Valdron --- index/server/pkg/server/endpoint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index 0626126ce..a0b55bad9 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -130,7 +130,7 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { version := c.Param("version") starterProjectName := c.Param("starterProjectName") downloadTmpLoc := path.Join("/tmp", starterProjectName) - offlineLoc := path.Join("/stacks", starterProjectName) + offlineLoc := path.Join("/registry/stacks", devfileName) devfileBytes, _ := fetchDevfile(c, devfileName, version) // TODO: add devfileIndex when telemetry is migrated if version != "default" { From 4817e695beaf3447f56fa05fb83f171dcaf1dfa0 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 1 Jun 2022 17:38:58 -0400 Subject: [PATCH 04/19] offline start project archives ignored from being archived. Signed-off-by: Michael Valdron --- build-tools/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-tools/build.sh b/build-tools/build.sh index 69098f8bf..1ac720f87 100755 --- a/build-tools/build.sh +++ b/build-tools/build.sh @@ -30,7 +30,8 @@ tar_files_and_cleanup() { -a -not -name "*.vsx" \ -a -not -name "." \ -a -not -name "logo.svg" \ - -a -not -name "logo.png" \) -maxdepth 1) + -a -not -name "logo.png" \ + -a -not -name "*.zip"\) -maxdepth 1) # There are files that need to be pulled into a tar archive if [[ ! -z $tarFiles ]]; then From e8da44eb4745753849bcc8079e814df3803a1113 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Thu, 2 Jun 2022 19:28:41 -0400 Subject: [PATCH 05/19] pushStackToRegistry ignores pushing zip archives into OCI registry Signed-off-by: Michael Valdron --- index/server/pkg/server/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index/server/pkg/server/registry.go b/index/server/pkg/server/registry.go index ffbfe7dbb..0e0e97d29 100644 --- a/index/server/pkg/server/registry.go +++ b/index/server/pkg/server/registry.go @@ -24,8 +24,8 @@ func pushStackToRegistry(versionComponent indexSchema.Version, stackName string) memoryStore := content.NewMemoryStore() pushContents := []ocispec.Descriptor{} for _, resource := range versionComponent.Resources { - if resource == "meta.yaml" { - // Some registries may still have the meta.yaml in it, but we don't need it, so skip pushing it up + if fileExtension := filepath.Ext(resource); resource == "meta.yaml" || fileExtension == ".zip" { + // Some registries may still have the meta.yaml (we don't need it) or offline resources in it, so skip pushing these up continue } From c13b0a556c85db00fc553e6e4d35cb205e773002 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Thu, 2 Jun 2022 19:29:19 -0400 Subject: [PATCH 06/19] variable naming fixups. Signed-off-by: Michael Valdron --- index/server/pkg/server/endpoint.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index a0b55bad9..463292c5f 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -130,11 +130,11 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { version := c.Param("version") starterProjectName := c.Param("starterProjectName") downloadTmpLoc := path.Join("/tmp", starterProjectName) - offlineLoc := path.Join("/registry/stacks", devfileName) + stackLoc := path.Join(stacksPath, devfileName) devfileBytes, _ := fetchDevfile(c, devfileName, version) // TODO: add devfileIndex when telemetry is migrated if version != "default" { - offlineLoc = path.Join(offlineLoc, version) + stackLoc = path.Join(stackLoc, version) } if len(devfileBytes) == 0 { @@ -199,7 +199,7 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { } } else if starterProject.Zip != nil { if _, err = url.ParseRequestURI(starterProject.Zip.Location); err != nil { - localLoc := path.Join(offlineLoc, starterProject.Zip.Location) + localLoc := path.Join(stackLoc, starterProject.Zip.Location) log.Printf("zip location is not a valid http url: %v\nTrying local path %s..", err, localLoc) downloadBytes, err = ioutil.ReadFile(localLoc) From 44c73fae0641ef7fe4a958cc79a2f8ecb5de6c50 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 3 Jun 2022 18:06:30 -0400 Subject: [PATCH 07/19] changes to endpoint.go: - makeVersionMap function created - versionMap creation source in fetchDevfile replaced by makeVersionMap invoke - fetchDevfile invoke under download starter projects endpoint now returns devfile index schema - version injection in offline stack path fixed Signed-off-by: Michael Valdron --- index/server/pkg/server/endpoint.go | 87 +++++++++++++++++------------ 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index 463292c5f..0addc8471 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -131,10 +131,20 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { starterProjectName := c.Param("starterProjectName") downloadTmpLoc := path.Join("/tmp", starterProjectName) stackLoc := path.Join(stacksPath, devfileName) - devfileBytes, _ := fetchDevfile(c, devfileName, version) // TODO: add devfileIndex when telemetry is migrated + devfileBytes, devfileIndex := fetchDevfile(c, devfileName, version) - if version != "default" { - stackLoc = path.Join(stackLoc, version) + if len(devfileIndex.Versions) != 0 { + versionMap, err := makeVersionMap(devfileIndex) + if err != nil { + log.Print(err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": "failed to parse the stack version", + }) + return + } + + stackLoc = path.Join(stackLoc, versionMap[version].Version) } if len(devfileBytes) == 0 { @@ -383,6 +393,35 @@ func buildIndexAPIResponse(c *gin.Context, wantV1Index bool) { } } +// makeVersionMap creates a map of versions for a given devfile index schema. +func makeVersionMap(devfileIndex indexSchema.Schema) (map[string]indexSchema.Version, error) { + versionMap := make(map[string]indexSchema.Version) + var latestVersion string + for _, versionElement := range devfileIndex.Versions { + versionMap[versionElement.Version] = versionElement + if versionElement.Default { + versionMap["default"] = versionElement + } + if latestVersion != "" { + latest, err := versionpkg.NewVersion(latestVersion) + if err != nil { + return map[string]indexSchema.Version{}, err + } + current, err := versionpkg.NewVersion(versionElement.Version) + if err != nil { + return map[string]indexSchema.Version{}, err + } + if current.GreaterThan(latest) { + latestVersion = versionElement.Version + } + } else { + latestVersion = versionElement.Version + } + } + versionMap["latest"] = versionMap[latestVersion] + return versionMap, nil +} + // fetchDevfile retrieves a specified devfile by fetching stacks from the OCI // registry and samples from the `samplesPath` given by server. Also retrieves index // schema from `indexPath` given by server. @@ -450,41 +489,15 @@ func fetchDevfile(c *gin.Context, name string, version string) ([]byte, indexSch sampleDevfilePath = path.Join(samplesPath, devfileIndex.Name, devfileName) } } else { - versionMap := make(map[string]indexSchema.Version) - var latestVersion string - for _, versionElement := range devfileIndex.Versions { - versionMap[versionElement.Version] = versionElement - if versionElement.Default { - versionMap["default"] = versionElement - } - if latestVersion != "" { - latest, err := versionpkg.NewVersion(latestVersion) - if err != nil { - log.Print(err.Error()) - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - "status": fmt.Sprintf("failed to parse the stack version %s for stack %s", latestVersion, name), - }) - return []byte{}, indexSchema.Schema{} - } - current, err := versionpkg.NewVersion(versionElement.Version) - if err != nil { - log.Print(err.Error()) - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - "status": fmt.Sprintf("failed to parse the stack version %s for stack %s", versionElement.Version, name), - }) - return []byte{}, indexSchema.Schema{} - } - if current.GreaterThan(latest) { - latestVersion = versionElement.Version - } - } else { - latestVersion = versionElement.Version - } + versionMap, err := makeVersionMap(devfileIndex) + if err != nil { + log.Print(err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": "failed to parse the stack version", + }) + return []byte{}, indexSchema.Schema{} } - versionMap["latest"] = versionMap[latestVersion] - if foundVersion, ok := versionMap[version]; ok { if devfileIndex.Type == indexSchema.StackDevfileType { bytes, err = pullStackFromRegistry(foundVersion) From 38163489d405d0742954cd00c4ef85779c905c6e Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 3 Jun 2022 18:11:46 -0400 Subject: [PATCH 08/19] offline 'go-starter' starter project definition added to the go stack devfiles. Signed-off-by: Michael Valdron --- tests/registry/stacks/go/1.1.0/devfile.yaml | 3 +++ tests/registry/stacks/go/1.2.0/devfile.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/registry/stacks/go/1.1.0/devfile.yaml b/tests/registry/stacks/go/1.1.0/devfile.yaml index 001d47077..48420d106 100644 --- a/tests/registry/stacks/go/1.1.0/devfile.yaml +++ b/tests/registry/stacks/go/1.1.0/devfile.yaml @@ -17,6 +17,9 @@ starterProjects: revision: main remotes: origin: https://github.com/devfile-samples/devfile-stack-go.git + - name: go-starter-offline + zip: + location: go-starter.zip components: - container: endpoints: diff --git a/tests/registry/stacks/go/1.2.0/devfile.yaml b/tests/registry/stacks/go/1.2.0/devfile.yaml index a879fa8fd..19d52c43a 100644 --- a/tests/registry/stacks/go/1.2.0/devfile.yaml +++ b/tests/registry/stacks/go/1.2.0/devfile.yaml @@ -17,6 +17,9 @@ starterProjects: revision: main remotes: origin: https://github.com/devfile-samples/devfile-stack-go.git + - name: go-starter-offline + zip: + location: go-starter.zip components: - container: endpoints: From 3ddd74262e791adb124c244663a4efb125f8e0f5 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 3 Jun 2022 18:13:50 -0400 Subject: [PATCH 09/19] starter project downloader script for offline starter project testing added. Signed-off-by: Michael Valdron --- .ci/Dockerfile | 3 ++ build-tools/dl_starter_projects.sh | 83 ++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 build-tools/dl_starter_projects.sh diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 5797963ed..d324529bc 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -21,6 +21,9 @@ COPY build-tools /build-tools COPY index/ /index COPY tests/registry /registry +# Download offline starter projects +RUN bash /build-tools/dl_starter_projects.sh + # Run the registry build tools RUN /build-tools/build.sh /registry /build diff --git a/build-tools/dl_starter_projects.sh b/build-tools/dl_starter_projects.sh new file mode 100644 index 000000000..d297b0d1a --- /dev/null +++ b/build-tools/dl_starter_projects.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Path of stacks directory in the registry +STACKS_DIR=/registry/stacks +# List of starter projects to use offline +OFFLINE_STARTER_PROJECTS=("go-starter") + +# Downloads a starter project from a remote git repository and packages it as a zip archive +# to be used as an offline resource. +download_git_starter_project() { + stack_root=$1 + name=$2 + remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.remotes.origin" $stack_root/devfile.yaml) + + mkdir -p $stack_root/$name + + git clone $remote_url $stack_root/$name + + cd $stack_root/$name && rm -rf ./.git && zip -q $stack_root/$name.zip * .[^.]* && cd - + + rm -rf $stack_root/$name +} + +# Downloads a starter project from a remote zip archive source +# to be used as an offline resource. +download_zip_starter_project() { + stack_root=$1 + name=$2 + remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").zip.location" $stack_root/devfile.yaml) + + curl -L $remote_url -o $stack_root/$name.zip +} + +# Read stacks list +read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')" + +echo "Downloading offline starter projects.." +for starter_project in ${OFFLINE_STARTER_PROJECTS[@]} +do + for stack in ${stacks[@]} + do + stack_root=$STACKS_DIR/$stack + stack_devfile=$stack_root/devfile.yaml + # Read version list for stack + read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')" + if [[ ${#versions[@]} -gt 0 ]] + then + for version in ${versions[@]} + do + stack_root=$STACKS_DIR/$stack/$version + stack_devfile=$stack_root/devfile.yaml + if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] + then + if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.." + download_git_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!" + elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.." + download_zip_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!" + fi + fi + done + elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] + then + if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack}.." + download_git_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack}..done!" + elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack}.." + download_zip_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack}..done!" + fi + fi + done +done +echo "Downloading offline starter projects..done!" From 4eadc6fd7fc25ece8d7ae4ecaf34aeb583e03131 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 3 Jun 2022 18:14:59 -0400 Subject: [PATCH 10/19] download offline starter project test cases added. Signed-off-by: Michael Valdron --- .../pkg/tests/indexserver_tests.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/integration/pkg/tests/indexserver_tests.go b/tests/integration/pkg/tests/indexserver_tests.go index bcf82add0..5dbb74ea2 100644 --- a/tests/integration/pkg/tests/indexserver_tests.go +++ b/tests/integration/pkg/tests/indexserver_tests.go @@ -318,6 +318,40 @@ var _ = ginkgo.Describe("[Verify index server is working properly]", func() { })) }) + ginkgo.It("/devfiles//starter-projects/ endpoint should return an offline zip archive for devfile starter project", func() { + resp, err := http.Get(config.Registry + "/devfiles/go/starter-projects/go-starter-offline") + var bytes []byte + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer resp.Body.Close() + + bytes, err = ioutil.ReadAll(resp.Body) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusAccepted)) + gomega.Expect(bytes).ToNot(gomega.BeEmpty()) + gomega.Expect(bytes).To(gomega.Satisfy(func(file []byte) bool { + return http.DetectContentType(file) == "application/zip" + })) + }) + + ginkgo.It("/devfiles///starter-projects/ endpoint should return an offline zip archive for devfile starter project", func() { + resp, err := http.Get(config.Registry + "/devfiles/go/1.2.0/starter-projects/go-starter-offline") + var bytes []byte + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer resp.Body.Close() + + bytes, err = ioutil.ReadAll(resp.Body) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusAccepted)) + gomega.Expect(bytes).ToNot(gomega.BeEmpty()) + gomega.Expect(bytes).To(gomega.Satisfy(func(file []byte) bool { + return http.DetectContentType(file) == "application/zip" + })) + }) + ginkgo.It("/devfiles//starter-projects/ endpoint should return an error for a devfile that doesn't exist", func() { resp, err := http.Get(config.Registry + "/devfiles/fake-stack/starter-projects/springbootproject") From b93be511fa83b389f7d5d6303e19c5635d6278b1 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 6 Jun 2022 14:52:35 -0400 Subject: [PATCH 11/19] dl_starter_projects.sh takes starter project names as parameters instead of a constant. Signed-off-by: Michael Valdron --- .ci/Dockerfile | 2 +- build-tools/dl_starter_projects.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index d324529bc..8cdf37877 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -22,7 +22,7 @@ COPY index/ /index COPY tests/registry /registry # Download offline starter projects -RUN bash /build-tools/dl_starter_projects.sh +RUN bash /build-tools/dl_starter_projects.sh go-starter # Run the registry build tools RUN /build-tools/build.sh /registry /build diff --git a/build-tools/dl_starter_projects.sh b/build-tools/dl_starter_projects.sh index d297b0d1a..8f1d967f8 100644 --- a/build-tools/dl_starter_projects.sh +++ b/build-tools/dl_starter_projects.sh @@ -1,9 +1,15 @@ #!/bin/bash +if [ -z "$@" ] +then + echo "No starter projects specified." + exit 0 +fi + # Path of stacks directory in the registry STACKS_DIR=/registry/stacks # List of starter projects to use offline -OFFLINE_STARTER_PROJECTS=("go-starter") +offline_starter_projects=( "$@" ) # Downloads a starter project from a remote git repository and packages it as a zip archive # to be used as an offline resource. @@ -35,7 +41,7 @@ download_zip_starter_project() { read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')" echo "Downloading offline starter projects.." -for starter_project in ${OFFLINE_STARTER_PROJECTS[@]} +for starter_project in ${offline_starter_projects[@]} do for stack in ${stacks[@]} do From 1dbcfbeb869eb9cedda6b20076d07c34c38575d0 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 6 Jun 2022 14:59:26 -0400 Subject: [PATCH 12/19] comments added. Signed-off-by: Michael Valdron --- build-tools/dl_starter_projects.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build-tools/dl_starter_projects.sh b/build-tools/dl_starter_projects.sh index 8f1d967f8..809fdfe16 100644 --- a/build-tools/dl_starter_projects.sh +++ b/build-tools/dl_starter_projects.sh @@ -49,19 +49,23 @@ do stack_devfile=$stack_root/devfile.yaml # Read version list for stack read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')" + # If multi version stack if [[ ${#versions[@]} -gt 0 ]] then for version in ${versions[@]} do stack_root=$STACKS_DIR/$stack/$version stack_devfile=$stack_root/devfile.yaml + # If the specified starter project is found if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] then + # Starter project has a git remote if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] then echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.." download_git_starter_project $stack_root $starter_project echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!" + # Starter project has a zip remote elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] then echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.." @@ -70,13 +74,16 @@ do fi fi done + # If not multi version stack & the specified starter project is found elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] then + # Starter project has a git remote if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] then echo "Downloading ${starter_project} starter project in stack ${stack}.." download_git_starter_project $stack_root $starter_project echo "Downloading ${starter_project} starter project in stack ${stack}..done!" + # Starter project has a zip remote elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] then echo "Downloading ${starter_project} starter project in stack ${stack}.." From 8c26747704c8dbd89cacfa9f54129a4772959b43 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Thu, 9 Jun 2022 19:33:21 -0400 Subject: [PATCH 13/19] dl_starter_projects.sh now handles git repos with multiple revisions (branches) and with a subdir defined. Signed-off-by: Michael Valdron --- build-tools/dl_starter_projects.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/build-tools/dl_starter_projects.sh b/build-tools/dl_starter_projects.sh index 809fdfe16..14f38666d 100644 --- a/build-tools/dl_starter_projects.sh +++ b/build-tools/dl_starter_projects.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ -z "$@" ] +if [[ -z "$@" ]] then echo "No starter projects specified." exit 0 @@ -16,13 +16,32 @@ offline_starter_projects=( "$@" ) download_git_starter_project() { stack_root=$1 name=$2 - remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.remotes.origin" $stack_root/devfile.yaml) + remote_name=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.checkoutFrom.remote" $stack_root/devfile.yaml) + revision=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.checkoutFrom.revision" $stack_root/devfile.yaml) + subDir=$(yq e ".starterProjects[] | select(.name == \"${name}\").subDir" $stack_root/devfile.yaml) + + if [ "${remote_name}" == "null" ] + then + remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.remotes.origin" $stack_root/devfile.yaml) + else + remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.remotes.${remote_name}" $stack_root/devfile.yaml) + fi mkdir -p $stack_root/$name git clone $remote_url $stack_root/$name - cd $stack_root/$name && rm -rf ./.git && zip -q $stack_root/$name.zip * .[^.]* && cd - + if [ "${revision}" != "null" ] + then + cd $stack_root/$name && git checkout $revision && cd - + fi + + if [ "${subDir}" != "null" ] + then + cd $stack_root/$name/$subDir && zip -q $stack_root/$name.zip * .[^.]* && cd - + else + cd $stack_root/$name && rm -rf ./.git && zip -q $stack_root/$name.zip * .[^.]* && cd - + fi rm -rf $stack_root/$name } From 1a2a2de7c47de16f5940b68537a8a0f63ab0b640 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Thu, 9 Jun 2022 22:53:51 -0400 Subject: [PATCH 14/19] fixed issue with version handling. Signed-off-by: Michael Valdron --- index/server/pkg/server/endpoint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index 0addc8471..b56754e9d 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -133,7 +133,7 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { stackLoc := path.Join(stacksPath, devfileName) devfileBytes, devfileIndex := fetchDevfile(c, devfileName, version) - if len(devfileIndex.Versions) != 0 { + if len(devfileIndex.Versions) > 1 { versionMap, err := makeVersionMap(devfileIndex) if err != nil { log.Print(err.Error()) From 3f654174cfac7a98b4d7c706d0154954f85e037c Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Thu, 9 Jun 2022 22:54:47 -0400 Subject: [PATCH 15/19] integration test case where the zip location for an offline starter project is invalid added. Signed-off-by: Michael Valdron --- tests/integration/pkg/tests/indexserver_tests.go | 7 +++++++ tests/registry/stacks/java-maven/devfile.yaml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/tests/integration/pkg/tests/indexserver_tests.go b/tests/integration/pkg/tests/indexserver_tests.go index 5dbb74ea2..cb31368d5 100644 --- a/tests/integration/pkg/tests/indexserver_tests.go +++ b/tests/integration/pkg/tests/indexserver_tests.go @@ -352,6 +352,13 @@ var _ = ginkgo.Describe("[Verify index server is working properly]", func() { })) }) + ginkgo.It("/devfiles//starter-projects/ endpoint should return an error for an offline starter project file location that doesn't exist", func() { + resp, err := http.Get(config.Registry + "/devfiles/java-maven/starter-projects/springbootproject-offline") + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusInternalServerError)) + }) + ginkgo.It("/devfiles//starter-projects/ endpoint should return an error for a devfile that doesn't exist", func() { resp, err := http.Get(config.Registry + "/devfiles/fake-stack/starter-projects/springbootproject") diff --git a/tests/registry/stacks/java-maven/devfile.yaml b/tests/registry/stacks/java-maven/devfile.yaml index 76ae4a6af..ee80996bf 100644 --- a/tests/registry/stacks/java-maven/devfile.yaml +++ b/tests/registry/stacks/java-maven/devfile.yaml @@ -12,6 +12,9 @@ metadata: - "arm64" - "s390x" starterProjects: + - name: springbootproject-offline + zip: + location: springbootproject.zip - name: springbootproject git: remotes: From 33639be86b5059f9944f0dac4ce234338326e23e Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 14 Jun 2022 19:31:24 -0400 Subject: [PATCH 16/19] MakeVersionMap function moved to util package & unit test added for MakeVersionMap function. Signed-off-by: Michael Valdron --- index/server/pkg/server/endpoint.go | 34 +------------- index/server/pkg/util/util.go | 31 +++++++++++++ index/server/pkg/util/util_test.go | 69 +++++++++++++++++++++++++++-- 3 files changed, 99 insertions(+), 35 deletions(-) diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index b56754e9d..c30b47137 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -19,7 +19,6 @@ import ( indexSchema "github.com/devfile/registry-support/index/generator/schema" "github.com/devfile/registry-support/index/server/pkg/util" "github.com/gin-gonic/gin" - versionpkg "github.com/hashicorp/go-version" "github.com/prometheus/client_golang/prometheus" "gopkg.in/segmentio/analytics-go.v3" ) @@ -134,7 +133,7 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { devfileBytes, devfileIndex := fetchDevfile(c, devfileName, version) if len(devfileIndex.Versions) > 1 { - versionMap, err := makeVersionMap(devfileIndex) + versionMap, err := util.MakeVersionMap(devfileIndex) if err != nil { log.Print(err.Error()) c.JSON(http.StatusInternalServerError, gin.H{ @@ -393,35 +392,6 @@ func buildIndexAPIResponse(c *gin.Context, wantV1Index bool) { } } -// makeVersionMap creates a map of versions for a given devfile index schema. -func makeVersionMap(devfileIndex indexSchema.Schema) (map[string]indexSchema.Version, error) { - versionMap := make(map[string]indexSchema.Version) - var latestVersion string - for _, versionElement := range devfileIndex.Versions { - versionMap[versionElement.Version] = versionElement - if versionElement.Default { - versionMap["default"] = versionElement - } - if latestVersion != "" { - latest, err := versionpkg.NewVersion(latestVersion) - if err != nil { - return map[string]indexSchema.Version{}, err - } - current, err := versionpkg.NewVersion(versionElement.Version) - if err != nil { - return map[string]indexSchema.Version{}, err - } - if current.GreaterThan(latest) { - latestVersion = versionElement.Version - } - } else { - latestVersion = versionElement.Version - } - } - versionMap["latest"] = versionMap[latestVersion] - return versionMap, nil -} - // fetchDevfile retrieves a specified devfile by fetching stacks from the OCI // registry and samples from the `samplesPath` given by server. Also retrieves index // schema from `indexPath` given by server. @@ -489,7 +459,7 @@ func fetchDevfile(c *gin.Context, name string, version string) ([]byte, indexSch sampleDevfilePath = path.Join(samplesPath, devfileIndex.Name, devfileName) } } else { - versionMap, err := makeVersionMap(devfileIndex) + versionMap, err := util.MakeVersionMap(devfileIndex) if err != nil { log.Print(err.Error()) c.JSON(http.StatusInternalServerError, gin.H{ diff --git a/index/server/pkg/util/util.go b/index/server/pkg/util/util.go index 618b43da1..5048d12ed 100644 --- a/index/server/pkg/util/util.go +++ b/index/server/pkg/util/util.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" + versionpkg "github.com/hashicorp/go-version" + indexLibrary "github.com/devfile/registry-support/index/generator/library" indexSchema "github.com/devfile/registry-support/index/generator/schema" ) @@ -182,3 +184,32 @@ func IsTelemetryEnabled() bool { } return false } + +// MakeVersionMap creates a map of versions for a given devfile index schema. +func MakeVersionMap(devfileIndex indexSchema.Schema) (map[string]indexSchema.Version, error) { + versionMap := make(map[string]indexSchema.Version) + var latestVersion string + for _, versionElement := range devfileIndex.Versions { + versionMap[versionElement.Version] = versionElement + if versionElement.Default { + versionMap["default"] = versionElement + } + if latestVersion != "" { + latest, err := versionpkg.NewVersion(latestVersion) + if err != nil { + return map[string]indexSchema.Version{}, err + } + current, err := versionpkg.NewVersion(versionElement.Version) + if err != nil { + return map[string]indexSchema.Version{}, err + } + if current.GreaterThan(latest) { + latestVersion = versionElement.Version + } + } else { + latestVersion = versionElement.Version + } + } + versionMap["latest"] = versionMap[latestVersion] + return versionMap, nil +} diff --git a/index/server/pkg/util/util_test.go b/index/server/pkg/util/util_test.go index 158ba07fe..959b481be 100644 --- a/index/server/pkg/util/util_test.go +++ b/index/server/pkg/util/util_test.go @@ -2,11 +2,13 @@ package util import ( "encoding/json" - "github.com/devfile/registry-support/index/generator/schema" + "fmt" "io/ioutil" "os" "reflect" "testing" + + indexSchema "github.com/devfile/registry-support/index/generator/schema" ) func TestIsHtmlRequested(t *testing.T) { @@ -206,12 +208,12 @@ func TestConvertToOldIndexFormat(t *testing.T) { if err != nil { t.Errorf("Failed to oldIndexStruct.json: %v", err) } - var inputIndex []schema.Schema + var inputIndex []indexSchema.Schema err = json.Unmarshal(bytes, &inputIndex) if err != nil { t.Errorf("Failed to unmarshal inputIndex json") } - var wantIndex []schema.Schema + var wantIndex []indexSchema.Schema err = json.Unmarshal(expected, &wantIndex) if err != nil { t.Errorf("Failed to unmarshal wantIndex json") @@ -225,3 +227,64 @@ func TestConvertToOldIndexFormat(t *testing.T) { } }) } + +func TestMakeVersionMap(t *testing.T) { + devfileIndex := indexSchema.Schema{ + Name: "Test Devfile", + Version: "2.2.0", + Attributes: nil, + DisplayName: "", + Description: "", + Type: "", + Tags: []string{}, + Architectures: []string{}, + Icon: "", + GlobalMemoryLimit: "", + ProjectType: "", + Language: "", + Links: map[string]string{}, + Resources: []string{}, + StarterProjects: []string{}, + Git: &indexSchema.Git{}, + Provider: "", + SupportUrl: "", + Versions: []indexSchema.Version{ + { + Version: "1.1.0", + Default: true, + }, + {Version: "1.2.0"}, + }, + } + tests := []struct { + key string + wantVal string + }{ + { + key: "default", + wantVal: "1.1.0", + }, + { + key: "latest", + wantVal: "1.2.0", + }, + { + key: "1.1.0", + wantVal: "1.1.0", + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("Test generate version map with key %s", test.key), func(t *testing.T) { + versionMap, err := MakeVersionMap(devfileIndex) + if err != nil { + t.Errorf("Was not expecting error with MakeVersionMap: %v", err) + } + + if !reflect.DeepEqual(test.wantVal, versionMap[test.key].Version) { + t.Errorf("Was expecting '%s' to map to '%s' not '%s'", + test.key, test.wantVal, versionMap[test.key].Version) + } + }) + } +} From 3439b2fc54ee43cae295359b3a359ee0d15d9857 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 15 Jun 2022 14:25:59 -0400 Subject: [PATCH 17/19] only push zip archives with -offline suffix. Signed-off-by: Michael Valdron --- build-tools/dl_starter_projects.sh | 16 +++++++++------- index/server/pkg/server/registry.go | 3 ++- tests/registry/stacks/go/1.1.0/devfile.yaml | 2 +- tests/registry/stacks/go/1.2.0/devfile.yaml | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/build-tools/dl_starter_projects.sh b/build-tools/dl_starter_projects.sh index 14f38666d..a21301cca 100644 --- a/build-tools/dl_starter_projects.sh +++ b/build-tools/dl_starter_projects.sh @@ -19,6 +19,7 @@ download_git_starter_project() { remote_name=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.checkoutFrom.remote" $stack_root/devfile.yaml) revision=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.checkoutFrom.revision" $stack_root/devfile.yaml) subDir=$(yq e ".starterProjects[] | select(.name == \"${name}\").subDir" $stack_root/devfile.yaml) + local_path=${stack_root}/${name}-offline if [ "${remote_name}" == "null" ] then @@ -27,23 +28,23 @@ download_git_starter_project() { remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.remotes.${remote_name}" $stack_root/devfile.yaml) fi - mkdir -p $stack_root/$name + mkdir -p $local_path - git clone $remote_url $stack_root/$name + git clone $remote_url $local_path if [ "${revision}" != "null" ] then - cd $stack_root/$name && git checkout $revision && cd - + cd $local_path && git checkout $revision && cd - fi if [ "${subDir}" != "null" ] then - cd $stack_root/$name/$subDir && zip -q $stack_root/$name.zip * .[^.]* && cd - + cd $local_path/$subDir && zip -q ${local_path}.zip * .[^.]* && cd - else - cd $stack_root/$name && rm -rf ./.git && zip -q $stack_root/$name.zip * .[^.]* && cd - + cd $local_path && rm -rf ./.git && zip -q ${local_path}.zip * .[^.]* && cd - fi - rm -rf $stack_root/$name + rm -rf $local_path } # Downloads a starter project from a remote zip archive source @@ -52,8 +53,9 @@ download_zip_starter_project() { stack_root=$1 name=$2 remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").zip.location" $stack_root/devfile.yaml) + local_path=${stack_root}/${name}-offline - curl -L $remote_url -o $stack_root/$name.zip + curl -L $remote_url -o ${local_path}.zip } # Read stacks list diff --git a/index/server/pkg/server/registry.go b/index/server/pkg/server/registry.go index 0e0e97d29..718440f8f 100644 --- a/index/server/pkg/server/registry.go +++ b/index/server/pkg/server/registry.go @@ -9,6 +9,7 @@ import ( "os" "path" "path/filepath" + "strings" indexSchema "github.com/devfile/registry-support/index/generator/schema" @@ -24,7 +25,7 @@ func pushStackToRegistry(versionComponent indexSchema.Version, stackName string) memoryStore := content.NewMemoryStore() pushContents := []ocispec.Descriptor{} for _, resource := range versionComponent.Resources { - if fileExtension := filepath.Ext(resource); resource == "meta.yaml" || fileExtension == ".zip" { + if resource == "meta.yaml" || strings.HasSuffix(resource, "-offline.zip") { // Some registries may still have the meta.yaml (we don't need it) or offline resources in it, so skip pushing these up continue } diff --git a/tests/registry/stacks/go/1.1.0/devfile.yaml b/tests/registry/stacks/go/1.1.0/devfile.yaml index 48420d106..d4726d07f 100644 --- a/tests/registry/stacks/go/1.1.0/devfile.yaml +++ b/tests/registry/stacks/go/1.1.0/devfile.yaml @@ -19,7 +19,7 @@ starterProjects: origin: https://github.com/devfile-samples/devfile-stack-go.git - name: go-starter-offline zip: - location: go-starter.zip + location: go-starter-offline.zip components: - container: endpoints: diff --git a/tests/registry/stacks/go/1.2.0/devfile.yaml b/tests/registry/stacks/go/1.2.0/devfile.yaml index 19d52c43a..4cb263fd6 100644 --- a/tests/registry/stacks/go/1.2.0/devfile.yaml +++ b/tests/registry/stacks/go/1.2.0/devfile.yaml @@ -19,7 +19,7 @@ starterProjects: origin: https://github.com/devfile-samples/devfile-stack-go.git - name: go-starter-offline zip: - location: go-starter.zip + location: go-starter-offline.zip components: - container: endpoints: From 9f8ae9ef3dcf83fdc002f39a4b7187cfd782ac0c Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 15 Jun 2022 16:35:54 -0400 Subject: [PATCH 18/19] MakeVersionMap extra test cases added including bad version test. Signed-off-by: Michael Valdron --- index/server/pkg/util/util_test.go | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/index/server/pkg/util/util_test.go b/index/server/pkg/util/util_test.go index 959b481be..14c8f927b 100644 --- a/index/server/pkg/util/util_test.go +++ b/index/server/pkg/util/util_test.go @@ -272,6 +272,14 @@ func TestMakeVersionMap(t *testing.T) { key: "1.1.0", wantVal: "1.1.0", }, + { + key: "", + wantVal: "", + }, + { + key: "1.3.0", + wantVal: "", + }, } for _, test := range tests { @@ -288,3 +296,36 @@ func TestMakeVersionMap(t *testing.T) { }) } } + +func TestMakeVersionMapOnBadVersion(t *testing.T) { + devfileIndex := indexSchema.Schema{ + Name: "Test Devfile", + Version: "2.2.0", + Attributes: nil, + DisplayName: "", + Description: "", + Type: "", + Tags: []string{}, + Architectures: []string{}, + Icon: "", + GlobalMemoryLimit: "", + ProjectType: "", + Language: "", + Links: map[string]string{}, + Resources: []string{}, + StarterProjects: []string{}, + Git: &indexSchema.Git{}, + Provider: "", + SupportUrl: "", + Versions: []indexSchema.Version{ + {Version: "fsdf-sf3v.-dfg"}, + {Version: "erdgf-v.-dd-,.fdgg"}, + }, + } + t.Run("Test generate version map with bad versioning", func(t *testing.T) { + _, err := MakeVersionMap(devfileIndex) + if err == nil { + t.Error("Was expecting malformed version error with MakeVersionMap") + } + }) +} From 26073891495e4e90b01fea582dca9252f0669f98 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 15 Jun 2022 17:35:18 -0400 Subject: [PATCH 19/19] offline subdir handling added with testing. Signed-off-by: Michael Valdron --- .ci/Dockerfile | 2 +- index/server/pkg/server/endpoint.go | 49 +++++++++++++++++++ .../pkg/tests/indexserver_tests.go | 17 +++++++ .../registry/stacks/java-quarkus/devfile.yaml | 4 ++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 8cdf37877..47e601c29 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -22,7 +22,7 @@ COPY index/ /index COPY tests/registry /registry # Download offline starter projects -RUN bash /build-tools/dl_starter_projects.sh go-starter +RUN bash /build-tools/dl_starter_projects.sh go-starter community # Run the registry build tools RUN /build-tools/build.sh /registry /build diff --git a/index/server/pkg/server/endpoint.go b/index/server/pkg/server/endpoint.go index c30b47137..2467f294c 100644 --- a/index/server/pkg/server/endpoint.go +++ b/index/server/pkg/server/endpoint.go @@ -15,6 +15,7 @@ import ( "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" "github.com/devfile/library/pkg/devfile/parser" "github.com/devfile/library/pkg/devfile/parser/data/v2/common" + dfutil "github.com/devfile/library/pkg/util" libutil "github.com/devfile/registry-support/index/generator/library" indexSchema "github.com/devfile/registry-support/index/generator/schema" "github.com/devfile/registry-support/index/server/pkg/util" @@ -211,6 +212,54 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) { localLoc := path.Join(stackLoc, starterProject.Zip.Location) log.Printf("zip location is not a valid http url: %v\nTrying local path %s..", err, localLoc) + // If subdirectory is specified for starter project download then extract subdirectory + // and create new archive for download. + if starterProject.SubDir != "" { + downloadFilePath := fmt.Sprintf("%s.zip", downloadTmpLoc) + + if _, err = os.Stat(downloadTmpLoc); os.IsExist(err) { + err = os.Remove(downloadTmpLoc) + if err != nil { + log.Print(err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": fmt.Sprintf("Problem removing existing temporary download directory '%s' for starter project %s", + downloadTmpLoc, + starterProjectName), + }) + return + } + } + + _, err = dfutil.Unzip(localLoc, downloadTmpLoc, starterProject.SubDir) + if err != nil { + log.Print(err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": fmt.Sprintf("Problem with reading subDir '%s' of starter project %s at %s", + starterProject.SubDir, + starterProjectName, + localLoc), + }) + return + } + + err = libutil.ZipDir(downloadTmpLoc, downloadFilePath) + if err != nil { + log.Print(err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + "status": fmt.Sprintf("Problem with archiving subDir '%s' of starter project %s at %s", + starterProject.SubDir, + starterProjectName, + downloadFilePath), + }) + return + } + + localLoc = downloadFilePath + } + downloadBytes, err = ioutil.ReadFile(localLoc) if err != nil { log.Print(err.Error()) diff --git a/tests/integration/pkg/tests/indexserver_tests.go b/tests/integration/pkg/tests/indexserver_tests.go index cb31368d5..4b4c2e582 100644 --- a/tests/integration/pkg/tests/indexserver_tests.go +++ b/tests/integration/pkg/tests/indexserver_tests.go @@ -352,6 +352,23 @@ var _ = ginkgo.Describe("[Verify index server is working properly]", func() { })) }) + ginkgo.It("/devfiles//starter-projects/ endpoint should return an offline zip archive of a subdir for devfile starter project", func() { + resp, err := http.Get(config.Registry + "/devfiles/java-quarkus/starter-projects/community-offline") + var bytes []byte + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer resp.Body.Close() + + bytes, err = ioutil.ReadAll(resp.Body) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusAccepted)) + gomega.Expect(bytes).ToNot(gomega.BeEmpty()) + gomega.Expect(bytes).To(gomega.Satisfy(func(file []byte) bool { + return http.DetectContentType(file) == "application/zip" + })) + }) + ginkgo.It("/devfiles//starter-projects/ endpoint should return an error for an offline starter project file location that doesn't exist", func() { resp, err := http.Get(config.Registry + "/devfiles/java-maven/starter-projects/springbootproject-offline") diff --git a/tests/registry/stacks/java-quarkus/devfile.yaml b/tests/registry/stacks/java-quarkus/devfile.yaml index 62c29c557..786562b6e 100644 --- a/tests/registry/stacks/java-quarkus/devfile.yaml +++ b/tests/registry/stacks/java-quarkus/devfile.yaml @@ -11,6 +11,10 @@ metadata: architectures: - "amd64" starterProjects: + - name: community-offline + zip: + location: community-offline.zip + subDir: src/main/java/org/acme - name: community zip: location: https://code.quarkus.io/d?e=io.quarkus%3Aquarkus-resteasy&e=io.quarkus%3Aquarkus-micrometer&e=io.quarkus%3Aquarkus-smallrye-health&e=io.quarkus%3Aquarkus-openshift&cn=devfile