From 6fe63a087eca07b8e9b7822650d36bc032819bc4 Mon Sep 17 00:00:00 2001 From: Dolapo Toki Date: Thu, 13 Aug 2020 16:21:37 -0700 Subject: [PATCH 1/3] wip: copy bin file type src -> dest --- go.mod | 1 + go.sum | 2 ++ internal/generate/generate_modules.go | 48 ++++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 3ee19610c..751e8ca33 100644 --- a/go.mod +++ b/go.mod @@ -28,5 +28,6 @@ require ( golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.2.2 + github.com/gabriel-vasile/mimetype v1.1.1 ) diff --git a/go.sum b/go.sum index 3998c6c96..88cf6e659 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gabriel-vasile/mimetype v1.1.1 h1:qbN9MPuRf3bstHu9zkI9jDWNfH//9+9kHxr9oRBBBOA= +github.com/gabriel-vasile/mimetype v1.1.1/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= diff --git a/internal/generate/generate_modules.go b/internal/generate/generate_modules.go index f688cd557..8e9583ff1 100644 --- a/internal/generate/generate_modules.go +++ b/internal/generate/generate_modules.go @@ -15,8 +15,9 @@ import ( "github.com/commitdev/zero/internal/module" "github.com/commitdev/zero/internal/util" "github.com/commitdev/zero/pkg/util/flog" - "github.com/commitdev/zero/pkg/util/fs" + + "github.com/gabriel-vasile/mimetype" ) // Generate accepts a projectconfig struct and renders the templates for all referenced modules @@ -52,7 +53,15 @@ func Generate(projectConfig projectconfig.ZeroProjectConfig) error { mod.Parameters, } - fileTemplates := newTemplates(moduleDir, outputDir, false) + fileTemplates, binFiles := newTemplates(moduleDir, outputDir, false) + + //FIXME: remove me; debug log; + // for _, binFile := range binFiles { + // fmt.Printf("%s\n", "Bin files to just copy over:") + // fmt.Printf("%+v\n", binFile) + // } + + // TODO: function to copy binFile src -> dest. executeTemplates(fileTemplates, templateData, delimiters) } @@ -65,9 +74,17 @@ type TemplateConfig struct { isTemplate bool } +type BinFileConfig struct { + source string + destination string + mineType string +} + +// FIXME: refactor function name // newTemplates walks the module directory to find all to be templated -func newTemplates(moduleDir string, outputDir string, overwrite bool) []*TemplateConfig { +func newTemplates(moduleDir string, outputDir string, overwrite bool) ([]*TemplateConfig, []*BinFileConfig) { templates := []*TemplateConfig{} + binFile := []*BinFileConfig{} paths, err := getAllFilePathsInDirectory(moduleDir) if err != nil { @@ -80,6 +97,29 @@ func newTemplates(moduleDir string, outputDir string, overwrite bool) []*Templat continue } + // detect the file type + detectedMIME, err := mimetype.DetectFile(path) + if err != nil { + panic(err) + } + + // detect if the file type is binary + isBinary := true + for mime := detectedMIME; mime != nil; mime = mime.Parent() { + if mime.Is("text/plain") { + isBinary = false + } + } + + if isBinary { + binFile = append(binFile, &BinFileConfig{ + source: path, + destination: fs.ReplacePath(path, moduleDir, outputDir), + mineType: detectedMIME.String(), + }) + continue + } + _, file := filepath.Split(path) hasTmpltSuffix := strings.HasSuffix(file, constants.TemplateExtn) if hasTmpltSuffix { @@ -100,7 +140,7 @@ func newTemplates(moduleDir string, outputDir string, overwrite bool) []*Templat isTemplate: hasTmpltSuffix, }) } - return templates + return templates, binFile } // getAllFilePathsInDirectory Recursively get all file paths in directory, including sub-directories. From 749735727635abe313750c5436d52bd4f2b34f87 Mon Sep 17 00:00:00 2001 From: Dolapo Toki Date: Thu, 13 Aug 2020 18:17:11 -0700 Subject: [PATCH 2/3] refactor; copy bin file from src -> dest --- internal/generate/generate_modules.go | 108 +++++++++++++++----------- 1 file changed, 62 insertions(+), 46 deletions(-) diff --git a/internal/generate/generate_modules.go b/internal/generate/generate_modules.go index 8e9583ff1..29e60a2a4 100644 --- a/internal/generate/generate_modules.go +++ b/internal/generate/generate_modules.go @@ -2,11 +2,12 @@ package generate import ( "fmt" + "io" + "log" "os" "path" "path/filepath" "regexp" - "strings" "sync" "text/template" @@ -53,38 +54,23 @@ func Generate(projectConfig projectconfig.ZeroProjectConfig) error { mod.Parameters, } - fileTemplates, binFiles := newTemplates(moduleDir, outputDir, false) + txtTypeFiles, binTypeFiles := sortFileType(moduleDir, outputDir, false) - //FIXME: remove me; debug log; - // for _, binFile := range binFiles { - // fmt.Printf("%s\n", "Bin files to just copy over:") - // fmt.Printf("%+v\n", binFile) - // } - - // TODO: function to copy binFile src -> dest. - - executeTemplates(fileTemplates, templateData, delimiters) + executeTemplates(txtTypeFiles, templateData, delimiters) + copyBinFiles(binTypeFiles) } return nil } -type TemplateConfig struct { - source string - destination string - isTemplate bool -} - -type BinFileConfig struct { +type fileConfig struct { source string destination string - mineType string } -// FIXME: refactor function name -// newTemplates walks the module directory to find all to be templated -func newTemplates(moduleDir string, outputDir string, overwrite bool) ([]*TemplateConfig, []*BinFileConfig) { - templates := []*TemplateConfig{} - binFile := []*BinFileConfig{} +// sortFileType walks the module directory to find and classify all files into bin / text/plain (non-bin) types. +func sortFileType(moduleDir string, outputDir string, overwrite bool) ([]*fileConfig, []*fileConfig) { + binTypeFiles := []*fileConfig{} + txtTypeFiles := []*fileConfig{} paths, err := getAllFilePathsInDirectory(moduleDir) if err != nil { @@ -97,13 +83,22 @@ func newTemplates(moduleDir string, outputDir string, overwrite bool) ([]*Templa continue } + outputPath := fs.ReplacePath(path, moduleDir, outputDir) + + if !overwrite { + if exists, _ := fs.FileExists(outputPath); exists { + flog.Warnf("%v already exists. skipping.", outputPath) + continue + } + } + // detect the file type detectedMIME, err := mimetype.DetectFile(path) if err != nil { panic(err) } - // detect if the file type is binary + // detect root file type isBinary := true for mime := detectedMIME; mime != nil; mime = mime.Parent() { if mime.Is("text/plain") { @@ -112,35 +107,19 @@ func newTemplates(moduleDir string, outputDir string, overwrite bool) ([]*Templa } if isBinary { - binFile = append(binFile, &BinFileConfig{ + binTypeFiles = append(binTypeFiles, &fileConfig{ source: path, - destination: fs.ReplacePath(path, moduleDir, outputDir), - mineType: detectedMIME.String(), + destination: outputPath, }) continue } - _, file := filepath.Split(path) - hasTmpltSuffix := strings.HasSuffix(file, constants.TemplateExtn) - if hasTmpltSuffix { - file = strings.Replace(file, constants.TemplateExtn, "", -1) - } - outputPath := fs.ReplacePath(path, moduleDir, outputDir) - - if !overwrite { - if exists, _ := fs.FileExists(outputPath); exists { - flog.Warnf("%v already exists. skipping.", outputPath) - continue - } - } - - templates = append(templates, &TemplateConfig{ + txtTypeFiles = append(txtTypeFiles, &fileConfig{ source: path, destination: outputPath, - isTemplate: hasTmpltSuffix, }) } - return templates, binFile + return txtTypeFiles, binTypeFiles } // getAllFilePathsInDirectory Recursively get all file paths in directory, including sub-directories. @@ -162,7 +141,7 @@ func getAllFilePathsInDirectory(moduleDir string) ([]string, error) { return paths, nil } -func executeTemplates(templates []*TemplateConfig, data interface{}, delimiters []string) { +func executeTemplates(templates []*fileConfig, data interface{}, delimiters []string) { var wg sync.WaitGroup leftDelim := delimiters[0] rightDelim := delimiters[1] @@ -202,3 +181,40 @@ func executeTemplates(templates []*TemplateConfig, data interface{}, delimiters wg.Wait() } + +func copyBinFiles(binTypeFiles []*fileConfig) { + + for _, binFile := range binTypeFiles { + source := binFile.source + dest := binFile.destination + + // create dir + outputDirPath, _ := path.Split(dest) + err := fs.CreateDirs(outputDirPath) + if err != nil { + flog.Errorf("Error creating directory '%s': %v", source, err) + } + + // create refs to src and dest + from, err := os.Open(source) + if err != nil { + flog.Errorf("Error opening file to read '%s' : %v", source, err) + } + defer from.Close() + + to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + log.Fatal(err) + flog.Errorf("Error creating file '%s': %v", dest, err) + } + defer to.Close() + + // copy file + _, err = io.Copy(to, from) + if err != nil { + flog.Errorf("Error copying file '%s' : %v", source, err) + } else { + flog.Successf("Finished copying file : %s", dest) + } + } +} From 0939621969f986e157c66e6e5c299b9b723482a9 Mon Sep 17 00:00:00 2001 From: Dolapo Toki Date: Thu, 13 Aug 2020 18:39:01 -0700 Subject: [PATCH 3/3] update registry --- README.md | 2 +- internal/registry/registry.go | 3 ++- internal/registry/registry_test.go | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 657591ffd..596c29970 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ $ zero init ## Sample project initilization ✔ Project Name: myapp-infra 🎉 Initializing project -✔ EKS + Go + React +✔ EKS + Go + React + Gatsby ✔ Should the created projects be checked into github automatically? (y/n): y ✔ What's the root of the github org to create repositories in?: github.com/myapp-org ✔ Existing AWS Profiles diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 02a304a52..0b2b904d1 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -10,9 +10,10 @@ func GetRegistry() Registry { return Registry{ // TODO: better place to store these options as configuration file or any source { - "EKS + Go + React", + "EKS + Go + React + Gatsby", []string{ "github.com/commitdev/zero-aws-eks-stack", + "github.com/commitdev/zero-deployable-landing-page", "github.com/commitdev/zero-deployable-backend", "github.com/commitdev/zero-deployable-react-frontend", }, diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go index 4fd8b1bef..6d28016cd 100644 --- a/internal/registry/registry_test.go +++ b/internal/registry/registry_test.go @@ -13,7 +13,7 @@ func TestAvailableLabels(t *testing.T) { t.Run("should be same order as declared", func(t *testing.T) { labels := registry.AvailableLabels(reg) assert.Equal(t, labels, []string{ - "EKS + Go + React", + "EKS + Go + React + Gatsby", "foo", "bar", "lorem", @@ -27,7 +27,7 @@ func TestGetModulesByName(t *testing.T) { reg := testRegistry() t.Run("should return modules of specified stack", func(t *testing.T) { - assert.Equal(t, registry.GetModulesByName(reg, "EKS + Go + React"), + assert.Equal(t, registry.GetModulesByName(reg, "EKS + Go + React + Gatsby"), []string{"module-source 1", "module-source 2"}) assert.Equal(t, registry.GetModulesByName(reg, "lorem"), []string{"module-source 5"}) assert.Equal(t, registry.GetModulesByName(reg, "ipsum"), []string{"module-source 6"}) @@ -37,7 +37,7 @@ func TestGetModulesByName(t *testing.T) { func testRegistry() registry.Registry { return registry.Registry{ - {"EKS + Go + React", []string{"module-source 1", "module-source 2"}}, + {"EKS + Go + React + Gatsby", []string{"module-source 1", "module-source 2"}}, {"foo", []string{"module-source 3"}}, {"bar", []string{"module-source 4"}}, {"lorem", []string{"module-source 5"}},