From f363ba014faf4a84562113e5d433fbd9a704f16d Mon Sep 17 00:00:00 2001 From: Bill Monkman Date: Fri, 5 Jun 2020 15:20:57 -0700 Subject: [PATCH 1/3] Refactor loading modules and creating project data Split prompting into a few smaller, more specific functions Split module tests into internal and external --- go.mod | 3 +- go.sum | 8 +-- internal/config/moduleconfig/module_config.go | 11 ++-- internal/context/init.go | 38 ++++++++++--- internal/generate/generate_modules.go | 18 +++--- internal/module/module.go | 57 ++++++------------- internal/module/module_internal_test.go | 19 +++++++ internal/module/module_test.go | 49 +++------------- 8 files changed, 92 insertions(+), 111 deletions(-) create mode 100644 internal/module/module_internal_test.go diff --git a/go.mod b/go.mod index 9abb24a48..6940f00b3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/commitdev/zero go 1.12 require ( - github.com/766b/go-outliner v0.0.0-20180511142203-fc6edecdadd7 // indirect github.com/aws/aws-sdk-go v1.25.33 github.com/chzyer/logex v1.1.10 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect @@ -26,7 +25,7 @@ require ( github.com/spf13/cobra v0.0.6 github.com/stretchr/testify v1.4.0 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect - golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect + golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.5 ) diff --git a/go.sum b/go.sum index 6e2005733..66295da6d 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ cloud.google.com/go v0.45.1 h1:lRi0CHyU+ytlvylOlFKKq0af6JncuyoRh1J+QJBqQx0= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -github.com/766b/go-outliner v0.0.0-20180511142203-fc6edecdadd7 h1:cJXisB2yAM61AzMutv7X+KM8F3xVLxGH99S8VmaSlps= -github.com/766b/go-outliner v0.0.0-20180511142203-fc6edecdadd7/go.mod h1:1SzhThoS5lcKfE4IFOLQJ04WCmFpaAiPe8H9yqXyYSU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -183,8 +181,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= @@ -248,8 +244,8 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index e472c42b8..c1e4840b5 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -2,7 +2,6 @@ package moduleconfig import ( "io/ioutil" - "log" "github.com/k0kubun/pp" yaml "gopkg.in/yaml.v2" @@ -32,16 +31,16 @@ type TemplateConfig struct { Output string } -func LoadModuleConfig(filePath string) *ModuleConfig { - config := &ModuleConfig{} +func LoadModuleConfig(filePath string) (ModuleConfig, error) { + config := ModuleConfig{} data, err := ioutil.ReadFile(filePath) if err != nil { - log.Panicf("failed to read config: %v", err) + return config, err } err = yaml.Unmarshal(data, &config) if err != nil { - log.Panicf("failed to parse config: %v", err) + return config, err } pp.Println("Module Config:", config) - return config + return config, nil } diff --git a/internal/context/init.go b/internal/context/init.go index e212e4f87..8732f6244 100644 --- a/internal/context/init.go +++ b/internal/context/init.go @@ -9,8 +9,8 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sts" - "github.com/commitdev/zero/internal/config" "github.com/commitdev/zero/internal/config/globalconfig" + "github.com/commitdev/zero/internal/config/moduleconfig" "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/internal/module" project "github.com/commitdev/zero/pkg/credentials" @@ -38,31 +38,55 @@ func Init(projectName string, outDir string) *projectconfig.ZeroProjectConfig { projectConfig.Context["ShouldPushRepoUpstream"] = promptPushRepoUpstream() projectConfig.Context["GithubRootOrg"] = promptGithubRootOrg() projectConfig.Context["githubPersonalToken"] = promptGithubPersonalToken(projectName) - projectConfig.Modules = chooseStack(getRegistry()) // chooseCloudProvider(&projectConfig) // fmt.Println(&projectConfig) // s := project.GetSecrets(rootDir) // fillProviderDetails(&projectConfig, s) // fmt.Println(&projectConfig) + moduleSources := chooseStack(getRegistry()) + moduleConfig := loadAllModules(moduleSources) + for _ = range moduleConfig { + // TODO: initialize module structs inside project + } + + projectParameters := promptAllModules(moduleConfig) + for _ = range projectParameters { + // TODO: Add parameters to module structs inside project + } - promptAllModules(&projectConfig) // TODO: load ~/.zero/config.yml (or credentials) // TODO: prompt global credentials return &projectConfig } -func promptAllModules(projectConfig *projectconfig.ZeroProjectConfig) { +func loadAllModules(moduleSources []string) map[string]moduleconfig.ModuleConfig { // TODO: do we need to run through the modules and extract first // or we need to run through twice, potentially still need to pre-process for global auths - for _, moduleSource := range projectConfig.Modules { - mod, _ := module.NewTemplateModule(config.ModuleInstance{Source: moduleSource}) - err := mod.PromptParams(projectConfig.Context) + + modules := make(map[string]moduleconfig.ModuleConfig) + + for _, moduleSource := range moduleSources { + mod, err := module.FetchModule(moduleSource) + if err != nil { + exit.Fatal("Unable to load module: %v\n", err) + } + modules[mod.Name] = mod + } + return modules +} + +func promptAllModules(modules map[string]moduleconfig.ModuleConfig) map[string]string { + parameterValues := make(map[string]string) + for _, config := range modules { + var err error + parameterValues, err = module.PromptParams(config, parameterValues) if err != nil { exit.Fatal("Exiting prompt: %v\n", err) } } + return parameterValues } // global configs diff --git a/internal/generate/generate_modules.go b/internal/generate/generate_modules.go index 0ffaf14d8..4b2731688 100644 --- a/internal/generate/generate_modules.go +++ b/internal/generate/generate_modules.go @@ -23,15 +23,17 @@ import ( func GenerateModules(cfg *config.GeneratorConfig) { var templateModules []*module.TemplateModule - // Initiate all the modules defined in the config - for _, moduleConfig := range cfg.Modules { - mod, err := module.NewTemplateModule(moduleConfig) + // TODO: Refactor this since the module struct is changing - if err != nil { - exit.Error("module failed to load: %s", err) - } - templateModules = append(templateModules, mod) - } + // Initiate all the modules defined in the config + // for _, moduleConfig := range cfg.Modules { + //mod, err := module.NewTemplateModule(moduleConfig) + + // if err != nil { + // exit.Error("module failed to load: %s", err) + // } + // templateModules = append(templateModules, mod) + // } // Prompt for module params and execute each of the generator modules for _, mod := range templateModules { diff --git a/internal/module/module.go b/internal/module/module.go index 4babb1810..f0fc9c743 100644 --- a/internal/module/module.go +++ b/internal/module/module.go @@ -10,7 +10,6 @@ import ( "os/exec" "path" "regexp" - "sync" "github.com/commitdev/zero/internal/config" "github.com/commitdev/zero/internal/config/moduleconfig" @@ -26,34 +25,20 @@ type TemplateModule struct { Config moduleconfig.ModuleConfig } -type ProgressTracking struct { - sync.Mutex - downloaded map[string]int -} - -// Init downloads the remote template files and parses the module config yaml -func NewTemplateModule(moduleCfg config.ModuleInstance) (*TemplateModule, error) { - var templateModule TemplateModule - templateModule.Source = moduleCfg.Source - templateModule.Params = moduleCfg.Params - templateModule.Overwrite = moduleCfg.Overwrite - templateModule.Output = moduleCfg.Output - - p := &ProgressTracking{} - sourcePath := GetSourceDir(templateModule.Source) - - if !IsLocal(templateModule.Source) { - err := getter.Get(sourcePath, templateModule.Source, getter.WithProgress(p)) +// FetchModule downloads the remote module source (or loads the local files) and parses the module config yaml +func FetchModule(source string) (moduleconfig.ModuleConfig, error) { + config := moduleconfig.ModuleConfig{} + localPath := GetSourceDir(source) + if !isLocal(source) { + err := getter.Get(localPath, source) if err != nil { - return nil, err + return config, err } } - configPath := path.Join(sourcePath, constants.ZeroModuleYml) - moduleConfig := moduleconfig.LoadModuleConfig(configPath) - templateModule.Config = *moduleConfig - - return &templateModule, nil + configPath := path.Join(localPath, constants.ZeroModuleYml) + config, err := moduleconfig.LoadModuleConfig(configPath) + return config, err } func appendProjectEnvToCmdEnv(envMap map[string]string, envList []string) []string { @@ -71,6 +56,11 @@ func sanitizePromptResult(str string) string { return re.ReplaceAllString(str, "") } +// TODO : Use this function signature instead +func PromptParams(moduleConfig moduleconfig.ModuleConfig, parameters map[string]string) (map[string]string, error) { + return map[string]string{}, nil +} + // PromptParams renders series of prompt UI based on the config func (m *TemplateModule) PromptParams(projectContext map[string]string) error { for _, promptConfig := range m.Config.Prompts { @@ -128,7 +118,7 @@ func (m *TemplateModule) PromptParams(projectContext map[string]string) error { // GetSourcePath gets a unique local source directory name. For local modules, it use the local directory func GetSourceDir(source string) string { - if !IsLocal(source) { + if !isLocal(source) { h := md5.New() io.WriteString(h, source) source = base64.StdEncoding.EncodeToString(h.Sum(nil)) @@ -139,7 +129,7 @@ func GetSourceDir(source string) string { } // IsLocal uses the go-getter FileDetector to check if source is a file -func IsLocal(source string) bool { +func isLocal(source string) bool { pwd := util.GetCwd() // ref: https://github.com/hashicorp/go-getter/blob/master/detect_test.go @@ -159,16 +149,3 @@ func withPWD(pwd string) func(*getter.Client) error { return nil } } - -func (p *ProgressTracking) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) (body io.ReadCloser) { - p.Lock() - defer p.Unlock() - - if p.downloaded == nil { - p.downloaded = map[string]int{} - } - - v, _ := p.downloaded[src] - p.downloaded[src] = v + 1 - return stream -} diff --git a/internal/module/module_internal_test.go b/internal/module/module_internal_test.go new file mode 100644 index 000000000..8ec9e0826 --- /dev/null +++ b/internal/module/module_internal_test.go @@ -0,0 +1,19 @@ +package module + +import ( + "testing" +) + +func TestIsLocal(t *testing.T) { + source := "./tests/test_data/modules" + res := isLocal(source) + if !res { + t.Errorf("Error, source %s SHOULD BE determined as local", source) + } + + source = "https://github.com/commitdev/my-repo" + res = isLocal(source) + if res { + t.Errorf("Error, source %s SHOULD NOT BE determined as local", source) + } +} diff --git a/internal/module/module_test.go b/internal/module/module_test.go index 6fd50c512..a75d1aff7 100644 --- a/internal/module/module_test.go +++ b/internal/module/module_test.go @@ -1,59 +1,24 @@ -package module +package module_test import ( "testing" -) - -func TestIsLocal(t *testing.T) { - source := "./tests/test_data/modules/ci" - res := IsLocal(source) - if !res { - t.Errorf("Error, source %s SHOULD BE determined as local", source) - } - source = "https://github.com/zthomas/react-mui-kit" - res = IsLocal(source) - if res { - t.Errorf("Error, source %s SHOULD NOT BE determined as local", source) - } -} + "github.com/commitdev/zero/internal/module" +) func TestGetSourceDir(t *testing.T) { - source := "tests/test_data/modules/ci" + source := "tests/test_data/modules" relativeSource := source - dir := GetSourceDir(source) + dir := module.GetSourceDir(source) t.Log("dir", dir) if dir != relativeSource { t.Errorf("Error, local sources should not be changed: %s", source) } - source = "github.com/zthomas/react-mui-kit" - dir = GetSourceDir(source) + source = "github.com/commitdev/my-repo" + dir = module.GetSourceDir(source) if dir == relativeSource { t.Errorf("Error, remote sources should be converted to a local dir: %s", source) } } - -// var testData = "../../tests/test_data/ci/" - -// func setupTeardown(t *testing.T) func(t *testing.T) { -// outputPath := "../../tmp/tests" -// os.RemoveAll(outputPath) -// return func(t *testing.T) { -// os.RemoveAll(outputPath) -// } -// } - -// func TestDirectoryTemplates(t *testing.T) { -// teardown := setupTeardown(t) -// defer teardown(t) - -// // TODO organize test utils and write assertions -// templator := NewDirTemplator("../../tests/test_data/modules/ci", []string{"{{", "}}"}) -// var data = map[string]string{ -// "ci": "github", -// } - -// templator.ExecuteTemplates(data, false, "tmp", "") -// } From 2c6897f27580ad83db35eeb28dbe4396e0520f8c Mon Sep 17 00:00:00 2001 From: Bill Monkman Date: Fri, 5 Jun 2020 16:22:09 -0700 Subject: [PATCH 2/3] Update internal/context/init.go Preserve backward compatibility Co-authored-by: David Cheung --- internal/context/init.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/context/init.go b/internal/context/init.go index 8732f6244..87c11b60e 100644 --- a/internal/context/init.go +++ b/internal/context/init.go @@ -51,7 +51,9 @@ func Init(projectName string, outDir string) *projectconfig.ZeroProjectConfig { } projectParameters := promptAllModules(moduleConfig) - for _ = range projectParameters { + + for k, v = range projectParameters { + projectConfig.Context[k] = v // TODO: Add parameters to module structs inside project } From ab95383d43de8826cb53ec2c7abf88b344f1650b Mon Sep 17 00:00:00 2001 From: Bill Monkman Date: Fri, 5 Jun 2020 16:28:35 -0700 Subject: [PATCH 3/3] Changed pluralization --- internal/context/init.go | 14 ++++++-------- internal/module/module.go | 1 + 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/internal/context/init.go b/internal/context/init.go index 87c11b60e..dc3193da6 100644 --- a/internal/context/init.go +++ b/internal/context/init.go @@ -45,15 +45,14 @@ func Init(projectName string, outDir string) *projectconfig.ZeroProjectConfig { // fillProviderDetails(&projectConfig, s) // fmt.Println(&projectConfig) moduleSources := chooseStack(getRegistry()) - moduleConfig := loadAllModules(moduleSources) - for _ = range moduleConfig { + moduleConfigs := loadAllModules(moduleSources) + for _ = range moduleConfigs { // TODO: initialize module structs inside project } - projectParameters := promptAllModules(moduleConfig) - + projectParameters := promptAllModules(moduleConfigs) for k, v = range projectParameters { - projectConfig.Context[k] = v + projectConfig.Context[k] = v // TODO: Add parameters to module structs inside project } @@ -63,10 +62,8 @@ func Init(projectName string, outDir string) *projectconfig.ZeroProjectConfig { return &projectConfig } +// loadAllModules takes a list of module sources, downloads those modules, and parses their config func loadAllModules(moduleSources []string) map[string]moduleconfig.ModuleConfig { - // TODO: do we need to run through the modules and extract first - // or we need to run through twice, potentially still need to pre-process for global auths - modules := make(map[string]moduleconfig.ModuleConfig) for _, moduleSource := range moduleSources { @@ -79,6 +76,7 @@ func loadAllModules(moduleSources []string) map[string]moduleconfig.ModuleConfig return modules } +// promptAllModules takes a map of all the modules and prompts the user for values for all the parameters func promptAllModules(modules map[string]moduleconfig.ModuleConfig) map[string]string { parameterValues := make(map[string]string) for _, config := range modules { diff --git a/internal/module/module.go b/internal/module/module.go index f0fc9c743..b525d9110 100644 --- a/internal/module/module.go +++ b/internal/module/module.go @@ -57,6 +57,7 @@ func sanitizePromptResult(str string) string { } // TODO : Use this function signature instead +// PromptParams renders series of prompt UI based on the config func PromptParams(moduleConfig moduleconfig.ModuleConfig, parameters map[string]string) (map[string]string, error) { return map[string]string{}, nil }