Refactor loading modules and creating project data

go.mod

@@ -3,7 +3,6 @@ module github.com/commitdev/zero
 go 1.12
 
 require (
-	github.com/766b/go-outliner v0.0.0-20180511142203-fc6edecdadd7 // indirect
 	github.com/aws/aws-sdk-go v1.25.33
 	github.com/chzyer/logex v1.1.10 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
@@ -26,7 +25,7 @@ require (
 	github.com/spf13/cobra v0.0.6
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
-	golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
+	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v2 v2.2.5
 )

go.sum

@@ -7,8 +7,6 @@ cloud.google.com/go v0.45.1 h1:lRi0CHyU+ytlvylOlFKKq0af6JncuyoRh1J+QJBqQx0=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-github.com/766b/go-outliner v0.0.0-20180511142203-fc6edecdadd7 h1:cJXisB2yAM61AzMutv7X+KM8F3xVLxGH99S8VmaSlps=
-github.com/766b/go-outliner v0.0.0-20180511142203-fc6edecdadd7/go.mod h1:1SzhThoS5lcKfE4IFOLQJ04WCmFpaAiPe8H9yqXyYSU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -183,8 +181,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
@@ -248,8 +244,8 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
-golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/config/moduleconfig/module_config.go

@@ -2,7 +2,6 @@ package moduleconfig
 
 import (
 	"io/ioutil"
-	"log"
 
 	"github.com/k0kubun/pp"
 	yaml "gopkg.in/yaml.v2"
@@ -32,16 +31,16 @@ type TemplateConfig struct {
 	Output     string
 }
 
-func LoadModuleConfig(filePath string) *ModuleConfig {
-	config := &ModuleConfig{}
+func LoadModuleConfig(filePath string) (ModuleConfig, error) {
+	config := ModuleConfig{}
 	data, err := ioutil.ReadFile(filePath)
 	if err != nil {
-		log.Panicf("failed to read config: %v", err)
+		return config, err
 	}
 	err = yaml.Unmarshal(data, &config)
 	if err != nil {
-		log.Panicf("failed to parse config: %v", err)
+		return config, err
 	}
 	pp.Println("Module Config:", config)
-	return config
+	return config, nil
 }

internal/context/init.go

@@ -9,8 +9,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/sts"
-	"github.com/commitdev/zero/internal/config"
 	"github.com/commitdev/zero/internal/config/globalconfig"
+	"github.com/commitdev/zero/internal/config/moduleconfig"
 	"github.com/commitdev/zero/internal/config/projectconfig"
 	"github.com/commitdev/zero/internal/module"
 	project "github.com/commitdev/zero/pkg/credentials"
@@ -38,31 +38,55 @@ func Init(projectName string, outDir string) *projectconfig.ZeroProjectConfig {
 	projectConfig.Context["ShouldPushRepoUpstream"] = promptPushRepoUpstream()
 	projectConfig.Context["GithubRootOrg"] = promptGithubRootOrg()
 	projectConfig.Context["githubPersonalToken"] = promptGithubPersonalToken(projectName)
-	projectConfig.Modules = chooseStack(getRegistry())
 
 	// chooseCloudProvider(&projectConfig)
 	// fmt.Println(&projectConfig)
 	// s := project.GetSecrets(rootDir)
 	// fillProviderDetails(&projectConfig, s)
 	// fmt.Println(&projectConfig)
+	moduleSources := chooseStack(getRegistry())
+	moduleConfigs := loadAllModules(moduleSources)
+	for _ = range moduleConfigs {
+		// TODO: initialize module structs inside project
+	}
+
+	projectParameters := promptAllModules(moduleConfigs)
+	for k, v = range projectParameters {
+		projectConfig.Context[k] = v
+		// TODO: Add parameters to module structs inside project
+	}
 
-	promptAllModules(&projectConfig)
 	// TODO: load ~/.zero/config.yml (or credentials)
 	// TODO: prompt global credentials
 
 	return &projectConfig
 }
 
-func promptAllModules(projectConfig *projectconfig.ZeroProjectConfig) {
-	// TODO: do we need to run through the modules and extract first
-	// or we need to run through twice, potentially still need to pre-process for global auths
-	for _, moduleSource := range projectConfig.Modules {
-		mod, _ := module.NewTemplateModule(config.ModuleInstance{Source: moduleSource})
-		err := mod.PromptParams(projectConfig.Context)
+// loadAllModules takes a list of module sources, downloads those modules, and parses their config
+func loadAllModules(moduleSources []string) map[string]moduleconfig.ModuleConfig {
+	modules := make(map[string]moduleconfig.ModuleConfig)
+
+	for _, moduleSource := range moduleSources {
+		mod, err := module.FetchModule(moduleSource)
+		if err != nil {
+			exit.Fatal("Unable to load module:  %v\n", err)
+		}
+		modules[mod.Name] = mod
+	}
+	return modules
+}
+
+// promptAllModules takes a map of all the modules and prompts the user for values for all the parameters
+func promptAllModules(modules map[string]moduleconfig.ModuleConfig) map[string]string {
+	parameterValues := make(map[string]string)
+	for _, config := range modules {
+		var err error
+		parameterValues, err = module.PromptParams(config, parameterValues)
 		if err != nil {
 			exit.Fatal("Exiting prompt:  %v\n", err)
 		}
 	}
+	return parameterValues
 }
 
 // global configs

internal/generate/generate_modules.go

@@ -23,15 +23,17 @@ import (
 func GenerateModules(cfg *config.GeneratorConfig) {
 	var templateModules []*module.TemplateModule
 
-	// Initiate all the modules defined in the config
-	for _, moduleConfig := range cfg.Modules {
-		mod, err := module.NewTemplateModule(moduleConfig)
+	// TODO: Refactor this since the module struct is changing
 
-		if err != nil {
-			exit.Error("module failed to load: %s", err)
-		}
-		templateModules = append(templateModules, mod)
-	}
+	// Initiate all the modules defined in the config
+	// for _, moduleConfig := range cfg.Modules {
+	//mod, err := module.NewTemplateModule(moduleConfig)
+
+	// if err != nil {
+	// 	exit.Error("module failed to load: %s", err)
+	// }
+	// templateModules = append(templateModules, mod)
+	// }
 
 	// Prompt for module params and execute each of the generator modules
 	for _, mod := range templateModules {

internal/module/module.go

@@ -10,7 +10,6 @@ import (
 	"os/exec"
 	"path"
 	"regexp"
-	"sync"
 
 	"github.com/commitdev/zero/internal/config"
 	"github.com/commitdev/zero/internal/config/moduleconfig"
@@ -26,34 +25,20 @@ type TemplateModule struct {
 	Config                moduleconfig.ModuleConfig
 }
 
-type ProgressTracking struct {
-	sync.Mutex
-	downloaded map[string]int
-}
-
-// Init downloads the remote template files and parses the module config yaml
-func NewTemplateModule(moduleCfg config.ModuleInstance) (*TemplateModule, error) {
-	var templateModule TemplateModule
-	templateModule.Source = moduleCfg.Source
-	templateModule.Params = moduleCfg.Params
-	templateModule.Overwrite = moduleCfg.Overwrite
-	templateModule.Output = moduleCfg.Output
-
-	p := &ProgressTracking{}
-	sourcePath := GetSourceDir(templateModule.Source)
-
-	if !IsLocal(templateModule.Source) {
-		err := getter.Get(sourcePath, templateModule.Source, getter.WithProgress(p))
+// FetchModule downloads the remote module source (or loads the local files) and parses the module config yaml
+func FetchModule(source string) (moduleconfig.ModuleConfig, error) {
+	config := moduleconfig.ModuleConfig{}
+	localPath := GetSourceDir(source)
+	if !isLocal(source) {
+		err := getter.Get(localPath, source)
 		if err != nil {
-			return nil, err
+			return config, err
 		}
 	}
 
-	configPath := path.Join(sourcePath, constants.ZeroModuleYml)
-	moduleConfig := moduleconfig.LoadModuleConfig(configPath)
-	templateModule.Config = *moduleConfig
-
-	return &templateModule, nil
+	configPath := path.Join(localPath, constants.ZeroModuleYml)
+	config, err := moduleconfig.LoadModuleConfig(configPath)
+	return config, err
 }
 
 func appendProjectEnvToCmdEnv(envMap map[string]string, envList []string) []string {
@@ -71,6 +56,12 @@ func sanitizePromptResult(str string) string {
 	return re.ReplaceAllString(str, "")
 }
 
+// TODO : Use this function signature instead
+// PromptParams renders series of prompt UI based on the config
+func PromptParams(moduleConfig moduleconfig.ModuleConfig, parameters map[string]string) (map[string]string, error) {
+	return map[string]string{}, nil
+}
+
 // PromptParams renders series of prompt UI based on the config
 func (m *TemplateModule) PromptParams(projectContext map[string]string) error {
 	for _, promptConfig := range m.Config.Prompts {
@@ -128,7 +119,7 @@ func (m *TemplateModule) PromptParams(projectContext map[string]string) error {
 
 // GetSourcePath gets a unique local source directory name. For local modules, it use the local directory
 func GetSourceDir(source string) string {
-	if !IsLocal(source) {
+	if !isLocal(source) {
 		h := md5.New()
 		io.WriteString(h, source)
 		source = base64.StdEncoding.EncodeToString(h.Sum(nil))
@@ -139,7 +130,7 @@ func GetSourceDir(source string) string {
 }
 
 // IsLocal uses the go-getter FileDetector to check if source is a file
-func IsLocal(source string) bool {
+func isLocal(source string) bool {
 	pwd := util.GetCwd()
 
 	// ref: https://github.com/hashicorp/go-getter/blob/master/detect_test.go
@@ -159,16 +150,3 @@ func withPWD(pwd string) func(*getter.Client) error {
 		return nil
 	}
 }
-
-func (p *ProgressTracking) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) (body io.ReadCloser) {
-	p.Lock()
-	defer p.Unlock()
-
-	if p.downloaded == nil {
-		p.downloaded = map[string]int{}
-	}
-
-	v, _ := p.downloaded[src]
-	p.downloaded[src] = v + 1
-	return stream
-}

internal/module/module_internal_test.go

@@ -0,0 +1,19 @@
+package module
+
+import (
+	"testing"
+)
+
+func TestIsLocal(t *testing.T) {
+	source := "./tests/test_data/modules"
+	res := isLocal(source)
+	if !res {
+		t.Errorf("Error, source %s SHOULD BE determined as local", source)
+	}
+
+	source = "https://github.com/commitdev/my-repo"
+	res = isLocal(source)
+	if res {
+		t.Errorf("Error, source %s SHOULD NOT BE determined as local", source)
+	}
+}

internal/module/module_test.go

@@ -1,59 +1,24 @@
-package module
+package module_test
 
 import (
 	"testing"
-)
-
-func TestIsLocal(t *testing.T) {
-	source := "./tests/test_data/modules/ci"
-	res := IsLocal(source)
-	if !res {
-		t.Errorf("Error, source %s SHOULD BE determined as local", source)
-	}
 
-	source = "https://github.com/zthomas/react-mui-kit"
-	res = IsLocal(source)
-	if res {
-		t.Errorf("Error, source %s SHOULD NOT BE determined as local", source)
-	}
-}
+	"github.com/commitdev/zero/internal/module"
+)
 
 func TestGetSourceDir(t *testing.T) {
-	source := "tests/test_data/modules/ci"
+	source := "tests/test_data/modules"
 	relativeSource := source
-	dir := GetSourceDir(source)
+	dir := module.GetSourceDir(source)
 
 	t.Log("dir", dir)
 	if dir != relativeSource {
 		t.Errorf("Error, local sources should not be changed: %s", source)
 	}
 
-	source = "github.com/zthomas/react-mui-kit"
-	dir = GetSourceDir(source)
+	source = "github.com/commitdev/my-repo"
+	dir = module.GetSourceDir(source)
 	if dir == relativeSource {
 		t.Errorf("Error, remote sources should be converted to a local dir: %s", source)
 	}
 }
-
-// var testData = "../../tests/test_data/ci/"
-
-// func setupTeardown(t *testing.T) func(t *testing.T) {
-// 	outputPath := "../../tmp/tests"
-// 	os.RemoveAll(outputPath)
-// 	return func(t *testing.T) {
-// 		os.RemoveAll(outputPath)
-// 	}
-// }
-
-// func TestDirectoryTemplates(t *testing.T) {
-// 	teardown := setupTeardown(t)
-// 	defer teardown(t)
-
-// 	// TODO organize test utils and write assertions
-// 	templator := NewDirTemplator("../../tests/test_data/modules/ci", []string{"{{", "}}"})
-// 	var data = map[string]string{
-// 		"ci": "github",
-// 	}
-
-// 	templator.ExecuteTemplates(data, false, "tmp", "")
-// }