From baebfcf70d156989bfc1c2863afbf02b77316323 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 3 Mar 2021 15:58:24 -0500 Subject: [PATCH 01/18] [SEMVER-MAJOR] condition support to module params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit changes includes - removing global config - module config’s parameter to support condition - module config’s parameter to support custom typed prompts - prompts to not return value, instead update map(can support multi-value) --- go.mod | 1 + internal/config/moduleconfig/module_config.go | 64 +++++++++++-- internal/init/custom-prompts.go | 41 +++++++++ internal/init/init.go | 59 +++--------- internal/init/prompts.go | 91 +++++++++++++++---- internal/init/prompts_test.go | 34 +++++-- internal/module/module_test.go | 12 +++ internal/registry/registry.go | 6 ++ pkg/credentials/credentials.go | 16 ++++ tests/test_data/modules/ci/zero-module.yml | 31 +++++++ 10 files changed, 273 insertions(+), 82 deletions(-) create mode 100644 internal/init/custom-prompts.go diff --git a/go.mod b/go.mod index 33a65a2ce..04972d7d5 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/matryer/is v1.3.0 // indirect github.com/mattn/go-colorable v0.1.2 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v0.0.6 github.com/stretchr/testify v1.5.1 diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index 296d7d1c3..c8d8e6036 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -9,8 +9,10 @@ import ( yaml "gopkg.in/yaml.v2" + "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/pkg/util/flog" "github.com/iancoleman/strcase" + "github.com/pkg/errors" ) type ModuleConfig struct { @@ -25,20 +27,23 @@ type ModuleConfig struct { } type Parameter struct { - Field string - Label string `yaml:"label,omitempty"` - Options []string `yaml:"options,omitempty"` - Execute string `yaml:"execute,omitempty"` - Value string `yaml:"value,omitempty"` - Default string `yaml:"default,omitempty"` - Info string `yaml:"info,omitempty"` - FieldValidation Validate `yaml:"fieldValidation,omitempty"` + Field string + Label string `yaml:"label,omitempty"` + Options []string `yaml:"options,omitempty"` + Execute string `yaml:"execute,omitempty"` + Value string `yaml:"value,omitempty"` + Default string `yaml:"default,omitempty"` + Info string `yaml:"info,omitempty"` + FieldValidation Validate `yaml:"fieldValidation,omitempty"` + Type string `yaml:"type,omitempty"` + OmitFromProjectFile bool `yaml:"omitFromProjectFile,omitempty"` + Conditions []Condition `yaml:"conditions,omitempty"` } type Condition struct { Action string `yaml:"action"` MatchField string `yaml:"matchField"` - WhenValue string `yaml:"whenValue"` + WhenValue string `yaml:"whenValue,omitempty"` Data []string `yaml:"data,omitempty"` } @@ -76,6 +81,7 @@ func LoadModuleConfig(filePath string) (ModuleConfig, error) { return config, err } + validateParams(config.Parameters) missing := config.collectMissing() if len(missing) > 0 { flog.Errorf("%v is missing information", filePath) @@ -90,6 +96,15 @@ func LoadModuleConfig(filePath string) (ModuleConfig, error) { return config, nil } +func validateParams(params []Parameter) error { + for _, param := range params { + if param.Type != "" { + return errors.Errorf("type is not supported") + } + } + return nil +} + // Recurses through a datastructure to find any missing data. // This assumes several things: // 1. The structure matches that defined by ModuleConfig and its child datastructures. @@ -158,3 +173,34 @@ func findMissing(obj reflect.Value, path, metadata string, missing *[]string) { } } } + +func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[string]string { + moduleParams := make(projectconfig.Parameters) + // Loop through all the prompted values and find the ones relevant to this module + for parameterKey, parameterValue := range allParams { + for _, moduleParameter := range module.Parameters { + if moduleParameter.Field == parameterKey { + if moduleParameter.OmitFromProjectFile { + flog.Debugf("Omitted %s from %s", parameterKey, module.Name) + } else { + moduleParams[parameterKey] = parameterValue + } + } + } + } + return moduleParams +} + +func SummarizeConditions(module ModuleConfig) []projectconfig.Condition { + moduleConditions := []projectconfig.Condition{} + for _, condition := range module.Conditions { + newCond := projectconfig.Condition{ + Action: condition.Action, + MatchField: condition.MatchField, + WhenValue: condition.WhenValue, + Data: condition.Data, + } + moduleConditions = append(moduleConditions, newCond) + } + return moduleConditions +} diff --git a/internal/init/custom-prompts.go b/internal/init/custom-prompts.go new file mode 100644 index 000000000..3da7931c7 --- /dev/null +++ b/internal/init/custom-prompts.go @@ -0,0 +1,41 @@ +package init + +import ( + "github.com/commitdev/zero/internal/config/moduleconfig" + project "github.com/commitdev/zero/pkg/credentials" + "github.com/commitdev/zero/pkg/util/flog" + "github.com/k0kubun/pp" +) + +func CustomPromptHandler(promptType string, params map[string]string) { + switch promptType { + + case "AWSProfilePicker": + AWSProfilePicker(params) + } +} + +func AWSProfilePicker(params map[string]string) { + profiles, err := project.GetAWSProfiles() + if err != nil { + profiles = []string{} + } + + awsPrompt := PromptHandler{ + Parameter: moduleconfig.Parameter{ + Field: "aws_profile", + Label: "Select AWS Profile", + Options: profiles, + }, + Condition: NoCondition, + Validate: NoValidation, + } + _, value := promptParameter(awsPrompt) + pp.Print(value) + credErr := project.FillAWSProfile(value, params) + if credErr != nil { + flog.Errorf("Failed to retrieve profile, falling back to User input") + params["useExistingAwsProfile"] = "no" + } + pp.Print(params) +} diff --git a/internal/init/init.go b/internal/init/init.go index 1f273dff7..fe0ba001f 100644 --- a/internal/init/init.go +++ b/internal/init/init.go @@ -22,7 +22,10 @@ import ( func Init(outDir string, localModulePath string) *projectconfig.ZeroProjectConfig { projectConfig := defaultProjConfig() - projectConfig.Name = getProjectNamePrompt().GetParam(projectConfig.Parameters) + projectRootParams := map[string]string{} + promptName := getProjectNamePrompt() + promptName.RunPrompt(projectRootParams) + projectConfig.Name = projectRootParams[promptName.Field] rootDir := path.Join(outDir, projectConfig.Name) flog.Infof(":tada: Initializing project") @@ -41,44 +44,23 @@ func Init(outDir string, localModulePath string) *projectconfig.ZeroProjectConfi initParams := make(map[string]string) projectConfig.ShouldPushRepositories = true - initParams["ShouldPushRepositories"] = prompts["ShouldPushRepositories"].GetParam(initParams) + prompts["ShouldPushRepositories"].RunPrompt(initParams) if initParams["ShouldPushRepositories"] == "n" { projectConfig.ShouldPushRepositories = false } // Prompting for push-up stream, then conditionally prompting for github - initParams["GithubRootOrg"] = prompts["GithubRootOrg"].GetParam(initParams) - projectCredentials := globalconfig.GetProjectCredentials(projectConfig.Name) - credentialPrompts := getCredentialPrompts(projectCredentials, moduleConfigs) - projectCredentials = promptCredentialsAndFillProjectCreds(credentialPrompts, projectCredentials) - globalconfig.Save(projectCredentials) - projectParameters := promptAllModules(moduleConfigs, projectCredentials) + prompts["GithubRootOrg"].RunPrompt(initParams) + + projectData := promptAllModules(moduleConfigs) // Map parameter values back to specific modules for moduleName, module := range moduleConfigs { - repoName := prompts[moduleName].GetParam(initParams) + prompts[moduleName].RunPrompt(initParams) + repoName := initParams[prompts[moduleName].Field] repoURL := fmt.Sprintf("%s/%s", initParams["GithubRootOrg"], repoName) - projectModuleParams := make(projectconfig.Parameters) - projectModuleConditions := []projectconfig.Condition{} - - // Loop through all the prompted values and find the ones relevant to this module - for parameterKey, parameterValue := range projectParameters { - for _, moduleParameter := range module.Parameters { - if moduleParameter.Field == parameterKey { - projectModuleParams[parameterKey] = parameterValue - } - } - } - - for _, condition := range module.Conditions { - newCond := projectconfig.Condition{ - Action: condition.Action, - MatchField: condition.MatchField, - WhenValue: condition.WhenValue, - Data: condition.Data, - } - projectModuleConditions = append(projectModuleConditions, newCond) - } + projectModuleParams := moduleconfig.SummarizeParameters(module, projectData) + projectModuleConditions := moduleconfig.SummarizeConditions(module) projectConfig.Modules[moduleName] = projectconfig.NewModule( projectModuleParams, @@ -90,9 +72,6 @@ func Init(outDir string, localModulePath string) *projectconfig.ZeroProjectConfi ) } - // TODO: load ~/.zero/config.yml (or credentials) - // TODO: prompt global credentials - return &projectConfig } @@ -119,20 +98,6 @@ func loadAllModules(moduleSources []string) (map[string]moduleconfig.ModuleConfi return modules, mappedSources } -// promptAllModules takes a map of all the modules and prompts the user for values for all the parameters -func promptAllModules(modules map[string]moduleconfig.ModuleConfig, projectCredentials globalconfig.ProjectCredential) map[string]string { - parameterValues := map[string]string{"projectName": projectCredentials.ProjectName} - for _, config := range modules { - var err error - - parameterValues, err = PromptModuleParams(config, parameterValues, projectCredentials) - if err != nil { - exit.Fatal("Exiting prompt: %v\n", err) - } - } - return parameterValues -} - // Project name is prompt individually because the rest of the prompts // requires the projectName to populate defaults func getProjectNamePrompt() PromptHandler { diff --git a/internal/init/prompts.go b/internal/init/prompts.go index 018375c87..3cbd8294c 100644 --- a/internal/init/prompts.go +++ b/internal/init/prompts.go @@ -16,6 +16,7 @@ import ( "github.com/commitdev/zero/pkg/credentials" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" + "github.com/k0kubun/pp" "github.com/manifoldco/promptui" "gopkg.in/yaml.v2" ) @@ -103,7 +104,13 @@ func ValidateProjectName(input string) error { return nil } -func (p PromptHandler) GetParam(projectParams map[string]string) string { +// Getting param to fill in to zero-project.yml, there are multiple ways of obtaining the value +// values go into params depending on `Condition` as the highest precedence (Whether it gets this value) +// then follows this order to determine HOW it obtains that value +// 1. Execute (this could potentially be refactored into type + data) +// 2. type: specific ways of obtaining values (in AWS credential case it will set 2 values to the map) +//3. +func (p PromptHandler) RunPrompt(projectParams map[string]string) { var err error var result string @@ -117,6 +124,8 @@ func (p PromptHandler) GetParam(projectParams map[string]string) string { // it wouldnt leak things the module shouldnt have access to if p.Parameter.Execute != "" { result = executeCmd(p.Parameter.Execute, projectParams) + } else if p.Parameter.Type != "" { + CustomPromptHandler(p.Parameter.Type, projectParams) } else if p.Parameter.Value != "" { result = p.Parameter.Value } else { @@ -126,9 +135,11 @@ func (p PromptHandler) GetParam(projectParams map[string]string) string { exit.Fatal("Exiting prompt: %v\n", err) } - return sanitizeParameterValue(result) + // Append the result to parameter map + projectParams[p.Field] = sanitizeParameterValue(result) + } else { + flog.Debugf("Skipping prompt(%s) due to condition failed", p.Field) } - return "" } func promptParameter(prompt PromptHandler) (error, string) { @@ -165,6 +176,7 @@ func promptParameter(prompt PromptHandler) (error, string) { } func executeCmd(command string, envVars map[string]string) string { + pp.Print(envVars) cmd := exec.Command("bash", "-c", command) cmd.Env = util.AppendProjectEnvToCmdEnv(envVars, os.Environ()) out, err := cmd.Output() @@ -183,22 +195,22 @@ func sanitizeParameterValue(str string) string { } // PromptParams renders series of prompt UI based on the config -func PromptModuleParams(moduleConfig moduleconfig.ModuleConfig, parameters map[string]string, projectCredentials globalconfig.ProjectCredential) (map[string]string, error) { - credentialEnvs := projectCredentials.SelectedVendorsCredentialsAsEnv(moduleConfig.RequiredCredentials) - for _, promptConfig := range moduleConfig.Parameters { +func PromptModuleParams(moduleConfig moduleconfig.ModuleConfig, parameters map[string]string) (map[string]string, error) { + + for _, parameter := range moduleConfig.Parameters { // deduplicate fields already prompted and received - if _, isAlreadySet := parameters[promptConfig.Field]; isAlreadySet { + if _, isAlreadySet := parameters[parameter.Field]; isAlreadySet { continue } var validateFunc func(input string) error = nil // type:regex field validation for zero-module.yaml - if promptConfig.FieldValidation.Type == constants.RegexValidation { + if parameter.FieldValidation.Type == constants.RegexValidation { validateFunc = func(input string) error { - var regexRule = regexp.MustCompile(promptConfig.FieldValidation.Value) + var regexRule = regexp.MustCompile(parameter.FieldValidation.Value) if !regexRule.MatchString(input) { - return errors.New(promptConfig.FieldValidation.ErrorMessage) + return errors.New(parameter.FieldValidation.ErrorMessage) } return nil } @@ -206,21 +218,64 @@ func PromptModuleParams(moduleConfig moduleconfig.ModuleConfig, parameters map[s // TODO: type:fuction field validation for zero-module.yaml promptHandler := PromptHandler{ - Parameter: promptConfig, - Condition: NoCondition, + Parameter: parameter, + Condition: paramConditionsMapper(parameter.Conditions), Validate: validateFunc, } // merging the context of param and credentals // this treats credentialEnvs as throwaway, parameters is shared between modules // so credentials should not be in parameters as it gets returned to parent - for k, v := range parameters { - credentialEnvs[k] = v + // for k, v := range parameters { + // credentialEnvs[k] = v + // } + promptHandler.RunPrompt(parameters) + + // parameters[parameter.Field] = result + pp.Print("Module is done processing %s", moduleConfig.Name) + pp.Print(parameters) + } + return parameters, nil +} + +// promptAllModules takes a map of all the modules and prompts the user for values for all the parameters +// Important: This is done here because in this step we share the parameter across modules, +// meaning if module A and B both asks for region, it will reuse the response for both (and is deduped during runtime) +func promptAllModules(modules map[string]moduleconfig.ModuleConfig) map[string]string { + parameterValues := map[string]string{} + for _, config := range modules { + var err error + + parameterValues, err = PromptModuleParams(config, parameterValues) + if err != nil { + exit.Fatal("Exiting prompt: %v\n", err) } - result := promptHandler.GetParam(credentialEnvs) + } + return parameterValues +} - parameters[promptConfig.Field] = result +func paramConditionsMapper(conditions []moduleconfig.Condition) CustomConditionSignature { + if len(conditions) == 0 { + return NoCondition + } else { + return func(params map[string]string) bool { + // Prompts must pass every condition to proceed + for i := 0; i < len(conditions); i++ { + if !conditionHandler(conditions[i])(params) { + flog.Debugf("Did not meet condition %v, expected %v to be %v", conditions[i].Action, conditions[i].MatchField, conditions[i].WhenValue) + return false + } + } + return true + } + } +} +func conditionHandler(cond moduleconfig.Condition) CustomConditionSignature { + if cond.Action == "KeyMatchCondition" { + return KeyMatchCondition(cond.MatchField, cond.WhenValue) + } else { + flog.Errorf("Unsupported condition") + return nil } - return parameters, nil } func promptCredentialsAndFillProjectCreds(credentialPrompts []CredentialPrompts, creds globalconfig.ProjectCredential) globalconfig.ProjectCredential { @@ -232,7 +287,7 @@ func promptCredentialsAndFillProjectCreds(credentialPrompts []CredentialPrompts, // vendors like AWS have multiple prompts (accessKeyId and secretAccessKey) for _, prompt := range prompts.Prompts { - vendorPromptValues[prompt.Field] = prompt.GetParam(vendorPromptValues) + prompt.RunPrompt(vendorPromptValues) } promptsValues[vendor] = vendorPromptValues } diff --git a/internal/init/prompts_test.go b/internal/init/prompts_test.go index 35c971a8b..c2af1bda5 100644 --- a/internal/init/prompts_test.go +++ b/internal/init/prompts_test.go @@ -25,8 +25,8 @@ func TestGetParam(t *testing.T) { initPrompts.NoValidation, } - result := prompt.GetParam(projectParams) - assert.Equal(t, "my-acconut-id", result) + prompt.RunPrompt(projectParams) + assert.Equal(t, "my-acconut-id", projectParams[param.Field]) }) t.Run("executes with project context", func(t *testing.T) { @@ -41,10 +41,9 @@ func TestGetParam(t *testing.T) { initPrompts.NoValidation, } - result := prompt.GetParam(map[string]string{ - "INJECTEDENV": "SOME_ENV_VAR_VALUE", - }) - assert.Equal(t, "SOME_ENV_VAR_VALUE", result) + projectParams := map[string]string{"INJECTEDENV": "SOME_ENV_VAR_VALUE"} + prompt.RunPrompt(projectParams) + assert.Equal(t, "SOME_ENV_VAR_VALUE", projectParams[param.Field]) }) t.Run("Should return static value", func(t *testing.T) { @@ -59,8 +58,27 @@ func TestGetParam(t *testing.T) { initPrompts.NoValidation, } - result := prompt.GetParam(projectParams) - assert.Equal(t, "lorem-ipsum", result) + prompt.RunPrompt(projectParams) + assert.Equal(t, "lorem-ipsum", projectParams[param.Field]) }) + t.Run("Prompt value to retain existing params", func(t *testing.T) { + projectParams = map[string]string{ + "existing_value": "foo", + } + param := moduleconfig.Parameter{ + Field: "new_value", + Value: "bar", + } + + prompt := initPrompts.PromptHandler{ + param, + initPrompts.NoCondition, + initPrompts.NoValidation, + } + + prompt.RunPrompt(projectParams) + assert.Equal(t, "foo", projectParams["existing_value"]) + assert.Equal(t, "bar", projectParams[param.Field]) + }) } diff --git a/internal/module/module_test.go b/internal/module/module_test.go index 1bf553f58..b62c5eaf8 100644 --- a/internal/module/module_test.go +++ b/internal/module/module_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/commitdev/zero/internal/config/moduleconfig" + "github.com/k0kubun/pp" "github.com/stretchr/testify/assert" "github.com/commitdev/zero/internal/module" @@ -44,6 +45,17 @@ func TestParseModuleConfig(t *testing.T) { } assert.Equal(t, "platform", param.Field) assert.Equal(t, "CI Platform", param.Label) + assert.Equal(t, false, param.OmitFromProjectFile) + + useCredsParam, _ := findParameter(mod.Parameters, "useExistingAwsProfile") + assert.Equal(t, "useExistingAwsProfile", useCredsParam.Field) + assert.Equal(t, "Use credentials from an existing AWS profile?", useCredsParam.Label) + assert.Equal(t, true, useCredsParam.OmitFromProjectFile) + pp.Print(useCredsParam) + + awsCredsParam, _ := findParameter(mod.Parameters, "awsCredentials") + pp.Print(awsCredsParam) + }) t.Run("requiredCredentials are loaded", func(t *testing.T) { diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 7e7de8474..5ab043802 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -8,6 +8,12 @@ type Stack struct { func GetRegistry(path string) Registry { return Registry{ + { + "test", + []string{ + "/Users/davidcheung/projects/commit0/tests/test_data/modules/ci/", + }, + }, // TODO: better place to store these options as configuration file or any source { "EKS + Go + React + Gatsby", diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 2abe8b751..9caf67825 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -11,6 +11,11 @@ import ( "github.com/commitdev/zero/internal/config/globalconfig" ) +type AWSResourceConfig struct { + AccessKeyID string `yaml:"accessKeyId,omitempty" env:"AWS_ACCESS_KEY_ID,omitempty"` + SecretAccessKey string `yaml:"secretAccessKey,omitempty" env:"AWS_SECRET_ACCESS_KEY,omitempty"` +} + func AwsCredsPath() string { usr, err := user.Current() if err != nil { @@ -19,6 +24,17 @@ func AwsCredsPath() string { return filepath.Join(usr.HomeDir, ".aws/credentials") } +func FillAWSProfile(profileName string, paramsToFill map[string]string) error { + awsPath := AwsCredsPath() + awsCreds, err := credentials.NewSharedCredentials(awsPath, profileName).Get() + if err != nil { + return err + } + paramsToFill["accessKeyId"] = awsCreds.AccessKeyID + paramsToFill["secretAccessKey"] = awsCreds.SecretAccessKey + return nil +} + func GetAWSProfileProjectCredentials(profileName string, creds globalconfig.ProjectCredential) globalconfig.ProjectCredential { awsPath := AwsCredsPath() return GetAWSProfileCredentials(awsPath, profileName, creds) diff --git a/tests/test_data/modules/ci/zero-module.yml b/tests/test_data/modules/ci/zero-module.yml index 872ebdff8..364ca905a 100644 --- a/tests/test_data/modules/ci/zero-module.yml +++ b/tests/test_data/modules/ci/zero-module.yml @@ -31,3 +31,34 @@ parameters: options: - github - circlci + - field: circleci_api_key + label: "Circle CI API Key to setup your CI/CD for repositories" + conditions: + - action: KeyMatchCondition + matchField: platform + whenValue: "circlci" + - field: useExistingAwsProfile + label: "Use credentials from an existing AWS profile?" + options: + - "yes" + - "no" + omitFromProjectFile: yes + - field: profilePicker + omitFromProjectFile: yes + type: AWSProfilePicker + conditions: + - action: KeyMatchCondition + whenValue: "yes" + matchField: useExistingAwsProfile + - field: accessKeyId + label: AWS AccessKeyId + conditions: + - action: KeyMatchCondition + whenValue: "no" + matchField: useExistingAwsProfile + - field: secretAccessKey + label: AWS SecretAccessKey + conditions: + - action: KeyMatchCondition + whenValue: "no" + matchField: useExistingAwsProfile From 57917deec94f4a3b389d17f0c253a09352d3bb82 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 3 Mar 2021 16:31:06 -0500 Subject: [PATCH 02/18] remove credentials --- cmd/create.go | 4 +- internal/apply/apply.go | 5 - internal/config/globalconfig/global_config.go | 183 ------------------ .../config/globalconfig/global_config_test.go | 169 ---------------- internal/init/init.go | 120 ------------ internal/init/prompts.go | 31 --- pkg/credentials/credentials.go | 22 +-- pkg/credentials/credentials_test.go | 27 ++- 8 files changed, 23 insertions(+), 538 deletions(-) delete mode 100644 internal/config/globalconfig/global_config.go delete mode 100644 internal/config/globalconfig/global_config_test.go diff --git a/cmd/create.go b/cmd/create.go index 78c2066e4..ae810234f 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -5,7 +5,6 @@ import ( "path" "strings" - "github.com/commitdev/zero/internal/config/globalconfig" "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/internal/constants" "github.com/commitdev/zero/internal/generate" @@ -47,9 +46,8 @@ func Create(dir string, createConfigPath string) { if projectConfig.ShouldPushRepositories { flog.Infof(":up_arrow: Done Rendering - committing repositories to version control.") - globalConfig := globalconfig.GetProjectCredentials(projectConfig.Name) for _, module := range projectConfig.Modules { - vcs.InitializeRepository(module.Files.Repository, globalConfig.GithubResourceConfig.AccessToken) + vcs.InitializeRepository(module.Files.Repository, module.Parameters["github_access_token"]) } } else { flog.Infof(":up_arrow: Done Rendering - you will need to commit the created projects to version control.") diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 77ad4a2e3..02b00c735 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -13,7 +13,6 @@ import ( "github.com/commitdev/zero/internal/util" "github.com/hashicorp/terraform/dag" - "github.com/commitdev/zero/internal/config/globalconfig" "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" @@ -87,11 +86,7 @@ func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEn exit.Fatal("Failed to load module config, credentials cannot be injected properly") } - // Get project credentials for the makefile - credentials := globalconfig.GetProjectCredentials(projectConfig.Name) - credentialEnvs := credentials.SelectedVendorsCredentialsAsEnv(modConfig.RequiredCredentials) envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList) - envList = util.AppendProjectEnvToCmdEnv(credentialEnvs, envList) flog.Debugf("Env injected: %#v", envList) flog.Infof("Executing apply command for %s...", modConfig.Name) util.ExecuteCommand(exec.Command("make"), modulePath, envList) diff --git a/internal/config/globalconfig/global_config.go b/internal/config/globalconfig/global_config.go deleted file mode 100644 index e67377776..000000000 --- a/internal/config/globalconfig/global_config.go +++ /dev/null @@ -1,183 +0,0 @@ -package globalconfig - -import ( - "bytes" - "io/ioutil" - "log" - "os" - "os/user" - "path" - "reflect" - "strings" - - "github.com/commitdev/zero/internal/constants" - "github.com/commitdev/zero/internal/util" - "github.com/commitdev/zero/pkg/util/exit" - "github.com/commitdev/zero/pkg/util/flog" - yaml "gopkg.in/yaml.v2" -) - -var GetCredentialsPath = getCredentialsPath - -type ProjectCredentials map[string]ProjectCredential - -type ProjectCredential struct { - ProjectName string `yaml:"-"` - AWSResourceConfig `yaml:"aws,omitempty" vendor:"aws"` - GithubResourceConfig `yaml:"github,omitempty" vendor:"github"` - CircleCiResourceConfig `yaml:"circleci,omitempty" vendor:"circleci"` -} - -type AWSResourceConfig struct { - AccessKeyID string `yaml:"accessKeyId,omitempty" env:"AWS_ACCESS_KEY_ID,omitempty"` - SecretAccessKey string `yaml:"secretAccessKey,omitempty" env:"AWS_SECRET_ACCESS_KEY,omitempty"` -} -type GithubResourceConfig struct { - AccessToken string `yaml:"accessToken,omitempty" env:"GITHUB_ACCESS_TOKEN,omitempty"` -} -type CircleCiResourceConfig struct { - ApiKey string `yaml:"apiKey,omitempty" env:"CIRCLECI_API_KEY,omitempty"` -} - -func (p ProjectCredentials) Unmarshal(data []byte) error { - if len(data) == 0 { - return nil - } - err := yaml.NewDecoder(bytes.NewReader(data)).Decode(p) - if err != nil { - return err - } - for k, v := range p { - v.ProjectName = k - p[k] = v - } - return nil -} - -// AsEnvVars marshals ProjectCredential as a map of key/value strings suitable for environment variables -func (p ProjectCredential) AsEnvVars() map[string]string { - t := reflect.ValueOf(p) - - list := make(map[string]string) - list = gatherFieldTags(t, list) - - return list -} - -func gatherFieldTags(t reflect.Value, list map[string]string) map[string]string { - reflectType := t.Type() - - for i := 0; i < t.NumField(); i++ { - fieldValue := t.Field(i) - fieldType := reflectType.Field(i) - - if fieldType.Type.Kind() == reflect.Struct { - list = gatherFieldTags(fieldValue, list) - continue - } - - if env := fieldType.Tag.Get("env"); env != "" { - name, opts := parseTag(env) - if idx := strings.Index(opts, "omitempty"); idx != -1 && fieldValue.String() == "" { - continue - } - list[name] = fieldValue.String() - } - } - return list -} - -func (p ProjectCredential) SelectedVendorsCredentialsAsEnv(vendors []string) map[string]string { - t := reflect.ValueOf(p) - envs := map[string]string{} - for i := 0; i < t.NumField(); i++ { - childStruct := t.Type().Field(i) - childValue := t.Field(i) - if tag := childStruct.Tag.Get("vendor"); tag != "" && util.ItemInSlice(vendors, tag) { - envs = gatherFieldTags(childValue, envs) - } - } - return envs -} - -func parseTag(tag string) (string, string) { - if idx := strings.Index(tag, ","); idx != -1 { - return tag[:idx], tag[idx+1:] - } - return tag, "" -} - -func LoadUserCredentials() ProjectCredentials { - data := readOrCreateUserCredentialsFile() - - projects := ProjectCredentials{} - - err := projects.Unmarshal(data) - - if err != nil { - exit.Fatal("Failed to parse configuration: %v", err) - } - return projects -} - -func getCredentialsPath() string { - usr, err := user.Current() - if err != nil { - exit.Fatal("Failed to get user directory path: %v", err) - } - - rootDir := path.Join(usr.HomeDir, constants.ZeroHomeDirectory) - os.MkdirAll(rootDir, os.ModePerm) - filePath := path.Join(rootDir, constants.UserCredentialsYml) - return filePath -} - -func readOrCreateUserCredentialsFile() []byte { - credPath := GetCredentialsPath() - - _, fileStateErr := os.Stat(credPath) - if os.IsNotExist(fileStateErr) { - var file, fileStateErr = os.Create(credPath) - if fileStateErr != nil { - exit.Fatal("Failed to create config file: %v", fileStateErr) - } - flog.Debugf("Created credentials file: %s", credPath) - defer file.Close() - } - data, err := ioutil.ReadFile(credPath) - if err != nil { - exit.Fatal("Failed to read credentials file: %v", err) - } - flog.Debugf("Loaded credentials file: %s", credPath) - return data -} - -func GetProjectCredentials(targetProjectName string) ProjectCredential { - projects := LoadUserCredentials() - - if val, ok := projects[targetProjectName]; ok { - return val - } else { - p := ProjectCredential{ - ProjectName: targetProjectName, - } - projects[targetProjectName] = p - return p - } -} - -func Save(project ProjectCredential) { - projects := LoadUserCredentials() - projects[project.ProjectName] = project - flog.Debugf("Saved project credentials : %s", project.ProjectName) - writeCredentialsFile(projects) -} - -func writeCredentialsFile(projects ProjectCredentials) { - credsPath := GetCredentialsPath() - content, _ := yaml.Marshal(projects) - err := ioutil.WriteFile(credsPath, content, 0644) - if err != nil { - log.Panicf("failed to write config: %v", err) - } -} diff --git a/internal/config/globalconfig/global_config_test.go b/internal/config/globalconfig/global_config_test.go deleted file mode 100644 index eddfc8291..000000000 --- a/internal/config/globalconfig/global_config_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package globalconfig_test - -import ( - "fmt" - "io/ioutil" - "log" - "os" - "path" - "testing" - - "github.com/commitdev/zero/internal/config/globalconfig" - "github.com/stretchr/testify/assert" -) - -const baseTestFixturesDir = "../../../tests/test_data/configs/" - -var testCredentialFile = func() (func() string, func()) { - tmpConfigPath := getTmpConfig() - mockFunc := func() string { return tmpConfigPath } - teardownFunc := func() { os.RemoveAll(tmpConfigPath) } - return mockFunc, teardownFunc -} - -func getTmpConfig() string { - pathFrom := path.Join(baseTestFixturesDir, fmt.Sprintf("credentials%s.yml", "")) - pathTo := path.Join(baseTestFixturesDir, fmt.Sprintf("credentials%s.yml", "-tmp")) - copyFile(pathFrom, pathTo) - return pathTo -} - -func copyFile(from string, to string) { - bytesRead, err := ioutil.ReadFile(from) - if err != nil { - log.Fatal(err) - } - - err = ioutil.WriteFile(to, bytesRead, 0644) - if err != nil { - log.Fatal(err) - } -} -func TestReadOrCreateUserCredentialsFile(t *testing.T) { - globalconfig.GetCredentialsPath = func() string { - return path.Join(baseTestFixturesDir, "does-not-exist.yml") - } - credPath := globalconfig.GetCredentialsPath() - - defer os.RemoveAll(credPath) - _, fileStateErr := os.Stat(credPath) - assert.True(t, os.IsNotExist(fileStateErr), "File should not exist") - // attempting to read the file should create the file - globalconfig.GetProjectCredentials("any-project") - - stats, err := os.Stat(credPath) - assert.False(t, os.IsNotExist(err), "File should be created") - assert.Equal(t, "does-not-exist.yml", stats.Name(), "Should create yml automatically") -} - -func TestGetUserCredentials(t *testing.T) { - var teardownFn func() - globalconfig.GetCredentialsPath, teardownFn = testCredentialFile() - defer teardownFn() - - t.Run("Fixture file should have existing project with creds", func(t *testing.T) { - projectName := "my-project" - project := globalconfig.GetProjectCredentials(projectName) - - // Reading from fixtures: tests/test_data/configs/credentials.yml - assert.Equal(t, "AKIAABCD", project.AWSResourceConfig.AccessKeyID) - assert.Equal(t, "ZXCV", project.AWSResourceConfig.SecretAccessKey) - assert.Equal(t, "0987", project.GithubResourceConfig.AccessToken) - assert.Equal(t, "SOME_API_KEY", project.CircleCiResourceConfig.ApiKey) - }) - - t.Run("Fixture file should support multiple projects", func(t *testing.T) { - projectName := "another-project" - project := globalconfig.GetProjectCredentials(projectName) - assert.Equal(t, "654", project.GithubResourceConfig.AccessToken) - }) -} - -func TestEditUserCredentials(t *testing.T) { - var teardownFn func() - globalconfig.GetCredentialsPath, teardownFn = testCredentialFile() - defer teardownFn() - - t.Run("Should create new project if not exist", func(t *testing.T) { - projectName := "test-project3" - project := globalconfig.GetProjectCredentials(projectName) - project.AWSResourceConfig.AccessKeyID = "TEST_KEY_ID_1" - globalconfig.Save(project) - newKeyID := globalconfig.GetProjectCredentials(projectName).AWSResourceConfig.AccessKeyID - assert.Equal(t, "TEST_KEY_ID_1", newKeyID) - }) - t.Run("Should edit old project if already exist", func(t *testing.T) { - projectName := "my-project" - project := globalconfig.GetProjectCredentials(projectName) - project.AWSResourceConfig.AccessKeyID = "EDITED_ACCESS_KEY_ID" - globalconfig.Save(project) - newKeyID := globalconfig.GetProjectCredentials(projectName).AWSResourceConfig.AccessKeyID - assert.Equal(t, "EDITED_ACCESS_KEY_ID", newKeyID) - }) -} - -func TestMarshalProjectCredentialAsEnvVars(t *testing.T) { - t.Run("Should be able to marshal a ProjectCredential into env vars", func(t *testing.T) { - pc := globalconfig.ProjectCredential{ - AWSResourceConfig: globalconfig.AWSResourceConfig{ - AccessKeyID: "AKID", - SecretAccessKey: "SAK", - }, - CircleCiResourceConfig: globalconfig.CircleCiResourceConfig{ - ApiKey: "APIKEY", - }, - } - - envVars := pc.AsEnvVars() - assert.Equal(t, "AKID", envVars["AWS_ACCESS_KEY_ID"]) - assert.Equal(t, "SAK", envVars["AWS_SECRET_ACCESS_KEY"]) - assert.Equal(t, "APIKEY", envVars["CIRCLECI_API_KEY"]) - }) - - t.Run("should honor omitempty and left out empty values", func(t *testing.T) { - pc := globalconfig.ProjectCredential{} - - envVars := pc.AsEnvVars() - assert.Equal(t, 0, len(envVars)) - }) -} - -func TestMarshalSelectedVendorsCredentialsAsEnv(t *testing.T) { - pc := globalconfig.ProjectCredential{ - AWSResourceConfig: globalconfig.AWSResourceConfig{ - AccessKeyID: "AKID", - SecretAccessKey: "SAK", - }, - GithubResourceConfig: globalconfig.GithubResourceConfig{ - AccessToken: "FOOBAR", - }, - CircleCiResourceConfig: globalconfig.CircleCiResourceConfig{ - ApiKey: "APIKEY", - }, - } - - t.Run("cherry pick credentials by vendor", func(t *testing.T) { - envs := pc.SelectedVendorsCredentialsAsEnv([]string{"aws", "github"}) - assert.Equal(t, "AKID", envs["AWS_ACCESS_KEY_ID"]) - assert.Equal(t, "SAK", envs["AWS_SECRET_ACCESS_KEY"]) - assert.Equal(t, "FOOBAR", envs["GITHUB_ACCESS_TOKEN"]) - }) - - t.Run("omits vendors not selected", func(t *testing.T) { - envs := pc.SelectedVendorsCredentialsAsEnv([]string{"github"}) - assert.Equal(t, "FOOBAR", envs["GITHUB_ACCESS_TOKEN"]) - - _, hasAWSKeyID := envs["AWS_ACCESS_KEY_ID"] - assert.Equal(t, false, hasAWSKeyID) - _, hasAWSSecretAccessKey := envs["AWS_SECRET_ACCESS_KEY"] - assert.Equal(t, false, hasAWSSecretAccessKey) - _, hasCircleCIKey := envs["CIRCLECI_API_KEY"] - assert.Equal(t, false, hasCircleCIKey) - }) - - t.Run("omits vendors not selected", func(t *testing.T) { - envs := pc.SelectedVendorsCredentialsAsEnv([]string{}) - assert.Equal(t, 0, len(envs)) - }) - -} diff --git a/internal/init/init.go b/internal/init/init.go index fe0ba001f..af1415122 100644 --- a/internal/init/init.go +++ b/internal/init/init.go @@ -6,13 +6,10 @@ import ( "path" "sync" - "github.com/commitdev/zero/internal/config/globalconfig" "github.com/commitdev/zero/internal/config/moduleconfig" "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/internal/module" "github.com/commitdev/zero/internal/registry" - "github.com/commitdev/zero/internal/util" - project "github.com/commitdev/zero/pkg/credentials" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" "github.com/manifoldco/promptui" @@ -151,123 +148,6 @@ func getProjectPrompts(projectName string, modules map[string]moduleconfig.Modul return handlers } -func getCredentialPrompts(projectCredentials globalconfig.ProjectCredential, moduleConfigs map[string]moduleconfig.ModuleConfig) []CredentialPrompts { - var uniqueVendors []string - for _, module := range moduleConfigs { - uniqueVendors = appendToSet(uniqueVendors, module.RequiredCredentials) - } - - // map is to keep track of which vendor they belong to, to fill them back into the projectConfig - prompts := []CredentialPrompts{} - for _, vendor := range AvailableVendorOrders { - if util.ItemInSlice(uniqueVendors, vendor) { - vendorPrompts := CredentialPrompts{vendor, mapVendorToPrompts(projectCredentials, vendor)} - prompts = append(prompts, vendorPrompts) - } - } - return prompts -} - -func mapVendorToPrompts(projectCred globalconfig.ProjectCredential, vendor string) []PromptHandler { - var prompts []PromptHandler - profiles, err := project.GetAWSProfiles() - if err != nil { - profiles = []string{} - } - - // if no profiles available, dont prompt use to pick profile - customAwsPickProfileCondition := func(param map[string]string) bool { - if len(profiles) == 0 { - flog.Infof(":warning: No AWS profiles found, please manually input AWS credentials") - return false - } else { - return true - } - } - - // condition for prompting manual AWS credentials input - customAwsMustInputCondition := func(param map[string]string) bool { - toPickProfile := awsPickProfile - if val, ok := param["use_aws_profile"]; ok && val != toPickProfile { - return true - } - return false - } - - switch vendor { - case "aws": - awsPrompts := []PromptHandler{ - { - Parameter: moduleconfig.Parameter{ - Field: "use_aws_profile", - Label: "Use credentials from existing AWS profiles?", - Options: []string{awsPickProfile, awsManualInputCredentials}, - }, - Condition: customAwsPickProfileCondition, - Validate: NoValidation, - }, - { - Parameter: moduleconfig.Parameter{ - Field: "aws_profile", - Label: "Select AWS Profile", - Options: profiles, - }, - Condition: KeyMatchCondition("use_aws_profile", awsPickProfile), - Validate: NoValidation, - }, - { - Parameter: moduleconfig.Parameter{ - Field: "accessKeyId", - Label: "AWS Access Key ID", - Default: projectCred.AWSResourceConfig.AccessKeyID, - Info: `AWS Access Key ID/Secret: used for provisioning infrastructure in AWS -The token can be generated at https://console.aws.amazon.com/iam/home?#/security_credentials`, - }, - Condition: CustomCondition(customAwsMustInputCondition), - Validate: ValidateAKID, - }, - { - Parameter: moduleconfig.Parameter{ - Field: "secretAccessKey", - Label: "AWS Secret access key", - Default: projectCred.AWSResourceConfig.SecretAccessKey, - }, - Condition: CustomCondition(customAwsMustInputCondition), - Validate: ValidateSAK, - }, - } - prompts = append(prompts, awsPrompts...) - case "github": - githubPrompt := PromptHandler{ - Parameter: moduleconfig.Parameter{ - Field: "accessToken", - Label: "Github Personal Access Token with access to the above organization", - Default: projectCred.GithubResourceConfig.AccessToken, - Info: `Github personal access token: used for creating repositories for your project -Requires the following permissions: [repo::public_repo, admin::orgread:org] -The token can be created at https://github.com/settings/tokens`, - }, - Condition: NoCondition, - Validate: NoValidation, - } - prompts = append(prompts, githubPrompt) - case "circleci": - circleCiPrompt := PromptHandler{ - Parameter: moduleconfig.Parameter{ - Field: "apiKey", - Label: "Circleci api key for CI/CD", - Default: projectCred.CircleCiResourceConfig.ApiKey, - Info: `CircleCI api token: used for setting up CI/CD for your project -The token can be created at https://app.circleci.com/settings/user/tokens`, - }, - Condition: NoCondition, - Validate: NoValidation, - } - prompts = append(prompts, circleCiPrompt) - } - return prompts -} - func chooseCloudProvider(projectConfig *projectconfig.ZeroProjectConfig) { // @TODO move options into configs providerPrompt := promptui.Select{ diff --git a/internal/init/prompts.go b/internal/init/prompts.go index 3cbd8294c..abbde2bec 100644 --- a/internal/init/prompts.go +++ b/internal/init/prompts.go @@ -9,16 +9,13 @@ import ( "regexp" "strings" - "github.com/commitdev/zero/internal/config/globalconfig" "github.com/commitdev/zero/internal/config/moduleconfig" "github.com/commitdev/zero/internal/constants" "github.com/commitdev/zero/internal/util" - "github.com/commitdev/zero/pkg/credentials" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" "github.com/k0kubun/pp" "github.com/manifoldco/promptui" - "gopkg.in/yaml.v2" ) // Constant to maintain prompt orders so users can have the same flow, @@ -278,34 +275,6 @@ func conditionHandler(cond moduleconfig.Condition) CustomConditionSignature { } } -func promptCredentialsAndFillProjectCreds(credentialPrompts []CredentialPrompts, creds globalconfig.ProjectCredential) globalconfig.ProjectCredential { - promptsValues := map[string]map[string]string{} - - for _, prompts := range credentialPrompts { - vendor := prompts.Vendor - vendorPromptValues := map[string]string{} - - // vendors like AWS have multiple prompts (accessKeyId and secretAccessKey) - for _, prompt := range prompts.Prompts { - prompt.RunPrompt(vendorPromptValues) - } - promptsValues[vendor] = vendorPromptValues - } - - // FIXME: what is a good way to dynamically modify partial data of a struct - // current just marashing to yaml, then unmarshaling into the base struct - yamlContent, _ := yaml.Marshal(promptsValues) - yaml.Unmarshal(yamlContent, &creds) - - // Fill AWS credentials based on profile from ~/.aws/credentials - if val, ok := promptsValues["aws"]; ok { - if val["use_aws_profile"] == awsPickProfile { - creds = credentials.GetAWSProfileProjectCredentials(val["aws_profile"], creds) - } - } - return creds -} - func appendToSet(set []string, toAppend []string) []string { for _, appendee := range toAppend { if !util.ItemInSlice(set, appendee) { diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 9caf67825..3f96981e0 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -8,7 +8,6 @@ import ( "regexp" "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/commitdev/zero/internal/config/globalconfig" ) type AWSResourceConfig struct { @@ -16,7 +15,9 @@ type AWSResourceConfig struct { SecretAccessKey string `yaml:"secretAccessKey,omitempty" env:"AWS_SECRET_ACCESS_KEY,omitempty"` } -func AwsCredsPath() string { +var GetAWSCredsPath = awsCredsPath + +func awsCredsPath() string { usr, err := user.Current() if err != nil { log.Fatal(err) @@ -25,7 +26,7 @@ func AwsCredsPath() string { } func FillAWSProfile(profileName string, paramsToFill map[string]string) error { - awsPath := AwsCredsPath() + awsPath := GetAWSCredsPath() awsCreds, err := credentials.NewSharedCredentials(awsPath, profileName).Get() if err != nil { return err @@ -35,21 +36,6 @@ func FillAWSProfile(profileName string, paramsToFill map[string]string) error { return nil } -func GetAWSProfileProjectCredentials(profileName string, creds globalconfig.ProjectCredential) globalconfig.ProjectCredential { - awsPath := AwsCredsPath() - return GetAWSProfileCredentials(awsPath, profileName, creds) -} - -func GetAWSProfileCredentials(credsPath string, profileName string, creds globalconfig.ProjectCredential) globalconfig.ProjectCredential { - awsCreds, err := credentials.NewSharedCredentials(credsPath, profileName).Get() - if err != nil { - log.Fatal(err) - } - creds.AWSResourceConfig.AccessKeyID = awsCreds.AccessKeyID - creds.AWSResourceConfig.SecretAccessKey = awsCreds.SecretAccessKey - return creds -} - // GetAWSProfiles returns a list of AWS forprofiles set up on the user's sytem func GetAWSProfiles() ([]string, error) { usr, err := user.Current() diff --git a/pkg/credentials/credentials_test.go b/pkg/credentials/credentials_test.go index 9c9e3c62c..7b240d646 100644 --- a/pkg/credentials/credentials_test.go +++ b/pkg/credentials/credentials_test.go @@ -3,24 +3,33 @@ package credentials_test import ( "testing" - "github.com/commitdev/zero/internal/config/globalconfig" "github.com/commitdev/zero/pkg/credentials" "github.com/stretchr/testify/assert" ) func TestFillAWSProfileCredentials(t *testing.T) { mockAwsCredentialFilePath := "../../tests/test_data/aws/mock_credentials.yml" + credentials.GetAWSCredsPath = func() string { + return mockAwsCredentialFilePath + } t.Run("fills project credentials", func(t *testing.T) { - projectCreds := globalconfig.ProjectCredential{} - projectCreds = credentials.GetAWSProfileCredentials(mockAwsCredentialFilePath, "default", projectCreds) - assert.Equal(t, "MOCK1_ACCESS_KEY", projectCreds.AWSResourceConfig.AccessKeyID) - assert.Equal(t, "MOCK1_SECRET_ACCESS_KEY", projectCreds.AWSResourceConfig.SecretAccessKey) + params := map[string]string{} + err := credentials.FillAWSProfile("default", params) + if err != nil { + panic(err) + } + + assert.Equal(t, "MOCK1_ACCESS_KEY", params["accessKeyId"]) + assert.Equal(t, "MOCK1_SECRET_ACCESS_KEY", params["secretAccessKey"]) }) t.Run("supports non-default profiles", func(t *testing.T) { - projectCreds := globalconfig.ProjectCredential{} - projectCreds = credentials.GetAWSProfileCredentials(mockAwsCredentialFilePath, "foobar", projectCreds) - assert.Equal(t, "MOCK2_ACCESS_KEY", projectCreds.AWSResourceConfig.AccessKeyID) - assert.Equal(t, "MOCK2_SECRET_ACCESS_KEY", projectCreds.AWSResourceConfig.SecretAccessKey) + params := map[string]string{} + err := credentials.FillAWSProfile("foobar", params) + if err != nil { + panic(err) + } + assert.Equal(t, "MOCK2_ACCESS_KEY", params["accessKeyId"]) + assert.Equal(t, "MOCK2_SECRET_ACCESS_KEY", params["secretAccessKey"]) }) } From cd0fd51589ad58e39c5978b992a2d9a2320c9495 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 3 Mar 2021 18:23:06 -0500 Subject: [PATCH 03/18] dynamically populate from struct --- internal/util/util.go | 32 ++++++++++++++++++++++++++++++++ pkg/credentials/credentials.go | 18 +++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/internal/util/util.go b/internal/util/util.go index 3c22e996a..f2888bb11 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -10,6 +10,7 @@ import ( "os/exec" "path" "path/filepath" + "reflect" "strconv" "strings" "text/template" @@ -142,3 +143,34 @@ func ItemInSlice(slice []string, target string) bool { } return false } + +// Given a resource of struct type as +// type AWSCreds struct{ +// AccessKeyID string `yaml:"accessKeyId,omitempty"` +// SecretAccessKey string `yaml:"accessKeyId,omitempty"` +// }{ +// AccessKeyID: "FOO", +// SecretAccessKey: "BAR", +// } +// It will base on the tag, fill in the value to supplied map[string]string +func ReflectStructValueIntoMap(resource interface{}, tagName string, paramsToFill map[string]string) { + t := reflect.ValueOf(resource) + + for i := 0; i < t.NumField(); i++ { + + childStruct := t.Type().Field(i) + childValue := t.Field(i) + if childValue.Kind().String() != "string" { + continue + } + tag, _ := parseTag(childStruct.Tag.Get(tagName)) + paramsToFill[tag] = childValue.String() + } +} + +func parseTag(tag string) (string, string) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tag[idx+1:] + } + return tag, "" +} diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 3f96981e0..5786b72d7 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -8,6 +8,7 @@ import ( "regexp" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/commitdev/zero/internal/util" ) type AWSResourceConfig struct { @@ -25,14 +26,25 @@ func awsCredsPath() string { return filepath.Join(usr.HomeDir, ".aws/credentials") } +func fetchAWSConfig(awsPath string, profileName string) (error, AWSResourceConfig) { + + awsCreds, err := credentials.NewSharedCredentials(awsPath, profileName).Get() + if err != nil { + return err, AWSResourceConfig{} + } + return nil, AWSResourceConfig{ + AccessKeyID: awsCreds.AccessKeyID, + SecretAccessKey: awsCreds.SecretAccessKey, + } +} + func FillAWSProfile(profileName string, paramsToFill map[string]string) error { awsPath := GetAWSCredsPath() - awsCreds, err := credentials.NewSharedCredentials(awsPath, profileName).Get() + err, awsCreds := fetchAWSConfig(awsPath, profileName) if err != nil { return err } - paramsToFill["accessKeyId"] = awsCreds.AccessKeyID - paramsToFill["secretAccessKey"] = awsCreds.SecretAccessKey + util.ReflectStructValueIntoMap(awsCreds, "yaml", paramsToFill) return nil } From 7713479b71d1ffce5f454aabc3f4929354ee10cd Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 3 Mar 2021 19:08:19 -0500 Subject: [PATCH 04/18] allow overwriting the env-var while apply --- cmd/create.go | 6 +- internal/apply/apply.go | 20 +++- internal/apply/apply_test.go | 7 ++ internal/config/moduleconfig/module_config.go | 25 ++--- .../config/projectconfig/project_config.go | 16 ++++ internal/constants/constants.go | 13 ++- internal/init/custom-prompts.go | 3 - internal/init/init.go | 9 +- internal/init/prompts.go | 26 +++--- internal/init/prompts_test.go | 93 ++++++++++++++++++- internal/module/module_test.go | 39 +++++--- internal/registry/registry.go | 6 -- internal/util/util.go | 10 +- tests/test_data/apply/project1/Makefile | 1 + .../test_data/apply/project1/zero-module.yml | 2 + tests/test_data/apply/zero-project.yml | 1 + tests/test_data/modules/ci/zero-module.yml | 4 + 17 files changed, 214 insertions(+), 67 deletions(-) diff --git a/cmd/create.go b/cmd/create.go index ae810234f..e67a79ce4 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -47,7 +47,11 @@ func Create(dir string, createConfigPath string) { flog.Infof(":up_arrow: Done Rendering - committing repositories to version control.") for _, module := range projectConfig.Modules { - vcs.InitializeRepository(module.Files.Repository, module.Parameters["github_access_token"]) + err, githubApiKey := module.ReadVendorCredentials("github") + if err != nil { + flog.Errorf(err.Error()) + } + vcs.InitializeRepository(module.Files.Repository, githubApiKey) } } else { flog.Infof(":up_arrow: Done Rendering - you will need to commit the created projects to version control.") diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 02b00c735..841ae4fa4 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -13,6 +13,7 @@ import ( "github.com/commitdev/zero/internal/util" "github.com/hashicorp/terraform/dag" + "github.com/commitdev/zero/internal/config/moduleconfig" "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" @@ -86,7 +87,8 @@ func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEn exit.Fatal("Failed to load module config, credentials cannot be injected properly") } - envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList) + envVarTranslationMap := modConfig.GetParamEnvVarTranslationMap() + envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList, envVarTranslationMap) flog.Debugf("Env injected: %#v", envList) flog.Infof("Executing apply command for %s...", modConfig.Name) util.ExecuteCommand(exec.Command("make"), modulePath, envList) @@ -94,6 +96,15 @@ func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEn }) } +func getParameterDefinition(modConfig moduleconfig.ModuleConfig, field string) moduleconfig.Parameter { + for i := 0; i < len(modConfig.Parameters); i++ { + if field == modConfig.Parameters[i].Field { + return modConfig.Parameters[i] + } + } + return moduleconfig.Parameter{} +} + // promptEnvironments Prompts the user for the environments to apply against and returns a slice of strings representing the environments func promptEnvironments() []string { items := map[string][]string{ @@ -155,7 +166,12 @@ func summarizeAll(dir string, projectConfig projectconfig.ZeroProjectConfig, app } flog.Debugf("Loaded module: %s from %s", name, modulePath) - envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList) + modConfig, err := module.ParseModuleConfig(modulePath) + if err != nil { + exit.Fatal("Failed to load module config, credentials cannot be injected properly") + } + envVarTranslationMap := modConfig.GetParamEnvVarTranslationMap() + envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList, envVarTranslationMap) flog.Debugf("Env injected: %#v", envList) util.ExecuteCommand(exec.Command("make", "summary"), modulePath, envList) return nil diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index a88daee54..a4c1b41e4 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -38,4 +38,11 @@ func TestApply(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "baz: qux\n", string(content)) }) + + t.Run("Zero apply honors the envVarName overwrite from module definition", func(t *testing.T) { + content, err := ioutil.ReadFile(filepath.Join(tmpDir, "project1/feature.out")) + assert.NoError(t, err) + assert.Equal(t, "envVarName of viaEnvVarName: baz\n", string(content)) + }) + } diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index c8d8e6036..2b781f5bf 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -12,7 +12,6 @@ import ( "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/pkg/util/flog" "github.com/iancoleman/strcase" - "github.com/pkg/errors" ) type ModuleConfig struct { @@ -38,12 +37,13 @@ type Parameter struct { Type string `yaml:"type,omitempty"` OmitFromProjectFile bool `yaml:"omitFromProjectFile,omitempty"` Conditions []Condition `yaml:"conditions,omitempty"` + EnvVarName string `yaml:"envVarName,omitempty"` } type Condition struct { Action string `yaml:"action"` MatchField string `yaml:"matchField"` - WhenValue string `yaml:"whenValue,omitempty"` + WhenValue string `yaml:"whenValue"` Data []string `yaml:"data,omitempty"` } @@ -68,6 +68,17 @@ func (cfg ModuleConfig) collectMissing() []string { return missing } +func (cfg ModuleConfig) GetParamEnvVarTranslationMap() map[string]string { + translationMap := make(map[string]string) + for i := 0; i < len(cfg.Parameters); i++ { + param := cfg.Parameters[i] + if param.EnvVarName != "" { + translationMap[param.Field] = param.EnvVarName + } + } + return translationMap +} + func LoadModuleConfig(filePath string) (ModuleConfig, error) { config := ModuleConfig{} @@ -81,7 +92,6 @@ func LoadModuleConfig(filePath string) (ModuleConfig, error) { return config, err } - validateParams(config.Parameters) missing := config.collectMissing() if len(missing) > 0 { flog.Errorf("%v is missing information", filePath) @@ -96,15 +106,6 @@ func LoadModuleConfig(filePath string) (ModuleConfig, error) { return config, nil } -func validateParams(params []Parameter) error { - for _, param := range params { - if param.Type != "" { - return errors.Errorf("type is not supported") - } - } - return nil -} - // Recurses through a datastructure to find any missing data. // This assumes several things: // 1. The structure matches that defined by ModuleConfig and its child datastructures. diff --git a/internal/config/projectconfig/project_config.go b/internal/config/projectconfig/project_config.go index 445e90387..fce86f6fa 100644 --- a/internal/config/projectconfig/project_config.go +++ b/internal/config/projectconfig/project_config.go @@ -1,6 +1,7 @@ package projectconfig import ( + "errors" "io/ioutil" "log" @@ -29,6 +30,21 @@ type Module struct { Conditions []Condition `yaml:"conditions,omitempty"` } +func (m Module) ReadVendorCredentials(vendor string) (error, string) { + // this mapping could be useful for module config as well + vendorToParamMap := map[string]string{ + "github": "githubAccessToken", + "circleci": "circleciApiKey", + } + if parameterKey, ok := vendorToParamMap[vendor]; ok { + if val, ok := m.Parameters[parameterKey]; ok { + return nil, val + } + return errors.New("Parameter not found in module."), "" + } + return errors.New("Unsupported vendor provided."), "" +} + type Parameters map[string]string type Condition struct { diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 988474359..60368145c 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -1,13 +1,12 @@ package constants const ( - TemplatesDir = "tmp/templates" - ZeroProjectYml = "zero-project.yml" - ZeroModuleYml = "zero-module.yml" - UserCredentialsYml = "credentials.yml" - ZeroHomeDirectory = ".zero" - IgnoredPaths = "(?i)zero.module.yml|.git/" - TemplateExtn = ".tmpl" + TemplatesDir = "tmp/templates" + ZeroProjectYml = "zero-project.yml" + ZeroModuleYml = "zero-module.yml" + ZeroHomeDirectory = ".zero" + IgnoredPaths = "(?i)zero.module.yml|.git/" + TemplateExtn = ".tmpl" // prompt constants diff --git a/internal/init/custom-prompts.go b/internal/init/custom-prompts.go index 3da7931c7..ec649c2f6 100644 --- a/internal/init/custom-prompts.go +++ b/internal/init/custom-prompts.go @@ -4,7 +4,6 @@ import ( "github.com/commitdev/zero/internal/config/moduleconfig" project "github.com/commitdev/zero/pkg/credentials" "github.com/commitdev/zero/pkg/util/flog" - "github.com/k0kubun/pp" ) func CustomPromptHandler(promptType string, params map[string]string) { @@ -31,11 +30,9 @@ func AWSProfilePicker(params map[string]string) { Validate: NoValidation, } _, value := promptParameter(awsPrompt) - pp.Print(value) credErr := project.FillAWSProfile(value, params) if credErr != nil { flog.Errorf("Failed to retrieve profile, falling back to User input") params["useExistingAwsProfile"] = "no" } - pp.Print(params) } diff --git a/internal/init/init.go b/internal/init/init.go index af1415122..a6bcf227d 100644 --- a/internal/init/init.go +++ b/internal/init/init.go @@ -20,8 +20,9 @@ func Init(outDir string, localModulePath string) *projectconfig.ZeroProjectConfi projectConfig := defaultProjConfig() projectRootParams := map[string]string{} + emptyEnvVarTranslationMap := map[string]string{} promptName := getProjectNamePrompt() - promptName.RunPrompt(projectRootParams) + promptName.RunPrompt(projectRootParams, emptyEnvVarTranslationMap) projectConfig.Name = projectRootParams[promptName.Field] rootDir := path.Join(outDir, projectConfig.Name) @@ -41,19 +42,19 @@ func Init(outDir string, localModulePath string) *projectconfig.ZeroProjectConfi initParams := make(map[string]string) projectConfig.ShouldPushRepositories = true - prompts["ShouldPushRepositories"].RunPrompt(initParams) + prompts["ShouldPushRepositories"].RunPrompt(initParams, emptyEnvVarTranslationMap) if initParams["ShouldPushRepositories"] == "n" { projectConfig.ShouldPushRepositories = false } // Prompting for push-up stream, then conditionally prompting for github - prompts["GithubRootOrg"].RunPrompt(initParams) + prompts["GithubRootOrg"].RunPrompt(initParams, emptyEnvVarTranslationMap) projectData := promptAllModules(moduleConfigs) // Map parameter values back to specific modules for moduleName, module := range moduleConfigs { - prompts[moduleName].RunPrompt(initParams) + prompts[moduleName].RunPrompt(initParams, emptyEnvVarTranslationMap) repoName := initParams[prompts[moduleName].Field] repoURL := fmt.Sprintf("%s/%s", initParams["GithubRootOrg"], repoName) projectModuleParams := moduleconfig.SummarizeParameters(module, projectData) diff --git a/internal/init/prompts.go b/internal/init/prompts.go index abbde2bec..2698ff2dc 100644 --- a/internal/init/prompts.go +++ b/internal/init/prompts.go @@ -14,7 +14,6 @@ import ( "github.com/commitdev/zero/internal/util" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" - "github.com/k0kubun/pp" "github.com/manifoldco/promptui" ) @@ -107,7 +106,7 @@ func ValidateProjectName(input string) error { // 1. Execute (this could potentially be refactored into type + data) // 2. type: specific ways of obtaining values (in AWS credential case it will set 2 values to the map) //3. -func (p PromptHandler) RunPrompt(projectParams map[string]string) { +func (p PromptHandler) RunPrompt(projectParams map[string]string, envVarTranslationMap map[string]string) { var err error var result string @@ -120,7 +119,7 @@ func (p PromptHandler) RunPrompt(projectParams map[string]string) { // so if community module has an `execute: twitter tweet $ENV` // it wouldnt leak things the module shouldnt have access to if p.Parameter.Execute != "" { - result = executeCmd(p.Parameter.Execute, projectParams) + result = executeCmd(p.Parameter.Execute, projectParams, envVarTranslationMap) } else if p.Parameter.Type != "" { CustomPromptHandler(p.Parameter.Type, projectParams) } else if p.Parameter.Value != "" { @@ -172,10 +171,11 @@ func promptParameter(prompt PromptHandler) (error, string) { return nil, result } -func executeCmd(command string, envVars map[string]string) string { - pp.Print(envVars) +func executeCmd(command string, envVars map[string]string, envVarTranslationMap map[string]string) string { cmd := exec.Command("bash", "-c", command) - cmd.Env = util.AppendProjectEnvToCmdEnv(envVars, os.Environ()) + // Might need to pass down module's translation map as well, + // currently only works in `zero apply` + cmd.Env = util.AppendProjectEnvToCmdEnv(envVars, os.Environ(), envVarTranslationMap) out, err := cmd.Output() flog.Debugf("Running command: %s", command) if err != nil { @@ -193,7 +193,7 @@ func sanitizeParameterValue(str string) string { // PromptParams renders series of prompt UI based on the config func PromptModuleParams(moduleConfig moduleconfig.ModuleConfig, parameters map[string]string) (map[string]string, error) { - + envVarTranslationMap := moduleConfig.GetParamEnvVarTranslationMap() for _, parameter := range moduleConfig.Parameters { // deduplicate fields already prompted and received if _, isAlreadySet := parameters[parameter.Field]; isAlreadySet { @@ -225,12 +225,9 @@ func PromptModuleParams(moduleConfig moduleconfig.ModuleConfig, parameters map[s // for k, v := range parameters { // credentialEnvs[k] = v // } - promptHandler.RunPrompt(parameters) - - // parameters[parameter.Field] = result - pp.Print("Module is done processing %s", moduleConfig.Name) - pp.Print(parameters) + promptHandler.RunPrompt(parameters, envVarTranslationMap) } + flog.Debugf("Module %s prompt: \n %#v", moduleConfig.Name, parameters) return parameters, nil } @@ -257,8 +254,9 @@ func paramConditionsMapper(conditions []moduleconfig.Condition) CustomConditionS return func(params map[string]string) bool { // Prompts must pass every condition to proceed for i := 0; i < len(conditions); i++ { - if !conditionHandler(conditions[i])(params) { - flog.Debugf("Did not meet condition %v, expected %v to be %v", conditions[i].Action, conditions[i].MatchField, conditions[i].WhenValue) + cond := conditions[i] + if !conditionHandler(cond)(params) { + flog.Debugf("Did not meet condition %v, expected %v to be %v", cond.Action, cond.MatchField, cond.WhenValue) return false } } diff --git a/internal/init/prompts_test.go b/internal/init/prompts_test.go index c2af1bda5..190cf1706 100644 --- a/internal/init/prompts_test.go +++ b/internal/init/prompts_test.go @@ -12,6 +12,7 @@ import ( func TestGetParam(t *testing.T) { + envVarTranslationMap := map[string]string{} projectParams := map[string]string{} t.Run("Should execute params without prompt", func(t *testing.T) { param := moduleconfig.Parameter{ @@ -25,7 +26,7 @@ func TestGetParam(t *testing.T) { initPrompts.NoValidation, } - prompt.RunPrompt(projectParams) + prompt.RunPrompt(projectParams, envVarTranslationMap) assert.Equal(t, "my-acconut-id", projectParams[param.Field]) }) @@ -42,7 +43,7 @@ func TestGetParam(t *testing.T) { } projectParams := map[string]string{"INJECTEDENV": "SOME_ENV_VAR_VALUE"} - prompt.RunPrompt(projectParams) + prompt.RunPrompt(projectParams, envVarTranslationMap) assert.Equal(t, "SOME_ENV_VAR_VALUE", projectParams[param.Field]) }) @@ -58,7 +59,7 @@ func TestGetParam(t *testing.T) { initPrompts.NoValidation, } - prompt.RunPrompt(projectParams) + prompt.RunPrompt(projectParams, envVarTranslationMap) assert.Equal(t, "lorem-ipsum", projectParams[param.Field]) }) @@ -77,8 +78,92 @@ func TestGetParam(t *testing.T) { initPrompts.NoValidation, } - prompt.RunPrompt(projectParams) + prompt.RunPrompt(projectParams, envVarTranslationMap) assert.Equal(t, "foo", projectParams["existing_value"]) assert.Equal(t, "bar", projectParams[param.Field]) }) + + t.Run("Prompt to apply in order and allow EnvVarMapping", func(t *testing.T) { + + projectParams = map[string]string{} + params := []moduleconfig.Parameter{ + { + Field: "param1", + Value: "foo", + EnvVarName: "envvar1", + }, + { + Field: "param2", + Execute: "echo $envvar1 bar", + EnvVarName: "envvar2", + }, + { + Field: "param3", + Execute: "echo $envvar2 baz", + }, + } + module := moduleconfig.ModuleConfig{Parameters: params} + + projectParams, _ = initPrompts.PromptModuleParams(module, projectParams) + + assert.Equal(t, "foo", projectParams["param1"]) + assert.Equal(t, "foo bar", projectParams["param2"], "should reference param1 via env-var") + assert.Equal(t, "foo bar baz", projectParams["param3"], "should reference param2 via env-var") + }) + + t.Run("Prompt conditions", func(t *testing.T) { + + projectParams = map[string]string{} + params := []moduleconfig.Parameter{ + { + Field: "param1", + Value: "pass", + }, + { + Field: "passing_condition", + Value: "pass", + Conditions: []moduleconfig.Condition{ + { + Action: "KeyMatchCondition", + WhenValue: "pass", + MatchField: "param1", + }, + }, + }, + { + Field: "failing_condition", + Value: "pass", + Conditions: []moduleconfig.Condition{ + { + Action: "KeyMatchCondition", + WhenValue: "not foo", + MatchField: "param1", + }, + }, + }, + { + Field: "multiple_condition", + Value: "pass", + Conditions: []moduleconfig.Condition{ + { + Action: "KeyMatchCondition", + WhenValue: "pass", + MatchField: "param1", + }, + { + Action: "KeyMatchCondition", + WhenValue: "pass", + MatchField: "passing_condition", + }, + }, + }, + } + module := moduleconfig.ModuleConfig{Parameters: params} + projectParams, _ = initPrompts.PromptModuleParams(module, projectParams) + + assert.Equal(t, "pass", projectParams["param1"], "Value just hardcoded") + assert.Equal(t, "pass", projectParams["passing_condition"], "Expected to pass condition and set value") + assert.NotContains(t, projectParams, "failing_condition", "Expected to fail condition and not set value") + assert.Equal(t, "pass", projectParams["multiple_condition"], "Expected to pass multiple condition and set value") + }) } diff --git a/internal/module/module_test.go b/internal/module/module_test.go index b62c5eaf8..f0f55a50a 100644 --- a/internal/module/module_test.go +++ b/internal/module/module_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/commitdev/zero/internal/config/moduleconfig" - "github.com/k0kubun/pp" "github.com/stretchr/testify/assert" "github.com/commitdev/zero/internal/module" @@ -45,21 +44,39 @@ func TestParseModuleConfig(t *testing.T) { } assert.Equal(t, "platform", param.Field) assert.Equal(t, "CI Platform", param.Label) - assert.Equal(t, false, param.OmitFromProjectFile) - useCredsParam, _ := findParameter(mod.Parameters, "useExistingAwsProfile") - assert.Equal(t, "useExistingAwsProfile", useCredsParam.Field) - assert.Equal(t, "Use credentials from an existing AWS profile?", useCredsParam.Label) - assert.Equal(t, true, useCredsParam.OmitFromProjectFile) - pp.Print(useCredsParam) + }) - awsCredsParam, _ := findParameter(mod.Parameters, "awsCredentials") - pp.Print(awsCredsParam) + t.Run("OmitFromProjectFile default", func(t *testing.T) { + param, err := findParameter(mod.Parameters, "platform") + if err != nil { + panic(err) + } + assert.Equal(t, false, param.OmitFromProjectFile, "OmitFromProjectFile should default to false") + useCredsParam, useCredsErr := findParameter(mod.Parameters, "useExistingAwsProfile") + if useCredsErr != nil { + panic(useCredsErr) + } + assert.Equal(t, true, useCredsParam.OmitFromProjectFile, "OmitFromProjectFile should be read from file") + }) + t.Run("Parsing Conditions and Typed prompts from config", func(t *testing.T) { + param, err := findParameter(mod.Parameters, "profilePicker") + if err != nil { + panic(err) + } + assert.Equal(t, "AWSProfilePicker", param.Type) + assert.Equal(t, "KeyMatchCondition", param.Conditions[0].Action) + assert.Equal(t, "useExistingAwsProfile", param.Conditions[0].MatchField) + assert.Equal(t, "yes", param.Conditions[0].WhenValue) }) - t.Run("requiredCredentials are loaded", func(t *testing.T) { - assert.Equal(t, []string{"aws", "circleci", "github"}, mod.RequiredCredentials) + t.Run("parsing envVarName from module config", func(t *testing.T) { + param, err := findParameter(mod.Parameters, "accessKeyId") + if err != nil { + panic(err) + } + assert.Equal(t, "AWS_ACCESS_KEY_ID", param.EnvVarName) }) t.Run("TemplateConfig is unmarshaled", func(t *testing.T) { diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 5ab043802..7e7de8474 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -8,12 +8,6 @@ type Stack struct { func GetRegistry(path string) Registry { return Registry{ - { - "test", - []string{ - "/Users/davidcheung/projects/commit0/tests/test_data/modules/ci/", - }, - }, // TODO: better place to store these options as configuration file or any source { "EKS + Go + React + Gatsby", diff --git a/internal/util/util.go b/internal/util/util.go index f2888bb11..2a4eba626 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -114,11 +114,15 @@ func ExecuteCommandOutput(cmd *exec.Cmd, pathPrefix string, envars []string) str return string(out) } -// AppendProjectEnvToCmdEnv will add all the keys and values from envMap -// into envList as key-value pair strings (e.g.: "key=value") -func AppendProjectEnvToCmdEnv(envMap map[string]string, envList []string) []string { +// Allow module definition to use an alternative env-var-name than field while apply +func AppendProjectEnvToCmdEnv(envMap map[string]string, envList []string, translationMap map[string]string) []string { + for key, val := range envMap { if val != "" { + // overwrite key if exist in translation map + if val, ok := translationMap[key]; ok { + key = val + } envList = append(envList, fmt.Sprintf("%s=%s", key, val)) } } diff --git a/tests/test_data/apply/project1/Makefile b/tests/test_data/apply/project1/Makefile index 181788f40..78158f243 100644 --- a/tests/test_data/apply/project1/Makefile +++ b/tests/test_data/apply/project1/Makefile @@ -1,5 +1,6 @@ current_dir: @echo "foo: ${foo}" > project.out @echo "repo: ${REPOSITORY}" >> project.out + @echo "envVarName of viaEnvVarName: ${viaEnvVarName}" >> feature.out summary: diff --git a/tests/test_data/apply/project1/zero-module.yml b/tests/test_data/apply/project1/zero-module.yml index 5d6914f9b..e643090f5 100644 --- a/tests/test_data/apply/project1/zero-module.yml +++ b/tests/test_data/apply/project1/zero-module.yml @@ -17,3 +17,5 @@ requiredCredentials: parameters: - field: foo label: foo + - field: param1 + envVarName: viaEnvVarName diff --git a/tests/test_data/apply/zero-project.yml b/tests/test_data/apply/zero-project.yml index 09c239cfc..629a59b5f 100644 --- a/tests/test_data/apply/zero-project.yml +++ b/tests/test_data/apply/zero-project.yml @@ -4,6 +4,7 @@ modules: project1: parameters: foo: bar + param1: baz files: dir: project1 repo: github.com/commitdev/project1 diff --git a/tests/test_data/modules/ci/zero-module.yml b/tests/test_data/modules/ci/zero-module.yml index 364ca905a..a9f0f38b7 100644 --- a/tests/test_data/modules/ci/zero-module.yml +++ b/tests/test_data/modules/ci/zero-module.yml @@ -52,13 +52,17 @@ parameters: matchField: useExistingAwsProfile - field: accessKeyId label: AWS AccessKeyId + envVarName: "AWS_ACCESS_KEY_ID" conditions: - action: KeyMatchCondition whenValue: "no" matchField: useExistingAwsProfile - field: secretAccessKey + envVarName: "AWS_SECRET_ACCESS_KEY" label: AWS SecretAccessKey conditions: - action: KeyMatchCondition whenValue: "no" matchField: useExistingAwsProfile + - field: testExecute + execute: echo $AWS_ACCESS_KEY_ID From f4073e4ecce35aab2025ffefa7b5dbb23a778170 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 15:55:35 -0500 Subject: [PATCH 05/18] add comments to public functions --- cmd/create.go | 2 +- internal/config/moduleconfig/module_config.go | 7 +++++++ internal/config/projectconfig/project_config.go | 4 +++- internal/init/custom-prompts.go | 14 +++++++++++--- internal/init/prompts.go | 2 +- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cmd/create.go b/cmd/create.go index e67a79ce4..abfb7c6cc 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -47,7 +47,7 @@ func Create(dir string, createConfigPath string) { flog.Infof(":up_arrow: Done Rendering - committing repositories to version control.") for _, module := range projectConfig.Modules { - err, githubApiKey := module.ReadVendorCredentials("github") + err, githubApiKey := projectconfig.ReadVendorCredentialsFromModule(module, "github") if err != nil { flog.Errorf(err.Error()) } diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index 2b781f5bf..ebe229131 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -68,6 +68,9 @@ func (cfg ModuleConfig) collectMissing() []string { return missing } +// Module can get a map of parameter's field name => desired env-var name +// for zero apply / zero init's prompt execute, +// this is useful for translating params like AWS credentials for running the AWS cli func (cfg ModuleConfig) GetParamEnvVarTranslationMap() map[string]string { translationMap := make(map[string]string) for i := 0; i < len(cfg.Parameters); i++ { @@ -175,6 +178,8 @@ func findMissing(obj reflect.Value, path, metadata string, missing *[]string) { } } +// Receives all parameters gathered from prompts during `Zero init` +// and based on module definition to construct the parameters of interest for each module func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[string]string { moduleParams := make(projectconfig.Parameters) // Loop through all the prompted values and find the ones relevant to this module @@ -192,6 +197,8 @@ func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[s return moduleParams } +// During zero init, we parse the module config and append the condition into the zero-project file +// This gathers the conditions to append into project config func SummarizeConditions(module ModuleConfig) []projectconfig.Condition { moduleConditions := []projectconfig.Condition{} for _, condition := range module.Conditions { diff --git a/internal/config/projectconfig/project_config.go b/internal/config/projectconfig/project_config.go index fce86f6fa..8b850a2d3 100644 --- a/internal/config/projectconfig/project_config.go +++ b/internal/config/projectconfig/project_config.go @@ -30,7 +30,9 @@ type Module struct { Conditions []Condition `yaml:"conditions,omitempty"` } -func (m Module) ReadVendorCredentials(vendor string) (error, string) { +// Based on parsed project config's module, retrieve the vendor's credential +// for pre-defined functionalities. (eg: pushing repos to github) +func ReadVendorCredentialsFromModule(m Module, vendor string) (error, string) { // this mapping could be useful for module config as well vendorToParamMap := map[string]string{ "github": "githubAccessToken", diff --git a/internal/init/custom-prompts.go b/internal/init/custom-prompts.go index ec649c2f6..9d2c9c486 100644 --- a/internal/init/custom-prompts.go +++ b/internal/init/custom-prompts.go @@ -1,20 +1,28 @@ package init import ( + "errors" + "fmt" + "github.com/commitdev/zero/internal/config/moduleconfig" project "github.com/commitdev/zero/pkg/credentials" "github.com/commitdev/zero/pkg/util/flog" ) -func CustomPromptHandler(promptType string, params map[string]string) { +// zero-module's parameters allow prompts to specify types of custom actions +// this allows non-standard enum / string input to be added, such as AWS profile picker +func CustomPromptHandler(promptType string, params map[string]string) error { switch promptType { case "AWSProfilePicker": - AWSProfilePicker(params) + promptAWSProfilePicker(params) + default: + return errors.New(fmt.Sprintf("Unsupported custom prompt type %s.", promptType)) } + return nil } -func AWSProfilePicker(params map[string]string) { +func promptAWSProfilePicker(params map[string]string) { profiles, err := project.GetAWSProfiles() if err != nil { profiles = []string{} diff --git a/internal/init/prompts.go b/internal/init/prompts.go index 2698ff2dc..cc942aa8d 100644 --- a/internal/init/prompts.go +++ b/internal/init/prompts.go @@ -121,7 +121,7 @@ func (p PromptHandler) RunPrompt(projectParams map[string]string, envVarTranslat if p.Parameter.Execute != "" { result = executeCmd(p.Parameter.Execute, projectParams, envVarTranslationMap) } else if p.Parameter.Type != "" { - CustomPromptHandler(p.Parameter.Type, projectParams) + err = CustomPromptHandler(p.Parameter.Type, projectParams) } else if p.Parameter.Value != "" { result = p.Parameter.Value } else { From 5bdb4d23987e50b001006b38a209f6504908fa3d Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 17:11:47 -0500 Subject: [PATCH 06/18] fixup! add comments to public functions --- internal/config/moduleconfig/module_config.go | 9 +++++---- internal/config/projectconfig/project_config.go | 5 +++-- internal/init/custom-prompts.go | 1 + internal/init/prompts.go | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index ebe229131..cc9b1aa1f 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -68,7 +68,8 @@ func (cfg ModuleConfig) collectMissing() []string { return missing } -// Module can get a map of parameter's field name => desired env-var name +// GetParamEnvVarTranslationMap returns a map for translating parameter's `Field` into env-var keys +// It loops through each parameter then adds to translation map if applicable // for zero apply / zero init's prompt execute, // this is useful for translating params like AWS credentials for running the AWS cli func (cfg ModuleConfig) GetParamEnvVarTranslationMap() map[string]string { @@ -178,7 +179,7 @@ func findMissing(obj reflect.Value, path, metadata string, missing *[]string) { } } -// Receives all parameters gathered from prompts during `Zero init` +// SummarizeParameters receives all parameters gathered from prompts during `Zero init` // and based on module definition to construct the parameters of interest for each module func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[string]string { moduleParams := make(projectconfig.Parameters) @@ -197,8 +198,8 @@ func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[s return moduleParams } -// During zero init, we parse the module config and append the condition into the zero-project file -// This gathers the conditions to append into project config +// SummarizeConditions collects conditions from module-definition +// then append the condition into the zero-project configuration func SummarizeConditions(module ModuleConfig) []projectconfig.Condition { moduleConditions := []projectconfig.Condition{} for _, condition := range module.Conditions { diff --git a/internal/config/projectconfig/project_config.go b/internal/config/projectconfig/project_config.go index 8b850a2d3..3300011d1 100644 --- a/internal/config/projectconfig/project_config.go +++ b/internal/config/projectconfig/project_config.go @@ -30,8 +30,9 @@ type Module struct { Conditions []Condition `yaml:"conditions,omitempty"` } -// Based on parsed project config's module, retrieve the vendor's credential -// for pre-defined functionalities. (eg: pushing repos to github) +// ReadVendorCredentialsFromModule uses parsed project-config's module +// based on vendor parameter, retrieve the vendor's credential +// for pre-defined functionalities (eg: Github api key for pushing repos to github) func ReadVendorCredentialsFromModule(m Module, vendor string) (error, string) { // this mapping could be useful for module config as well vendorToParamMap := map[string]string{ diff --git a/internal/init/custom-prompts.go b/internal/init/custom-prompts.go index 9d2c9c486..e0bee6026 100644 --- a/internal/init/custom-prompts.go +++ b/internal/init/custom-prompts.go @@ -9,6 +9,7 @@ import ( "github.com/commitdev/zero/pkg/util/flog" ) +// CustomPromptHandler handles non-input and enum options prompts // zero-module's parameters allow prompts to specify types of custom actions // this allows non-standard enum / string input to be added, such as AWS profile picker func CustomPromptHandler(promptType string, params map[string]string) error { diff --git a/internal/init/prompts.go b/internal/init/prompts.go index cc942aa8d..2253f5d11 100644 --- a/internal/init/prompts.go +++ b/internal/init/prompts.go @@ -191,7 +191,7 @@ func sanitizeParameterValue(str string) string { return re.ReplaceAllString(str, "") } -// PromptParams renders series of prompt UI based on the config +// PromptModuleParams renders series of prompt UI based on the config func PromptModuleParams(moduleConfig moduleconfig.ModuleConfig, parameters map[string]string) (map[string]string, error) { envVarTranslationMap := moduleConfig.GetParamEnvVarTranslationMap() for _, parameter := range moduleConfig.Parameters { From 8c0fc8106f9dcaaf629b4875006c67f50ba0873c Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 17:21:55 -0500 Subject: [PATCH 07/18] fixup! fixup! add comments to public functions --- internal/init/prompts.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/init/prompts.go b/internal/init/prompts.go index 2253f5d11..72985795a 100644 --- a/internal/init/prompts.go +++ b/internal/init/prompts.go @@ -100,12 +100,14 @@ func ValidateProjectName(input string) error { return nil } -// Getting param to fill in to zero-project.yml, there are multiple ways of obtaining the value +// RunPrompt obtains the value of PromptHandler depending on the parameter's definition +// for the project config, there are multiple ways of obtaining the value // values go into params depending on `Condition` as the highest precedence (Whether it gets this value) // then follows this order to determine HOW it obtains that value // 1. Execute (this could potentially be refactored into type + data) // 2. type: specific ways of obtaining values (in AWS credential case it will set 2 values to the map) -//3. +// 3. value: directly assigns a value to a parameter +// 4. prompt: requires users to select an option OR input a string func (p PromptHandler) RunPrompt(projectParams map[string]string, envVarTranslationMap map[string]string) { var err error var result string From a11fbca0bdf924eece5184d10730b0a8843be5b8 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 17:26:33 -0500 Subject: [PATCH 08/18] fixup! fixup! fixup! add comments to public functions --- internal/util/util.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/util/util.go b/internal/util/util.go index 2a4eba626..a3da31351 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -114,7 +114,8 @@ func ExecuteCommandOutput(cmd *exec.Cmd, pathPrefix string, envars []string) str return string(out) } -// Allow module definition to use an alternative env-var-name than field while apply +// AppendProjectEnvToCmdEnv converts a key-value pair map into a slice of `key=value`s +// allow module definition to use an alternative env-var-name than field while apply func AppendProjectEnvToCmdEnv(envMap map[string]string, envList []string, translationMap map[string]string) []string { for key, val := range envMap { @@ -148,7 +149,7 @@ func ItemInSlice(slice []string, target string) bool { return false } -// Given a resource of struct type as +// ReflectStructValueIntoMap receives a resource of struct type as // type AWSCreds struct{ // AccessKeyID string `yaml:"accessKeyId,omitempty"` // SecretAccessKey string `yaml:"accessKeyId,omitempty"` From fec369dd9e36cba3967e3d469391aab6b4d99d15 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 17:38:49 -0500 Subject: [PATCH 09/18] fixup! fixup! fixup! fixup! add comments to public functions --- pkg/credentials/credentials.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 5786b72d7..ebbbc2db4 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -38,6 +38,8 @@ func fetchAWSConfig(awsPath string, profileName string) (error, AWSResourceConfi } } +// FillAWSProfile receives the AWS profile name, then parses +// the accessKeyId / secretAccessKey values into a map func FillAWSProfile(profileName string, paramsToFill map[string]string) error { awsPath := GetAWSCredsPath() err, awsCreds := fetchAWSConfig(awsPath, profileName) From 1c62c0c20c0b31ba88aca8c4c92a2b8c0b94659c Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 21:42:41 -0500 Subject: [PATCH 10/18] address comments --- README.md | 7 +++ docs/module-definition.md | 55 +++++++++++++++++++ internal/apply/apply.go | 10 ---- internal/config/moduleconfig/module_config.go | 7 ++- internal/init/custom-prompts.go | 2 +- pkg/credentials/credentials.go | 11 ++-- pkg/credentials/credentials_test.go | 8 +-- 7 files changed, 76 insertions(+), 24 deletions(-) create mode 100644 docs/module-definition.md diff --git a/README.md b/README.md index 418702f20..5206fed4c 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,13 @@ You need to [register a new domain](https://docs.aws.amazon.com/Route53/latest/D ___ +### Building blocks of Zero +### Module Definition +Module definition defines the information needed for the module to run (`zero apply`). +Also declares dependency used to determine the order of execution with other modules. + +See [`zero-module.yml` reference](./docs/module-definition.md) for details. +___ ## Using zero to spin up your own stack Using Zero to spin up your infrastructure and application is easy and straightforward. Using just a few commands, you can configure and deploy your very own scalable, high-performance, production-ready infrastructure. diff --git a/docs/module-definition.md b/docs/module-definition.md new file mode 100644 index 000000000..a002d3a3c --- /dev/null +++ b/docs/module-definition.md @@ -0,0 +1,55 @@ +## Module Definition: `zero-module.yml` +Module definition defines the information needed for the module to run (`zero apply`). +Also declares dependency used to determine the order of execution with other modules. + +| Parameters | type | Description | +|---------------|-----------------|----------------------------| +| `name` | string | Name of module | +| `description` | string | Description of the module | +| `author` | string | Author of the module | +| `icon` | string | Path to logo image | +| `parameters` | list(Parameter) | Parameters to prompt users | + +### Parameter: +Parameter defines the prompt during zero-init. +There are multiple ways of obtaining the value for each parameter. +Parameters may have `Conditions` and must be fulfilled when supplied, otherwise it skips the field entirely. + +The precedence for different types of parameter prompts are as follow. +1. Execute +2. type: specific ways of obtaining values (in AWS credential case it will set 2 values to the map) +3. value: directly assigns a value to a parameter +4. prompt: requires users to select an option OR input a string +Note: Default is supplied as the starting point of the user's manual input (Not when value passed in is empty) + +| Parameters | Type | Description | +|-----------------------|-----------------|---------------------------------------------------------------------------------------------------------------------------| +| `field` | string | key to store result for project definition | +| `label` | string | displayed name for the prompt | +| `options` | list(string) | A list of values for users to pick from | +| `default` | string | Defaults to this value during prompt | +| `value` | string | Skips prompt entirely when set | +| `info` | string | Displays during prompt as extra information guiding user's input | +| `fieldValidation` | Validation | Validations for the prompt value | +| `type` | enum(string) | Built in custom prompts: currently supports [`AWSProfilePicker`] | +| `execute` | string | executes commands and takes stdout as prompt result | +| `omitFromProjectFile` | bool | Field is skipped from adding to project definition | +| `conditions` | list(Condition) | Conditions for prompt to run, if supplied all conditions must pass | +| `envVarName` | string | During `zero apply` parameters are available as env-vars, defaults to field name but can be overwritten with `envVarName` | + +### Condition +| Parameters | Type | Description | +|--------------|--------------|-------------------------------------------------------------------| +| `action` | enum(string) | type of condition, currently supports [`KeyMatchCondition`] | +| `matchField` | string | Allows you to condition prompt based on another parameter's value | +| `WhenValue` | string | Matches for this value to satisfy the condition | +| `data` | list(string) | Supply extra data for condition to run | + + +### Validation + +| Parameters | type | Description | +|----------------|--------------|-------------------------------------| +| `type` | enum(string) | Currently supports [`regex`] | +| `value` | string | Regular expression string | +| `errorMessage` | string | Error message when validation fails | \ No newline at end of file diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 841ae4fa4..782153369 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -13,7 +13,6 @@ import ( "github.com/commitdev/zero/internal/util" "github.com/hashicorp/terraform/dag" - "github.com/commitdev/zero/internal/config/moduleconfig" "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" @@ -96,15 +95,6 @@ func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEn }) } -func getParameterDefinition(modConfig moduleconfig.ModuleConfig, field string) moduleconfig.Parameter { - for i := 0; i < len(modConfig.Parameters); i++ { - if field == modConfig.Parameters[i].Field { - return modConfig.Parameters[i] - } - } - return moduleconfig.Parameter{} -} - // promptEnvironments Prompts the user for the environments to apply against and returns a slice of strings representing the environments func promptEnvironments() []string { items := map[string][]string{ diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index cc9b1aa1f..2b7ce95b1 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -180,7 +180,8 @@ func findMissing(obj reflect.Value, path, metadata string, missing *[]string) { } // SummarizeParameters receives all parameters gathered from prompts during `Zero init` -// and based on module definition to construct the parameters of interest for each module +// and based on module definition to construct the parameters for each module for zero-project.yml +// filters out parameters defined as OmitFromProjectFile: true func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[string]string { moduleParams := make(projectconfig.Parameters) // Loop through all the prompted values and find the ones relevant to this module @@ -198,8 +199,8 @@ func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[s return moduleParams } -// SummarizeConditions collects conditions from module-definition -// then append the condition into the zero-project configuration +// SummarizeConditions based on conditions from zero-module.yml +// creates and returns slice of conditions for project config func SummarizeConditions(module ModuleConfig) []projectconfig.Condition { moduleConditions := []projectconfig.Condition{} for _, condition := range module.Conditions { diff --git a/internal/init/custom-prompts.go b/internal/init/custom-prompts.go index e0bee6026..e432dc954 100644 --- a/internal/init/custom-prompts.go +++ b/internal/init/custom-prompts.go @@ -39,7 +39,7 @@ func promptAWSProfilePicker(params map[string]string) { Validate: NoValidation, } _, value := promptParameter(awsPrompt) - credErr := project.FillAWSProfile(value, params) + credErr := project.FillAWSProfile("", value, params) if credErr != nil { flog.Errorf("Failed to retrieve profile, falling back to User input") params["useExistingAwsProfile"] = "no" diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index ebbbc2db4..f715c912c 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -16,8 +16,6 @@ type AWSResourceConfig struct { SecretAccessKey string `yaml:"secretAccessKey,omitempty" env:"AWS_SECRET_ACCESS_KEY,omitempty"` } -var GetAWSCredsPath = awsCredsPath - func awsCredsPath() string { usr, err := user.Current() if err != nil { @@ -40,9 +38,12 @@ func fetchAWSConfig(awsPath string, profileName string) (error, AWSResourceConfi // FillAWSProfile receives the AWS profile name, then parses // the accessKeyId / secretAccessKey values into a map -func FillAWSProfile(profileName string, paramsToFill map[string]string) error { - awsPath := GetAWSCredsPath() - err, awsCreds := fetchAWSConfig(awsPath, profileName) +func FillAWSProfile(pathToCredentials string, profileName string, paramsToFill map[string]string) error { + if pathToCredentials == "" { + pathToCredentials = awsCredsPath() + } + + err, awsCreds := fetchAWSConfig(pathToCredentials, profileName) if err != nil { return err } diff --git a/pkg/credentials/credentials_test.go b/pkg/credentials/credentials_test.go index 7b240d646..0fb70be26 100644 --- a/pkg/credentials/credentials_test.go +++ b/pkg/credentials/credentials_test.go @@ -9,12 +9,10 @@ import ( func TestFillAWSProfileCredentials(t *testing.T) { mockAwsCredentialFilePath := "../../tests/test_data/aws/mock_credentials.yml" - credentials.GetAWSCredsPath = func() string { - return mockAwsCredentialFilePath - } + t.Run("fills project credentials", func(t *testing.T) { params := map[string]string{} - err := credentials.FillAWSProfile("default", params) + err := credentials.FillAWSProfile(mockAwsCredentialFilePath, "default", params) if err != nil { panic(err) } @@ -25,7 +23,7 @@ func TestFillAWSProfileCredentials(t *testing.T) { t.Run("supports non-default profiles", func(t *testing.T) { params := map[string]string{} - err := credentials.FillAWSProfile("foobar", params) + err := credentials.FillAWSProfile(mockAwsCredentialFilePath, "foobar", params) if err != nil { panic(err) } From a09474f61cc2765dcf77f8bb8ab6ee1fc0de05a3 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 21:46:28 -0500 Subject: [PATCH 11/18] fixup! address comments --- internal/util/util.go | 2 +- pkg/credentials/credentials.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/util/util.go b/internal/util/util.go index a3da31351..8dc5826c4 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -152,7 +152,7 @@ func ItemInSlice(slice []string, target string) bool { // ReflectStructValueIntoMap receives a resource of struct type as // type AWSCreds struct{ // AccessKeyID string `yaml:"accessKeyId,omitempty"` -// SecretAccessKey string `yaml:"accessKeyId,omitempty"` +// SecretAccessKey string `yaml:"secretAccessKey,omitempty"` // }{ // AccessKeyID: "FOO", // SecretAccessKey: "BAR", diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index f715c912c..b58687519 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -12,8 +12,8 @@ import ( ) type AWSResourceConfig struct { - AccessKeyID string `yaml:"accessKeyId,omitempty" env:"AWS_ACCESS_KEY_ID,omitempty"` - SecretAccessKey string `yaml:"secretAccessKey,omitempty" env:"AWS_SECRET_ACCESS_KEY,omitempty"` + AccessKeyID string `key:"accessKeyId"` + SecretAccessKey string `key:"secretAccessKey"` } func awsCredsPath() string { @@ -38,16 +38,16 @@ func fetchAWSConfig(awsPath string, profileName string) (error, AWSResourceConfi // FillAWSProfile receives the AWS profile name, then parses // the accessKeyId / secretAccessKey values into a map -func FillAWSProfile(pathToCredentials string, profileName string, paramsToFill map[string]string) error { - if pathToCredentials == "" { - pathToCredentials = awsCredsPath() +func FillAWSProfile(pathToCredentialsFile string, profileName string, paramsToFill map[string]string) error { + if pathToCredentialsFile == "" { + pathToCredentialsFile = awsCredsPath() } - err, awsCreds := fetchAWSConfig(pathToCredentials, profileName) + err, awsCreds := fetchAWSConfig(pathToCredentialsFile, profileName) if err != nil { return err } - util.ReflectStructValueIntoMap(awsCreds, "yaml", paramsToFill) + util.ReflectStructValueIntoMap(awsCreds, "key", paramsToFill) return nil } From 272f642da414e9fdae92bf5c2fa1b230ef0538c7 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 9 Mar 2021 21:54:25 -0500 Subject: [PATCH 12/18] fixup! fixup! address comments --- internal/config/moduleconfig/module_config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index 2b7ce95b1..d6ba8e406 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -202,15 +202,15 @@ func SummarizeParameters(module ModuleConfig, allParams map[string]string) map[s // SummarizeConditions based on conditions from zero-module.yml // creates and returns slice of conditions for project config func SummarizeConditions(module ModuleConfig) []projectconfig.Condition { - moduleConditions := []projectconfig.Condition{} - for _, condition := range module.Conditions { - newCond := projectconfig.Condition{ + moduleConditions := make([]projectconfig.Condition, len(module.Conditions)) + + for i, condition := range module.Conditions { + moduleConditions[i] = projectconfig.Condition{ Action: condition.Action, MatchField: condition.MatchField, WhenValue: condition.WhenValue, Data: condition.Data, } - moduleConditions = append(moduleConditions, newCond) } return moduleConditions } From 2d61408d4d8b502fc95fd85ba63341ba23f01003 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 10 Mar 2021 12:45:12 -0500 Subject: [PATCH 13/18] project definition --- README.md | 5 +++++ docs/project-definition.md | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docs/project-definition.md diff --git a/README.md b/README.md index 5206fed4c..2b161034f 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,11 @@ You need to [register a new domain](https://docs.aws.amazon.com/Route53/latest/D ___ ### Building blocks of Zero + +### Project Definition: +Each project is defined by this project definition file, this manifest contains your project details, and is the source of truth for the templating(`zero create`) and provision(`zero apply`) steps. + +See [`zero-project.yml` reference](./docs/project-definition.md) for details. ### Module Definition Module definition defines the information needed for the module to run (`zero apply`). Also declares dependency used to determine the order of execution with other modules. diff --git a/docs/project-definition.md b/docs/project-definition.md new file mode 100644 index 000000000..8b65de44c --- /dev/null +++ b/docs/project-definition.md @@ -0,0 +1,27 @@ +### Project Definition: `zero-project.yml` +Each project is defined by this project definition file, this manifest contains your project details, and is the source of truth for the templating(`zero create`) and provision(`zero apply`) steps. + +`zero-project.yml` is created from `zero-init`, and defines which modules are part of the project and what each module's parameters are for the subsequence steps. + +| Parameters | Type | Description | +|--------------------------|--------------|------------------------------------------------| +| `name` | string | name of the project | +| `shouldPushRepositories` | boolean | whether to push the modules to version control | +| `modules` | map(modules) | a map containing modules of your project | + + +### Modules +| Parameters | Type | Description | +|--------------|-----------------|-------------------------------------------------------------------------| +| `parameters` | map(string) | key-value map of all the parameters to run the module | +| `files` | File | Stores information such as source-module location and destination | +| `dependsOn` | list(string) | a list of dependencies that should be fulfilled before this module | +| `conditions` | list(condition) | conditions to apply while templating out the module based on parameters | + +### Condition +| Parameters | Type | Description | +|--------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| `action` | enum(string) | type of condition, currently supports [`ignoreFile`] | +| `matchField` | string | Allows you to condition prompt based on another parameter's value | +| `WhenValue` | string | Matches for this value to satisfy the condition | +| `data` | list(string) | Supply extra data for condition to run `ignoreFile`: provide list of paths (file or directory path) to omit from module when condition is satisfied | \ No newline at end of file From 28f807bfc9df50842e926091baf69892c15816a4 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 10 Mar 2021 16:02:02 -0500 Subject: [PATCH 14/18] Update docs/module-definition.md Co-authored-by: Bill Monkman --- docs/module-definition.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/module-definition.md b/docs/module-definition.md index a002d3a3c..5ade56882 100644 --- a/docs/module-definition.md +++ b/docs/module-definition.md @@ -1,6 +1,6 @@ ## Module Definition: `zero-module.yml` -Module definition defines the information needed for the module to run (`zero apply`). -Also declares dependency used to determine the order of execution with other modules. +This file is the definition of a Zero module. It contains a list of all the required parameters to be able to prompt a user for choices during `zero init`, information about how to template the contents of the module during `zero create`, and the information needed for the module to run (`zero apply`). +It also declares the module's dependencies to determine the order of execution in relation to other modules. | Parameters | type | Description | |---------------|-----------------|----------------------------| @@ -52,4 +52,4 @@ Note: Default is supplied as the starting point of the user's manual input (Not |----------------|--------------|-------------------------------------| | `type` | enum(string) | Currently supports [`regex`] | | `value` | string | Regular expression string | -| `errorMessage` | string | Error message when validation fails | \ No newline at end of file +| `errorMessage` | string | Error message when validation fails | From b6c2fac206e9926c6f9a54ed8a29b3a8df0dd848 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 10 Mar 2021 16:02:10 -0500 Subject: [PATCH 15/18] Update docs/project-definition.md Co-authored-by: Bill Monkman --- docs/project-definition.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/project-definition.md b/docs/project-definition.md index 8b65de44c..2ca3bf641 100644 --- a/docs/project-definition.md +++ b/docs/project-definition.md @@ -1,7 +1,7 @@ ### Project Definition: `zero-project.yml` -Each project is defined by this project definition file, this manifest contains your project details, and is the source of truth for the templating(`zero create`) and provision(`zero apply`) steps. +Each project is defined by this file. This manifest reflects all the options a user chose during the `zero init` step. It defines which modules are part of the project, each of their parameters, and is the source of truth for the templating (`zero create`) and provision (`zero apply`) steps. -`zero-project.yml` is created from `zero-init`, and defines which modules are part of the project and what each module's parameters are for the subsequence steps. +_Note: This file contains credentials, so make sure it is not shared with others._ | Parameters | Type | Description | |--------------------------|--------------|------------------------------------------------| @@ -24,4 +24,4 @@ Each project is defined by this project definition file, this manifest contains | `action` | enum(string) | type of condition, currently supports [`ignoreFile`] | | `matchField` | string | Allows you to condition prompt based on another parameter's value | | `WhenValue` | string | Matches for this value to satisfy the condition | -| `data` | list(string) | Supply extra data for condition to run `ignoreFile`: provide list of paths (file or directory path) to omit from module when condition is satisfied | \ No newline at end of file +| `data` | list(string) | Supply extra data for condition to run `ignoreFile`: provide list of paths (file or directory path) to omit from module when condition is satisfied | From c3b7b6f8e8afc1c8527290debc92b8ecbbfdf10b Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 10 Mar 2021 18:34:12 -0500 Subject: [PATCH 16/18] better error message to users --- internal/init/custom-prompts.go | 15 +++++++++------ internal/init/prompts.go | 12 ++++++++---- internal/init/prompts_test.go | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/internal/init/custom-prompts.go b/internal/init/custom-prompts.go index e432dc954..73a431152 100644 --- a/internal/init/custom-prompts.go +++ b/internal/init/custom-prompts.go @@ -6,7 +6,6 @@ import ( "github.com/commitdev/zero/internal/config/moduleconfig" project "github.com/commitdev/zero/pkg/credentials" - "github.com/commitdev/zero/pkg/util/flog" ) // CustomPromptHandler handles non-input and enum options prompts @@ -16,17 +15,21 @@ func CustomPromptHandler(promptType string, params map[string]string) error { switch promptType { case "AWSProfilePicker": - promptAWSProfilePicker(params) + err := promptAWSProfilePicker(params) + if err != nil { + params["useExistingAwsProfile"] = "no" + return err + } default: return errors.New(fmt.Sprintf("Unsupported custom prompt type %s.", promptType)) } return nil } -func promptAWSProfilePicker(params map[string]string) { +func promptAWSProfilePicker(params map[string]string) error { profiles, err := project.GetAWSProfiles() if err != nil { - profiles = []string{} + return err } awsPrompt := PromptHandler{ @@ -41,7 +44,7 @@ func promptAWSProfilePicker(params map[string]string) { _, value := promptParameter(awsPrompt) credErr := project.FillAWSProfile("", value, params) if credErr != nil { - flog.Errorf("Failed to retrieve profile, falling back to User input") - params["useExistingAwsProfile"] = "no" + return errors.New("Failed to retrieve profile, falling back to User input") } + return nil } diff --git a/internal/init/prompts.go b/internal/init/prompts.go index 72985795a..b02752b19 100644 --- a/internal/init/prompts.go +++ b/internal/init/prompts.go @@ -108,7 +108,7 @@ func ValidateProjectName(input string) error { // 2. type: specific ways of obtaining values (in AWS credential case it will set 2 values to the map) // 3. value: directly assigns a value to a parameter // 4. prompt: requires users to select an option OR input a string -func (p PromptHandler) RunPrompt(projectParams map[string]string, envVarTranslationMap map[string]string) { +func (p PromptHandler) RunPrompt(projectParams map[string]string, envVarTranslationMap map[string]string) error { var err error var result string @@ -130,7 +130,7 @@ func (p PromptHandler) RunPrompt(projectParams map[string]string, envVarTranslat err, result = promptParameter(p) } if err != nil { - exit.Fatal("Exiting prompt: %v\n", err) + return err } // Append the result to parameter map @@ -138,6 +138,7 @@ func (p PromptHandler) RunPrompt(projectParams map[string]string, envVarTranslat } else { flog.Debugf("Skipping prompt(%s) due to condition failed", p.Field) } + return nil } func promptParameter(prompt PromptHandler) (error, string) { @@ -227,7 +228,10 @@ func PromptModuleParams(moduleConfig moduleconfig.ModuleConfig, parameters map[s // for k, v := range parameters { // credentialEnvs[k] = v // } - promptHandler.RunPrompt(parameters, envVarTranslationMap) + err := promptHandler.RunPrompt(parameters, envVarTranslationMap) + if err != nil { + return parameters, err + } } flog.Debugf("Module %s prompt: \n %#v", moduleConfig.Name, parameters) return parameters, nil @@ -243,7 +247,7 @@ func promptAllModules(modules map[string]moduleconfig.ModuleConfig) map[string]s parameterValues, err = PromptModuleParams(config, parameterValues) if err != nil { - exit.Fatal("Exiting prompt: %v\n", err) + exit.Fatal("Exiting prompt(%s): %v\n", config.Name, err) } } return parameterValues diff --git a/internal/init/prompts_test.go b/internal/init/prompts_test.go index 190cf1706..75fbe5f1b 100644 --- a/internal/init/prompts_test.go +++ b/internal/init/prompts_test.go @@ -166,4 +166,18 @@ func TestGetParam(t *testing.T) { assert.NotContains(t, projectParams, "failing_condition", "Expected to fail condition and not set value") assert.Equal(t, "pass", projectParams["multiple_condition"], "Expected to pass multiple condition and set value") }) + + t.Run("Should return error upon unsupported custom prompt type", func(t *testing.T) { + + projectParams = map[string]string{} + params := []moduleconfig.Parameter{ + { + Field: "param1", + Type: "random-type", + }, + } + module := moduleconfig.ModuleConfig{Parameters: params} + _, err := initPrompts.PromptModuleParams(module, projectParams) + assert.Equal(t, "Unsupported custom prompt type random-type.", err.Error()) + }) } From 89dbe412d797330980290780cc6bb7c85be7dac5 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 10 Mar 2021 18:45:29 -0500 Subject: [PATCH 17/18] explain template parameters --- docs/module-definition.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/module-definition.md b/docs/module-definition.md index 5ade56882..654255f8d 100644 --- a/docs/module-definition.md +++ b/docs/module-definition.md @@ -2,13 +2,22 @@ This file is the definition of a Zero module. It contains a list of all the required parameters to be able to prompt a user for choices during `zero init`, information about how to template the contents of the module during `zero create`, and the information needed for the module to run (`zero apply`). It also declares the module's dependencies to determine the order of execution in relation to other modules. -| Parameters | type | Description | -|---------------|-----------------|----------------------------| -| `name` | string | Name of module | -| `description` | string | Description of the module | -| `author` | string | Author of the module | -| `icon` | string | Path to logo image | -| `parameters` | list(Parameter) | Parameters to prompt users | +| Parameters | type | Description | +|---------------|-----------------|--------------------------------------------------| +| `name` | string | Name of module | +| `description` | string | Description of the module | +| `template` | template | default settings for templating out the module | +| `author` | string | Author of the module | +| `icon` | string | Path to logo image | +| `parameters` | list(Parameter) | Parameters to prompt users | + +### Template +| Parameters | Type | Description | +|--------------|---------|-----------------------------------------------------------------------| +| `strictMode` | boolean | whether strict mode is enabled | +| `delimiters` | tuple | A tuple of open delimiter and ending delimiter eg: `<%` and `%>` | +| `inputDir` | string | Folder to template from the module, becomes the module root for users | +| `outputDir` | string | local directory name for the module, gets commited to version control | ### Parameter: Parameter defines the prompt during zero-init. @@ -45,7 +54,6 @@ Note: Default is supplied as the starting point of the user's manual input (Not | `WhenValue` | string | Matches for this value to satisfy the condition | | `data` | list(string) | Supply extra data for condition to run | - ### Validation | Parameters | type | Description | From 082bbe9ae65e35221c8e2d559fe05a4d6430b2f6 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Wed, 10 Mar 2021 18:52:25 -0500 Subject: [PATCH 18/18] condition clarification between module and param --- docs/module-definition.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/module-definition.md b/docs/module-definition.md index 654255f8d..b59cb30c4 100644 --- a/docs/module-definition.md +++ b/docs/module-definition.md @@ -11,6 +11,7 @@ It also declares the module's dependencies to determine the order of execution | `icon` | string | Path to logo image | | `parameters` | list(Parameter) | Parameters to prompt users | + ### Template | Parameters | Type | Description | |--------------|---------|-----------------------------------------------------------------------| @@ -19,6 +20,17 @@ It also declares the module's dependencies to determine the order of execution | `inputDir` | string | Folder to template from the module, becomes the module root for users | | `outputDir` | string | local directory name for the module, gets commited to version control | +### Condition(module) +Module conditions are considered during template phase (`zero create`), based on parameters supplied from project-definition, +modules can decide to have specific files ignored from the user's module. For example if user picks `userAuth: no`, we can ignore the auth resources via templating. + +| Parameters | Type | Description | +|--------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| `action` | enum(string) | type of condition, currently supports [`ignoreFile`] | +| `matchField` | string | Allows you to condition prompt based on another parameter's value | +| `WhenValue` | string | Matches for this value to satisfy the condition | +| `data` | list(string) | Supply extra data for condition to run `ignoreFile`: provide list of paths (file or directory path) to omit from module when condition is satisfied | + ### Parameter: Parameter defines the prompt during zero-init. There are multiple ways of obtaining the value for each parameter. @@ -46,7 +58,11 @@ Note: Default is supplied as the starting point of the user's manual input (Not | `conditions` | list(Condition) | Conditions for prompt to run, if supplied all conditions must pass | | `envVarName` | string | During `zero apply` parameters are available as env-vars, defaults to field name but can be overwritten with `envVarName` | -### Condition +### Condition(paramters) +Parameters conditions are considered while running user prompts, prompts are +executed in order of the yml, and will be skipped if conditions are not satisfied. +For example if a user decide to not use circleCI, condition can be used to skip the circleCI_api_key prompt. + | Parameters | Type | Description | |--------------|--------------|-------------------------------------------------------------------| | `action` | enum(string) | type of condition, currently supports [`KeyMatchCondition`] |