Merge pull request #180 from commitdev/process-modules-as-a-graph

Added dependsOn to module and project config, switched apply to apply…

Author: bmonkman

go.mod

@@ -3,14 +3,12 @@ module github.com/commitdev/zero
 go 1.13
 
 require (
-	github.com/aws/aws-sdk-go v1.25.33
-	github.com/chzyer/logex v1.1.10 // indirect
-	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
-	github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect
+	github.com/aws/aws-sdk-go v1.30.12
 	github.com/coreos/go-semver v0.2.0
-	github.com/google/go-cmp v0.3.0
+	github.com/google/go-cmp v0.3.1
 	github.com/google/uuid v1.1.1
-	github.com/hashicorp/go-getter v1.4.0
+	github.com/hashicorp/go-getter v1.4.2-0.20200106182914-9813cbd4eb02
+	github.com/hashicorp/terraform v0.12.26
 	github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a // indirect
 	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 	github.com/k0kubun/pp v3.0.1+incompatible
@@ -23,7 +21,7 @@ require (
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/spf13/cobra v0.0.6
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.5.1
 	github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
 	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect

go.sum

@@ -11,6 +11,7 @@ import (
 
 	"github.com/commitdev/zero/internal/module"
 	"github.com/commitdev/zero/internal/util"
+	"github.com/hashicorp/terraform/dag"
 
 	"github.com/commitdev/zero/internal/config/globalconfig"
 	"github.com/commitdev/zero/internal/config/projectconfig"
@@ -50,8 +51,18 @@ Only a single environment may be suitable for an initial test, but for a real sy
 func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEnvironments []string) {
 	environmentArg := fmt.Sprintf("ENVIRONMENT=%s", strings.Join(applyEnvironments, ","))
 
-	// Go through each of the modules and run `make`
-	for _, mod := range projectConfig.Modules {
+	graph := projectConfig.GetDAG()
+
+	// Walk the graph of modules and run `make`
+	root := []dag.Vertex{projectconfig.GraphRootName}
+	graph.DepthFirstWalk(root, func(v dag.Vertex, depth int) error {
+		// Don't process the root
+		if depth == 0 {
+			return nil
+		}
+
+		name := v.(string)
+		mod := projectConfig.Modules[name]
 		// Add env vars for the makefile
 		envList := []string{
 			environmentArg,
@@ -72,7 +83,8 @@ func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEn
 		envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList)
 		envList = util.AppendProjectEnvToCmdEnv(credentials.AsEnvVars(), envList)
 		util.ExecuteCommand(exec.Command("make"), modulePath, envList)
-	}
+		return nil
+	})
 }
 
 // promptEnvironments Prompts the user for the environments to apply against and returns a slice of strings representing the environments

internal/apply/apply.go

@@ -10,6 +10,7 @@ type ModuleConfig struct {
 	Name                string
 	Description         string
 	Author              string
+	DependsOn           []string `yaml:"dependsOn,omitempty"`
 	TemplateConfig      `yaml:"template"`
 	RequiredCredentials []string `yaml:"requiredCredentials"`
 	Parameters          []Parameter

internal/config/moduleconfig/module_config.go

@@ -4,37 +4,27 @@ import (
 	"io/ioutil"
 	"log"
 
+	"github.com/hashicorp/terraform/dag"
 	"github.com/k0kubun/pp"
 	yaml "gopkg.in/yaml.v2"
 )
 
+// GraphRootName represents the root of the graph of modules in a project
+const GraphRootName = "graphRoot"
+
 type ZeroProjectConfig struct {
 	Name                   string `yaml:"name"`
 	ShouldPushRepositories bool
-	Infrastructure         Infrastructure // TODO simplify and flatten / rename?
 	Parameters             map[string]string
 	Modules                Modules `yaml:"modules"`
 }
 
-type Infrastructure struct {
-	AWS *AWS
-}
-
-type AWS struct {
-	AccountID string `yaml:"accountId"`
-	Region    string
-	Terraform terraform // TODO simplify and flatten?
-}
-
-type terraform struct {
-	RemoteState bool
-}
-
 type Modules map[string]Module
 
 type Module struct {
+	DependsOn  []string   `yaml:"dependsOn,omitempty"`
 	Parameters Parameters `yaml:"parameters,omitempty"`
-	Files      Files      `yaml:"files,omitempty"`
+	Files      Files
 }
 
 type Parameters map[string]string
@@ -63,9 +53,33 @@ func (c *ZeroProjectConfig) Print() {
 	pp.Println(c)
 }
 
-func NewModule(parameters Parameters, directory string, repository string, source string) Module {
+// GetDAG returns a graph of the module names used in this project config
+func (c *ZeroProjectConfig) GetDAG() dag.AcyclicGraph {
+	var g dag.AcyclicGraph
+
+	// Add vertices to graph
+	g.Add(GraphRootName)
+	for name := range c.Modules {
+		g.Add(name)
+	}
+
+	// Connect modules in graph
+	for name, m := range c.Modules {
+		if len(m.DependsOn) == 0 {
+			g.Connect(dag.BasicEdge(GraphRootName, name))
+		} else {
+			for _, dependencyName := range m.DependsOn {
+				g.Connect(dag.BasicEdge(dependencyName, name))
+			}
+		}
+	}
+	return g
+}
+
+func NewModule(parameters Parameters, directory string, repository string, source string, dependsOn []string) Module {
 	return Module{
 		Parameters: parameters,
+		DependsOn:  dependsOn,
 		Files: Files{
 			Directory:  directory,
 			Repository: repository,

internal/config/projectconfig/project_config.go

@@ -4,11 +4,14 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/commitdev/zero/internal/config/projectconfig"
+	"github.com/commitdev/zero/internal/constants"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestLoadConfig(t *testing.T) {
@@ -37,9 +40,9 @@ func TestLoadConfig(t *testing.T) {
 func eksGoReactSampleModules() projectconfig.Modules {
 	parameters := projectconfig.Parameters{"a": "b"}
 	return projectconfig.Modules{
-		"aws-eks-stack":             projectconfig.NewModule(parameters, "zero-aws-eks-stack", "github.com/something/repo1", "github.com/commitdev/zero-aws-eks-stack"),
-		"deployable-backend":        projectconfig.NewModule(parameters, "zero-deployable-backend", "github.com/something/repo2", "github.com/commitdev/zero-deployable-backend"),
-		"deployable-react-frontend": projectconfig.NewModule(parameters, "zero-deployable-react-frontend", "github.com/something/repo3", "github.com/commitdev/zero-deployable-react-frontend"),
+		"aws-eks-stack":             projectconfig.NewModule(parameters, "zero-aws-eks-stack", "github.com/something/repo1", "github.com/commitdev/zero-aws-eks-stack", []string{}),
+		"deployable-backend":        projectconfig.NewModule(parameters, "zero-deployable-backend", "github.com/something/repo2", "github.com/commitdev/zero-deployable-backend", []string{}),
+		"deployable-react-frontend": projectconfig.NewModule(parameters, "zero-deployable-react-frontend", "github.com/something/repo3", "github.com/commitdev/zero-deployable-react-frontend", []string{}),
 	}
 }
 
@@ -74,3 +77,37 @@ modules:
       source: github.com/commitdev/zero-deployable-react-frontend
 `
 }
+
+func TestProjectConfigModuleGraph(t *testing.T) {
+	configPath := filepath.Join("../../../tests/test_data/projectconfig/", constants.ZeroProjectYml)
+
+	t.Run("Should generate a valid, correct graph based on the project config", func(t *testing.T) {
+		pc := projectconfig.LoadConfig(configPath)
+		graph := pc.GetDAG()
+
+		// Validate the graph
+		assert.NoError(t, graph.Validate())
+
+		// Check the structure of the graph
+		root, err := graph.Root()
+		assert.NoError(t, err)
+		assert.Equal(t, "graphRoot", root)
+
+		want := `graphRoot
+  project1
+project1
+  project2
+  project3
+project2
+  project4
+project3
+  project4
+  project5
+project4
+project5
+`
+		assert.Equal(t, want, graph.String())
+
+	})
+
+}

internal/config/projectconfig/project_config_test.go

@@ -29,7 +29,7 @@ func TestGenerateModules(t *testing.T) {
 	projectConfig := projectconfig.ZeroProjectConfig{
 		Name: "foo",
 		Modules: projectconfig.Modules{
-			"mod1": projectconfig.NewModule(map[string]string{"test": "bar"}, tmpDir, "n/a", baseTestFixturesDir),
+			"mod1": projectconfig.NewModule(map[string]string{"test": "bar"}, tmpDir, "n/a", baseTestFixturesDir, []string{}),
 		},
 	}
 

internal/generate/generate_test.go

@@ -68,7 +68,7 @@ func Init(outDir string) *projectconfig.ZeroProjectConfig {
 			}
 
 		}
-		projectConfig.Modules[moduleName] = projectconfig.NewModule(projectModuleParams, repoName, repoURL, mappedSources[moduleName])
+		projectConfig.Modules[moduleName] = projectconfig.NewModule(projectModuleParams, repoName, repoURL, mappedSources[moduleName], module.DependsOn)
 	}
 
 	// TODO: load ~/.zero/config.yml (or credentials)
@@ -308,11 +308,7 @@ func chooseStack(reg registry.Registry) []string {
 
 func defaultProjConfig() projectconfig.ZeroProjectConfig {
 	return projectconfig.ZeroProjectConfig{
-		Name: "",
-		Infrastructure: projectconfig.Infrastructure{
-			AWS: nil,
-		},
-
+		Name:       "",
 		Parameters: map[string]string{},
 		Modules:    projectconfig.Modules{},
 	}

internal/init/init.go

@@ -0,0 +1,34 @@
+# Graph shape:
+#    2
+#  /  \
+# 1    4
+#  \  /
+#   3 - 5
+
+name: graph_test
+
+modules:
+    project1:
+        parameters:
+            foo: bar
+    project2:
+        dependsOn:
+        - project1
+        parameters:
+            baz: qux
+    project3:
+        dependsOn:
+        - project1
+        parameters:
+            baz: qux
+    project4:
+        dependsOn:
+        - project2
+        - project3
+        parameters:
+            baz: qux
+    project5:
+        dependsOn:
+        - project3
+        parameters:
+            baz: qux