From 9dd05c1b86823b6c07c7effe946b96f3d816ca2c Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Tue, 12 Aug 2014 15:34:16 +0200 Subject: [PATCH 1/4] Initial addition of project template --- pkg/template/example/project.json | 157 ++++++++++++++++++++++++++++ pkg/template/generator.go | 124 ++++++++++++++++++++++ pkg/template/generator/generator.go | 39 +++++++ pkg/template/generator/helpers.go | 78 ++++++++++++++ pkg/template/generator/template.go | 65 ++++++++++++ pkg/template/generator_test.go | 61 +++++++++++ pkg/template/service_links.go | 72 +++++++++++++ pkg/template/service_links_test.go | 17 +++ pkg/template/types.go | 93 ++++++++++++++++ 9 files changed, 706 insertions(+) create mode 100644 pkg/template/example/project.json create mode 100644 pkg/template/generator.go create mode 100644 pkg/template/generator/generator.go create mode 100644 pkg/template/generator/helpers.go create mode 100644 pkg/template/generator/template.go create mode 100644 pkg/template/generator_test.go create mode 100644 pkg/template/service_links.go create mode 100644 pkg/template/service_links_test.go create mode 100644 pkg/template/types.go diff --git a/pkg/template/example/project.json b/pkg/template/example/project.json new file mode 100644 index 000000000000..105c44b4b1cf --- /dev/null +++ b/pkg/template/example/project.json @@ -0,0 +1,157 @@ +{ + "id": "example1", + "name": "my-awesome-php-app", + "description": "Example PHP application with PostgreSQL database", + "buildConfig": [ + { + "name": "mfojtik/nginx-php-app", + "type": "docker", + "sourceUri": "https://raw.githubusercontent.com/mfojtik/phpapp/master/Dockerfile", + "imageRepository": "int.registry.com:5000/mfojtik/phpapp" + }, + { + "name": "postgres", + "type": "docker", + "imageRepository": "registry.hub.docker.com/postgres", + "sourceUri": "https://raw.githubusercontent.com/docker-library/postgres/docker/9.2/Dockerfile" + } + ], + "imageRepository": [ + { + "name": "mfojtik/nginx-php-app", + "url": "internal.registry.com:5000/mfojtik/phpapp" + }, + { + "name": "postgres", + "url": "registry.hub.docker.com/postgres" + } + ], + "parameters": [ + { + "name": "DB_PASSWORD", + "description": "PostgreSQL admin user password", + "type": "string", + "generate": "[a-zA-Z0-9]{8}" + }, + { + "name": "DB_USER", + "description": "PostgreSQL username", + "type": "string", + "generate": "admin[a-zA-Z0-9]{4}" + }, + { + "name": "DB_NAME", + "description": "PostgreSQL database name", + "type": "string", + "generate": "[GET:http://broken.url/test]" + }, + { + "name": "SAMPLE_VAR", + "description": "Sample", + "type": "string", + "value": "foo" + } + ], + "serviceLinks": [ + { + "from": "database", + "export": [ + { + "name": "POSTGRES_ADMIN_USERNAME", + "value": "${DB_USER}" + }, + { + "name": "POSTGRES_ADMIN_PASSWORD", + "value": "${DB_PASSWORD}" + }, + { + "name": "POSTGRES_DATABASE_NAME", + "value": "${DB_NAME}" + } + ], + "to": "frontend" + } + ], + "services": [ + { + "name": "database", + "description": "Standalone PostgreSQL 9.2 database service", + "labels": { + "name": "database-service" + }, + "deploymentConfig": { + "deployment": { + "podTemplate": { + "containers": [ + { + "name": "postgresql-1", + "image": { + "name": "postgres", + "tag": "9.2" + }, + "env": [ + { + "name": "POSTGRES_ADMIN_USERNAME", + "value": "${DB_USER}" + }, + { + "name": "POSTGRES_ADMIN_PASSWORD", + "value": "${DB_PASSWORD}" + }, + { + "name": "POSTGRES_DATABASE_NAME", + "value": "${DB_NAME}" + }, + { + "name": "FOO", + "value": "${BAR}" + } + ], + "ports": [ + { + "containerPort": 5432, + "hostPort": 5432 + } + ] + } + ] + } + } + } + }, + { + "name": "frontend", + "description": "Sample PHP 5.2 application served by NGINX", + "labels": { + "name": "frontend-service" + }, + "deploymentConfig": { + "deployment": { + "podTemplate": { + "containers": [ + { + "name": "nginx-php-app", + "hooks": { + "prestart": { + "cmd": "import_database.sh" + }, + "url": "git://github.com/user/myapp-hooks.git" + }, + "image": { + "name": "mfojtik/nginx-php-app", + "tag": "latest" + }, + "ports": [ + { + "containerPort": 8080, + "hostPort": 8080 + } + ] + } + ] + } + } + } + } + ] +} diff --git a/pkg/template/generator.go b/pkg/template/generator.go new file mode 100644 index 000000000000..66497776edaf --- /dev/null +++ b/pkg/template/generator.go @@ -0,0 +1,124 @@ +package template + +import ( + "fmt" + "regexp" + "strings" + + "github.com/openshift/origin/pkg/template/generator" +) + +var valueExp = regexp.MustCompile(`(\$\{([a-zA-Z0-9\_]+)\})`) + +type ParamHash map[string]Parameter + +// Generate the value for the Parameter if the default Value is not set and the +// Generator field is specified. Otherwise, just return the default Value +func (p *Parameter) GenerateValue() error { + if p.Value != "" || p.Generate == "" { + return nil + } + + if p.Seed == nil { + return fmt.Errorf("The random seed is not initialized.") + } + + g := new(generator.Generator) + g.SetSeed(p.Seed) + generatedValue, err := g.Generate(p.Generate).Value() + + if err != nil { + return err + } + p.Value = generatedValue + + return nil +} + +// The string representation of PValue +// +func (s PValue) String() string { + return string(s) +} + +// Replace references to parameters in PValue with their values. +// The format is specified in the `valueExp` constant ${PARAM_NAME}. +// +// If the referenced parameter is not defined, then the substitution is ignored. +func (s *PValue) Substitute(params ParamHash) { + newValue := *s + + for _, match := range valueExp.FindAllStringSubmatch(string(newValue), -1) { + // If the Parameter is not defined, then leave the value as it is + if params[match[2]].Value == "" { + continue + } + newValue = PValue(strings.Replace(string(newValue), match[1], params[match[2]].Value, 1)) + } + + *s = newValue +} + +// Generate Value field for defined Parameters. +// If the Parameter define Generate, then the Value is generated based +// on that template. The template is a pseudo-regexp formatted string. +// +// Example: +// +// s := generate.Template("[a-zA-Z0-9]{4}") +// // s: "Ga0b" +// +// s := generate.Template("[GET:http://example.com/new]") +// // s: +func (p *Template) ProcessParameters() { + for i, _ := range p.Parameters { + p.Parameters[i].Seed = p.Seed + if err := p.Parameters[i].GenerateValue(); err != nil { + fmt.Printf("ERROR: Unable to process parameter %s: %v\n", p.Parameters[i].Name, err) + p.Parameters[i].Value = p.Parameters[i].Generate + } + } +} + +// A shorthand method to get list of *all* container defined in the Template +// template +func (p *Template) Containers() []*Container { + var result []*Container + for _, s := range p.Services { + result = append(result, s.Containers()...) + } + return result +} + +// Convert Parameter slice to more effective data structure +func (p *Template) ParameterHash() ParamHash { + paramHash := make(ParamHash) + for _, p := range p.Parameters { + paramHash[p.Name] = p + } + return paramHash +} + +// Process all Env variables in the Project template and replace parameters +// referenced in their values with the Parameter values. +// +// The replacement is done in Containers and ServiceLinks. +func (p *Template) SubstituteEnvValues() { + + params := p.ParameterHash() + + for _, container := range p.Containers() { + (*container).Env.Process(params) + } + + for s, _ := range p.ServiceLinks { + p.ServiceLinks[s].Export.Process(params) + } +} + +// Substitute referenced parameters in Env values with parameter values. +func (e *Env) Process(params ParamHash) { + for i, _ := range *e { + (*e)[i].Value.Substitute(params) + } +} diff --git a/pkg/template/generator/generator.go b/pkg/template/generator/generator.go new file mode 100644 index 000000000000..b84ea7f8847a --- /dev/null +++ b/pkg/template/generator/generator.go @@ -0,0 +1,39 @@ +package generator + +import ( + "fmt" + "math/rand" + "net/http" +) + +type GeneratorType interface { + Value() (string, error) +} + +type PasswordGenerator struct { + length int +} + +func (g PasswordGenerator) Value() (string, error) { + return Template{Expression: fmt.Sprintf("[\\a]{%d}", g.length)}.Value() +} + +type Generator struct { + seed *rand.Rand +} + +func (g *Generator) SetSeed(r *rand.Rand) { + g.seed = r +} + +func (g Generator) Generate(t string) GeneratorType { + if g.seed == nil { + return nil + } + switch t { + case "password": + return PasswordGenerator{length: 8} + default: + return Template{Expression: t, HttpClient: &http.Client{}, seed: g.seed} + } +} diff --git a/pkg/template/generator/helpers.go b/pkg/template/generator/helpers.go new file mode 100644 index 000000000000..d73b2c6f4f13 --- /dev/null +++ b/pkg/template/generator/helpers.go @@ -0,0 +1,78 @@ +package generator + +import ( + "fmt" + "math/rand" + "strconv" + "strings" +) + +func alphabetSlice(from, to byte) (string, error) { + leftPos := strings.Index(Ascii, string(from)) + rightPos := strings.LastIndex(Ascii, string(to)) + if leftPos > rightPos { + return "", fmt.Errorf("Invalid range specified: %s-%s", string(from), string(to)) + } + return Ascii[leftPos:rightPos], nil +} + +func replaceWithGenerated(s *string, expresion string, ranges [][]byte, length int, seed *rand.Rand) error { + var alphabet string + for _, r := range ranges { + switch string(r[0]) + string(r[1]) { + case `\w`: + alphabet += Ascii + case `\d`: + alphabet += Numerals + case `\a`: + alphabet += Alphabet + Numerals + default: + if slice, err := alphabetSlice(r[0], r[1]); err != nil { + return err + } else { + alphabet += slice + } + } + } + if len(alphabet) == 0 { + return fmt.Errorf("Empty range in expresion: %s", expresion) + } + result := make([]byte, length, length) + for i := 0; i <= length-1; i++ { + result[i] = alphabet[seed.Intn(len(alphabet))] + } + *s = strings.Replace(*s, expresion, string(result), 1) + return nil +} + +func findExpresionPos(s string) GeneratorExprRanges { + matches := rangeExp.FindAllStringIndex(s, -1) + result := make(GeneratorExprRanges, len(matches), len(matches)) + for i, r := range matches { + result[i] = []byte{s[r[0]], s[r[1]-1]} + } + return result +} + +func rangesAndLength(s string) (string, int, error) { + l := strings.LastIndex(s, "{") + // If the length ({}) is not specified in expresion, + // then assume the length is 1 character + // + if l > 0 { + expr := s[0:strings.LastIndex(s, "{")] + length, err := parseLength(s) + return expr, length, err + } else { + return s, 1, nil + } +} + +func parseLength(s string) (int, error) { + lengthStr := string(s[strings.LastIndex(s, "{")+1 : len(s)-1]) + if l, err := strconv.Atoi(lengthStr); err != nil { + return 0, fmt.Errorf("Unable to parse length from %v", s) + } else { + return l, nil + } +} diff --git a/pkg/template/generator/template.go b/pkg/template/generator/template.go new file mode 100644 index 000000000000..80eba86b0553 --- /dev/null +++ b/pkg/template/generator/template.go @@ -0,0 +1,65 @@ +package generator + +import ( + "io/ioutil" + "math/rand" + "net/http" + "regexp" + "strings" +) + +type Template struct { + Expression string + HttpClient *http.Client + seed *rand.Rand +} + +const ( + Alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + Numerals = "0123456789" + Ascii = Alphabet + Numerals + "~!@#$%^&*()-_+={}[]\\|<,>.?/\"';:`" +) + +var ( + rangeExp = regexp.MustCompile(`([\\]?[a-zA-Z0-9]\-?[a-zA-Z0-9]?)`) + generatorsExp = regexp.MustCompile(`\[([a-zA-Z0-9\-\\]+)\](\{([0-9]+)\})`) + remoteExp = regexp.MustCompile(`\[GET\:(http(s)?:\/\/(.+))\]`) +) + +type GeneratorExprRanges [][]byte + +func (t Template) Value() (string, error) { + result := t.Expression + genMatches := generatorsExp.FindAllStringIndex(t.Expression, -1) + remMatches := remoteExp.FindAllStringIndex(t.Expression, -1) + + // Parse [a-zA-Z0-9]{length} types + // + for _, r := range genMatches { + ranges, length, err := rangesAndLength(t.Expression[r[0]:r[1]]) + if err != nil { + return "", err + } + positions := findExpresionPos(ranges) + err = replaceWithGenerated(&result, t.Expression[r[0]:r[1]], positions, length, t.seed) + if err != nil { + return "", err + } + } + + // Parse [GET:] parameters + // + for _, r := range remMatches { + response, err := t.HttpClient.Get(t.Expression[5 : len(t.Expression)-1]) + if err != nil { + return "", err + } + defer response.Body.Close() + val, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", err + } + result = strings.Replace(result, t.Expression[r[0]:r[1]], strings.TrimSpace(string(val)), 1) + } + return result, nil +} diff --git a/pkg/template/generator_test.go b/pkg/template/generator_test.go new file mode 100644 index 000000000000..404ae1995a1e --- /dev/null +++ b/pkg/template/generator_test.go @@ -0,0 +1,61 @@ +package template + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "strings" + "testing" + "time" +) + +const projectExampleJSON = "./example/project.json" + +var projectTempl Template + +func TestTemplateUnmarshal(t *testing.T) { + jsonFile, _ := ioutil.ReadFile(projectExampleJSON) + err := json.Unmarshal(jsonFile, &projectTempl) + if err != nil { + t.Errorf("Unable to parse the sample project.json: %v", err) + } + projectTempl.Seed = rand.New(rand.NewSource(time.Now().UnixNano())) +} + +func TestProcessParameters(t *testing.T) { + projectTempl.ProcessParameters() + + for _, p := range projectTempl.Parameters { + if p.Value == "" { + t.Errorf("Failed to process '%s' parameter", p.Name) + } + fmt.Printf("%s -> %s = %s\n", p.Name, p.Generate, p.Value) + } +} + +func TestSubstituteEnvValues(t *testing.T) { + projectTempl.SubstituteEnvValues() + + for _, c := range projectTempl.Containers() { + for _, e := range c.Env { + if strings.Contains(string(e.Value), "${") { + if e.Name != "FOO" { + t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) + } + } + fmt.Printf("%s=%s\n", e.Name, e.Value) + } + } + + for _, s := range projectTempl.ServiceLinks { + for _, e := range s.Export { + if strings.Contains(string(e.Value), "${") { + if e.Name != "FOO" { + t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) + } + } + fmt.Printf("%s=%s\n", e.Name, e.Value) + } + } +} diff --git a/pkg/template/service_links.go b/pkg/template/service_links.go new file mode 100644 index 000000000000..ea5a6f6f7f13 --- /dev/null +++ b/pkg/template/service_links.go @@ -0,0 +1,72 @@ +package template + +import "fmt" + +func (e *Env) Append(env *Env) { + *e = append(*e, *env...) +} + +func (e *Env) Exists(name string) bool { + for _, env := range *e { + if env.Name == name { + return true + } + } + return false +} + +func (s *Service) Containers() []*Container { + result := make([]*Container, len((*s).DeploymentConfig.Deployment.PodTemplate.Containers)) + + for i, _ := range s.DeploymentConfig.Deployment.PodTemplate.Containers { + result[i] = &s.DeploymentConfig.Deployment.PodTemplate.Containers[i] + } + + return result +} + +func (s *Service) ContainersEnv() []*Env { + var result []*Env + for _, c := range s.Containers() { + result = append(result, &c.Env) + } + return result +} + +func (s *Service) AddEnv(env Env) { + fmt.Printf("s.Containers() %+v\n", s.Containers()) + for _, c := range s.Containers() { + (*c).Env = append(c.Env, env...) + } +} + +func (p *Template) ServiceByName(name string) *Service { + for i, _ := range p.Services { + if p.Services[i].Name == name { + return &p.Services[i] + } + } + return nil +} + +func (p *Template) ProcessServiceLinks() { + var ( + fromService, toService *Service + ) + + for i, _ := range p.ServiceLinks { + fromService = p.ServiceByName(p.ServiceLinks[i].From) + if fromService == nil { + fmt.Printf("ERROR: Invalid FROM service in links: %+v\n", p.ServiceLinks[i].From) + continue + } + + toService = p.ServiceByName(p.ServiceLinks[i].To) + if toService == nil { + fmt.Printf("ERROR: Invalid TO service in links: %+v\n", p.ServiceLinks[i].To) + continue + } + + toService.AddEnv(p.ServiceLinks[i].Export) + } +} diff --git a/pkg/template/service_links_test.go b/pkg/template/service_links_test.go new file mode 100644 index 000000000000..679ac9ee4292 --- /dev/null +++ b/pkg/template/service_links_test.go @@ -0,0 +1,17 @@ +package template + +import "testing" + +func TestServiceLinks(t *testing.T) { + projectTempl.ProcessParameters() + projectTempl.ProcessServiceLinks() + + s := projectTempl.ServiceByName("frontend") + for _, env := range s.ContainersEnv() { + for _, export := range projectTempl.ServiceLinks[0].Export { + if env.Exists(export.Name) == false { + t.Errorf("Failed to export %s variable via serviceLinks to %s", export.Name, s.Name) + } + } + } +} diff --git a/pkg/template/types.go b/pkg/template/types.go new file mode 100644 index 000000000000..97f057a1703c --- /dev/null +++ b/pkg/template/types.go @@ -0,0 +1,93 @@ +package template + +import ( + "math/rand" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" +) + +type ( + Uri string + PValue string +) + +type Template struct { + api.JSONBase `json:",inline" yaml:",inline"` + BuildConfig []BuildConfig `json:"buildConfig" yaml:"buildConfig"` + ImageRepository []ImageRepository `json:"imageRepository" yaml:"imageRepository"` + Parameters []Parameter `json:"parameters" yaml:"parameters"` + ServiceLinks []ServiceLink `json:"serviceLinks" yaml:"serviceLinks"` + Services []Service `json:"services" yaml:"services"` + + Seed *rand.Rand +} + +type ImageRepository struct { + Name string `json:"name" yaml:"name"` + Url Uri `json:"url" yaml:"url"` +} + +type BuildConfig struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + SourceUri Uri `json:"sourceUri" yaml:"sourceUri"` + ImageRepository string `json:"imageRepository" yaml:"imageRepository"` +} + +type Parameter struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + Type string `json:"type" yaml:"type"` + Generate string `json:"generate" yaml:"generate"` + Value string `json:"value" yaml:"value"` + + Seed *rand.Rand +} + +type Env []struct { + Name string `json:"name" yaml:"name"` + Value PValue `json:"value" yaml:"value"` +} + +type ServiceLink struct { + From string `json:"from" yaml:"from"` + To string `json:"to" yaml:"to"` + Export Env `json:"export" yaml:"export"` +} + +type DeploymentConfig struct { + Deployment Deployment `json:"deployment" yaml:"deployment"` +} + +type Deployment struct { + PodTemplate PodTemplate `json:"podTemplate" yaml:"podTemplate"` +} + +type PodTemplate struct { + Containers []Container `json:"containers" yaml:"containers"` + Replicas int `json:"replicas" yaml:"replicas"` +} + +type Image struct { + Name string `json:"name" yaml:"name"` + Tag string `json:"tag" yaml:"tag"` +} + +type ContainerPort struct { + ContainerPort int `json:"containerPort" yaml:"containerPort"` + HostPort int `json:"hostPort" yaml:"hostPort"` +} + +type Container struct { + Name string `json:"name" yaml:"name"` + Image Image `json:"image" yaml:"image"` + Env Env `json:"env" yaml:"env"` + Ports []ContainerPort `json:"ports" yaml:"ports"` +} + +type Service struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + Labels map[string]PValue `json:"labels" yaml:"labels"` + DeploymentConfig DeploymentConfig `json:"deploymentConfig" yaml:"deploymentConfig"` +} From 83fa72a7efedf06f119892aa70c540ca4fa7aef0 Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Tue, 19 Aug 2014 12:26:26 +0200 Subject: [PATCH 2/4] Removed serviceLink and added ExampleTemplate_Transform() --- pkg/template/example/simple.json | 89 ++++++++++++++++++++++ pkg/template/generator_test.go | 61 --------------- pkg/template/service_links.go | 72 ----------------- pkg/template/service_links_test.go | 17 ----- pkg/template/{generator.go => template.go} | 57 +++++++++++--- pkg/template/template_test.go | 72 +++++++++++++++++ 6 files changed, 208 insertions(+), 160 deletions(-) create mode 100644 pkg/template/example/simple.json delete mode 100644 pkg/template/generator_test.go delete mode 100644 pkg/template/service_links.go delete mode 100644 pkg/template/service_links_test.go rename pkg/template/{generator.go => template.go} (69%) create mode 100644 pkg/template/template_test.go diff --git a/pkg/template/example/simple.json b/pkg/template/example/simple.json new file mode 100644 index 000000000000..b7a9dabfa867 --- /dev/null +++ b/pkg/template/example/simple.json @@ -0,0 +1,89 @@ +{ + "id": "example2", + "name": "my-awesome-php-app", + "description": "Example PHP application with PostgreSQL database", + "parameters": [ + { + "name": "DB_PASSWORD", + "description": "PostgreSQL admin user password", + "type": "string", + "generate": "[a-zA-Z0-9]{8}" + }, + { + "name": "DB_USER", + "description": "PostgreSQL username", + "type": "string", + "generate": "admin[a-zA-Z0-9]{4}" + }, + { + "name": "SAMPLE_VAR", + "description": "Sample", + "type": "string", + "value": "foo" + } + ], + "serviceLinks": [ + { + "from": "database", + "export": [ + { + "name": "POSTGRES_ADMIN_USERNAME", + "value": "${DB_USER}" + }, + { + "name": "POSTGRES_ADMIN_PASSWORD", + "value": "${DB_PASSWORD}" + }, + { + "name": "POSTGRES_DATABASE_NAME", + "value": "${DB_NAME}" + } + ], + "to": "frontend" + } + ], + "services": [ + { + "name": "database", + "description": "Standalone PostgreSQL 9.2 database service", + "labels": { + "name": "database-service" + }, + "deploymentConfig": { + "deployment": { + "podTemplate": { + "containers": [ + { + "name": "postgresql-1", + "image": { + "name": "postgres", + "tag": "9.2" + }, + "env": [ + { + "name": "POSTGRES_ADMIN_USERNAME", + "value": "${DB_USER}" + }, + { + "name": "POSTGRES_ADMIN_PASSWORD", + "value": "${DB_PASSWORD}" + }, + { + "name": "FOO", + "value": "${BAR}" + } + ], + "ports": [ + { + "containerPort": 5432, + "hostPort": 5432 + } + ] + } + ] + } + } + } + } + ] +} diff --git a/pkg/template/generator_test.go b/pkg/template/generator_test.go deleted file mode 100644 index 404ae1995a1e..000000000000 --- a/pkg/template/generator_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package template - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "math/rand" - "strings" - "testing" - "time" -) - -const projectExampleJSON = "./example/project.json" - -var projectTempl Template - -func TestTemplateUnmarshal(t *testing.T) { - jsonFile, _ := ioutil.ReadFile(projectExampleJSON) - err := json.Unmarshal(jsonFile, &projectTempl) - if err != nil { - t.Errorf("Unable to parse the sample project.json: %v", err) - } - projectTempl.Seed = rand.New(rand.NewSource(time.Now().UnixNano())) -} - -func TestProcessParameters(t *testing.T) { - projectTempl.ProcessParameters() - - for _, p := range projectTempl.Parameters { - if p.Value == "" { - t.Errorf("Failed to process '%s' parameter", p.Name) - } - fmt.Printf("%s -> %s = %s\n", p.Name, p.Generate, p.Value) - } -} - -func TestSubstituteEnvValues(t *testing.T) { - projectTempl.SubstituteEnvValues() - - for _, c := range projectTempl.Containers() { - for _, e := range c.Env { - if strings.Contains(string(e.Value), "${") { - if e.Name != "FOO" { - t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) - } - } - fmt.Printf("%s=%s\n", e.Name, e.Value) - } - } - - for _, s := range projectTempl.ServiceLinks { - for _, e := range s.Export { - if strings.Contains(string(e.Value), "${") { - if e.Name != "FOO" { - t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) - } - } - fmt.Printf("%s=%s\n", e.Name, e.Value) - } - } -} diff --git a/pkg/template/service_links.go b/pkg/template/service_links.go deleted file mode 100644 index ea5a6f6f7f13..000000000000 --- a/pkg/template/service_links.go +++ /dev/null @@ -1,72 +0,0 @@ -package template - -import "fmt" - -func (e *Env) Append(env *Env) { - *e = append(*e, *env...) -} - -func (e *Env) Exists(name string) bool { - for _, env := range *e { - if env.Name == name { - return true - } - } - return false -} - -func (s *Service) Containers() []*Container { - result := make([]*Container, len((*s).DeploymentConfig.Deployment.PodTemplate.Containers)) - - for i, _ := range s.DeploymentConfig.Deployment.PodTemplate.Containers { - result[i] = &s.DeploymentConfig.Deployment.PodTemplate.Containers[i] - } - - return result -} - -func (s *Service) ContainersEnv() []*Env { - var result []*Env - for _, c := range s.Containers() { - result = append(result, &c.Env) - } - return result -} - -func (s *Service) AddEnv(env Env) { - fmt.Printf("s.Containers() %+v\n", s.Containers()) - for _, c := range s.Containers() { - (*c).Env = append(c.Env, env...) - } -} - -func (p *Template) ServiceByName(name string) *Service { - for i, _ := range p.Services { - if p.Services[i].Name == name { - return &p.Services[i] - } - } - return nil -} - -func (p *Template) ProcessServiceLinks() { - var ( - fromService, toService *Service - ) - - for i, _ := range p.ServiceLinks { - fromService = p.ServiceByName(p.ServiceLinks[i].From) - if fromService == nil { - fmt.Printf("ERROR: Invalid FROM service in links: %+v\n", p.ServiceLinks[i].From) - continue - } - - toService = p.ServiceByName(p.ServiceLinks[i].To) - if toService == nil { - fmt.Printf("ERROR: Invalid TO service in links: %+v\n", p.ServiceLinks[i].To) - continue - } - - toService.AddEnv(p.ServiceLinks[i].Export) - } -} diff --git a/pkg/template/service_links_test.go b/pkg/template/service_links_test.go deleted file mode 100644 index 679ac9ee4292..000000000000 --- a/pkg/template/service_links_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package template - -import "testing" - -func TestServiceLinks(t *testing.T) { - projectTempl.ProcessParameters() - projectTempl.ProcessServiceLinks() - - s := projectTempl.ServiceByName("frontend") - for _, env := range s.ContainersEnv() { - for _, export := range projectTempl.ServiceLinks[0].Export { - if env.Exists(export.Name) == false { - t.Errorf("Failed to export %s variable via serviceLinks to %s", export.Name, s.Name) - } - } - } -} diff --git a/pkg/template/generator.go b/pkg/template/template.go similarity index 69% rename from pkg/template/generator.go rename to pkg/template/template.go index 66497776edaf..8a86ad716031 100644 --- a/pkg/template/generator.go +++ b/pkg/template/template.go @@ -1,16 +1,20 @@ package template import ( + "encoding/json" "fmt" + "io/ioutil" + "math/rand" "regexp" "strings" + "time" "github.com/openshift/origin/pkg/template/generator" ) var valueExp = regexp.MustCompile(`(\$\{([a-zA-Z0-9\_]+)\})`) -type ParamHash map[string]Parameter +type ParamMap map[string]Parameter // Generate the value for the Parameter if the default Value is not set and the // Generator field is specified. Otherwise, just return the default Value @@ -45,7 +49,7 @@ func (s PValue) String() string { // The format is specified in the `valueExp` constant ${PARAM_NAME}. // // If the referenced parameter is not defined, then the substitution is ignored. -func (s *PValue) Substitute(params ParamHash) { +func (s *PValue) Substitute(params ParamMap) { newValue := *s for _, match := range valueExp.FindAllStringSubmatch(string(newValue), -1) { @@ -80,6 +84,17 @@ func (p *Template) ProcessParameters() { } } +// Return the list of containers associated with the service +func (s *Service) Containers() []*Container { + result := make([]*Container, len((*s).DeploymentConfig.Deployment.PodTemplate.Containers)) + + for i, _ := range s.DeploymentConfig.Deployment.PodTemplate.Containers { + result[i] = &s.DeploymentConfig.Deployment.PodTemplate.Containers[i] + } + + return result +} + // A shorthand method to get list of *all* container defined in the Template // template func (p *Template) Containers() []*Container { @@ -91,21 +106,20 @@ func (p *Template) Containers() []*Container { } // Convert Parameter slice to more effective data structure -func (p *Template) ParameterHash() ParamHash { - paramHash := make(ParamHash) +func (p *Template) CreateParameterMap() ParamMap { + ParamMap := make(ParamMap) for _, p := range p.Parameters { - paramHash[p.Name] = p + ParamMap[p.Name] = p } - return paramHash + return ParamMap } // Process all Env variables in the Project template and replace parameters // referenced in their values with the Parameter values. // // The replacement is done in Containers and ServiceLinks. -func (p *Template) SubstituteEnvValues() { - - params := p.ParameterHash() +func (p *Template) Process() { + params := p.CreateParameterMap() for _, container := range p.Containers() { (*container).Env.Process(params) @@ -117,8 +131,31 @@ func (p *Template) SubstituteEnvValues() { } // Substitute referenced parameters in Env values with parameter values. -func (e *Env) Process(params ParamHash) { +func (e *Env) Process(params ParamMap) { for i, _ := range *e { (*e)[i].Value.Substitute(params) } } + +func (t *Template) Transform() ([]byte, error) { + t.ProcessParameters() + t.Process() + return json.Marshal(*t) +} + +func NewTemplate(jsonData []byte) (*Template, error) { + var template Template + if err := json.Unmarshal(jsonData, &template); err != nil { + return nil, err + } + template.Seed = rand.New(rand.NewSource(time.Now().UnixNano())) + return &template, nil +} + +func NewTemplateFromFile(filename string) (*Template, error) { + jsonData, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return NewTemplate(jsonData) +} diff --git a/pkg/template/template_test.go b/pkg/template/template_test.go new file mode 100644 index 000000000000..baf7314b0971 --- /dev/null +++ b/pkg/template/template_test.go @@ -0,0 +1,72 @@ +package template + +import ( + "fmt" + "math/rand" + "strings" + "testing" +) + +var sampleTemplate *Template + +func TestNewTemplate(t *testing.T) { + var err error + sampleTemplate, err = NewTemplateFromFile("example/project.json") + if err != nil { + t.Errorf("Unable to process the JSON template: %v", err) + } +} + +func TestProcessParameters(t *testing.T) { + sampleTemplate.ProcessParameters() + + for _, p := range sampleTemplate.Parameters { + if p.Value == "" { + t.Errorf("Failed to process '%s' parameter", p.Name) + } + fmt.Printf("%s -> %s = %s\n", p.Name, p.Generate, p.Value) + } +} + +func TestProcessTemplate(t *testing.T) { + sampleTemplate.Process() + + for _, c := range sampleTemplate.Containers() { + for _, e := range c.Env { + if strings.Contains(string(e.Value), "${") { + if e.Name != "FOO" { + t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) + } + } + fmt.Printf("%s=%s\n", e.Name, e.Value) + } + } + + for _, s := range sampleTemplate.ServiceLinks { + for _, e := range s.Export { + if strings.Contains(string(e.Value), "${") { + if e.Name != "FOO" { + t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) + } + } + fmt.Printf("%s=%s\n", e.Name, e.Value) + } + } +} + +func ExampleTemplate_Transform() { + template, err := NewTemplateFromFile("example/simple.json") + if err != nil { + fmt.Printf("Unable to process example/simple.json template: %v", err) + } + + // In this example, we want always produce the same result: + // + template.Seed = rand.New(rand.NewSource(1337)) + + result, _ := template.Transform() + + fmt.Println(string(result)) + // Output: + // {"id":"example2","buildConfig":null,"imageRepository":null,"parameters":[{"name":"DB_PASSWORD","description":"PostgreSQL admin user password","type":"string","generate":"[a-zA-Z0-9]{8}","value":"bQPdwNJi","Seed":{}},{"name":"DB_USER","description":"PostgreSQL username","type":"string","generate":"admin[a-zA-Z0-9]{4}","value":"adminJwWP","Seed":{}},{"name":"SAMPLE_VAR","description":"Sample","type":"string","generate":"","value":"foo","Seed":{}}],"serviceLinks":[{"from":"database","to":"frontend","export":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"POSTGRES_DATABASE_NAME","value":"${DB_NAME}"}]}],"services":[{"name":"database","description":"Standalone PostgreSQL 9.2 database service","labels":{"name":"database-service"},"deploymentConfig":{"deployment":{"podTemplate":{"containers":[{"name":"postgresql-1","image":{"name":"postgres","tag":"9.2"},"env":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"FOO","value":"${BAR}"}],"ports":[{"containerPort":5432,"hostPort":5432}]}],"replicas":0}}}}],"Seed":{}} +} From c8fd12014619d3f1427e43937f47399e9a6b6ba8 Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Tue, 19 Aug 2014 13:21:18 +0200 Subject: [PATCH 3/4] Added possibility to define custom parameters for transformation --- pkg/template/example/simple.json | 2 +- pkg/template/template.go | 7 ++++--- pkg/template/template_test.go | 11 ++++++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/template/example/simple.json b/pkg/template/example/simple.json index b7a9dabfa867..45357c31b099 100644 --- a/pkg/template/example/simple.json +++ b/pkg/template/example/simple.json @@ -70,7 +70,7 @@ }, { "name": "FOO", - "value": "${BAR}" + "value": "${CUSTOM_PARAM1}" } ], "ports": [ diff --git a/pkg/template/template.go b/pkg/template/template.go index 8a86ad716031..882839ff109d 100644 --- a/pkg/template/template.go +++ b/pkg/template/template.go @@ -74,7 +74,8 @@ func (s *PValue) Substitute(params ParamMap) { // // s := generate.Template("[GET:http://example.com/new]") // // s: -func (p *Template) ProcessParameters() { +func (p *Template) ProcessParameters(customParams []Parameter) { + p.Parameters = append(p.Parameters, customParams...) for i, _ := range p.Parameters { p.Parameters[i].Seed = p.Seed if err := p.Parameters[i].GenerateValue(); err != nil { @@ -137,8 +138,8 @@ func (e *Env) Process(params ParamMap) { } } -func (t *Template) Transform() ([]byte, error) { - t.ProcessParameters() +func (t *Template) Transform(customParams []Parameter) ([]byte, error) { + t.ProcessParameters(customParams) t.Process() return json.Marshal(*t) } diff --git a/pkg/template/template_test.go b/pkg/template/template_test.go index baf7314b0971..5e6eade4824e 100644 --- a/pkg/template/template_test.go +++ b/pkg/template/template_test.go @@ -18,7 +18,7 @@ func TestNewTemplate(t *testing.T) { } func TestProcessParameters(t *testing.T) { - sampleTemplate.ProcessParameters() + sampleTemplate.ProcessParameters([]Parameter{}) for _, p := range sampleTemplate.Parameters { if p.Value == "" { @@ -60,13 +60,18 @@ func ExampleTemplate_Transform() { fmt.Printf("Unable to process example/simple.json template: %v", err) } + customParams := make([]Parameter, 3) + customParams[0] = Parameter{Name: "CUSTOM_PARAM1", Value: "1"} + customParams[1] = Parameter{Name: "CUSTOM_PARAM2", Value: "2"} + customParams[2] = Parameter{Name: "CUSTOM_PARAM3", Value: "3"} + // In this example, we want always produce the same result: // template.Seed = rand.New(rand.NewSource(1337)) - result, _ := template.Transform() + result, _ := template.Transform(customParams) fmt.Println(string(result)) // Output: - // {"id":"example2","buildConfig":null,"imageRepository":null,"parameters":[{"name":"DB_PASSWORD","description":"PostgreSQL admin user password","type":"string","generate":"[a-zA-Z0-9]{8}","value":"bQPdwNJi","Seed":{}},{"name":"DB_USER","description":"PostgreSQL username","type":"string","generate":"admin[a-zA-Z0-9]{4}","value":"adminJwWP","Seed":{}},{"name":"SAMPLE_VAR","description":"Sample","type":"string","generate":"","value":"foo","Seed":{}}],"serviceLinks":[{"from":"database","to":"frontend","export":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"POSTGRES_DATABASE_NAME","value":"${DB_NAME}"}]}],"services":[{"name":"database","description":"Standalone PostgreSQL 9.2 database service","labels":{"name":"database-service"},"deploymentConfig":{"deployment":{"podTemplate":{"containers":[{"name":"postgresql-1","image":{"name":"postgres","tag":"9.2"},"env":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"FOO","value":"${BAR}"}],"ports":[{"containerPort":5432,"hostPort":5432}]}],"replicas":0}}}}],"Seed":{}} + // {"id":"example2","buildConfig":null,"imageRepository":null,"parameters":[{"name":"DB_PASSWORD","description":"PostgreSQL admin user password","type":"string","generate":"[a-zA-Z0-9]{8}","value":"bQPdwNJi","Seed":{}},{"name":"DB_USER","description":"PostgreSQL username","type":"string","generate":"admin[a-zA-Z0-9]{4}","value":"adminJwWP","Seed":{}},{"name":"SAMPLE_VAR","description":"Sample","type":"string","generate":"","value":"foo","Seed":{}},{"name":"CUSTOM_PARAM1","description":"","type":"","generate":"","value":"1","Seed":{}},{"name":"CUSTOM_PARAM2","description":"","type":"","generate":"","value":"2","Seed":{}},{"name":"CUSTOM_PARAM3","description":"","type":"","generate":"","value":"3","Seed":{}}],"serviceLinks":[{"from":"database","to":"frontend","export":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"POSTGRES_DATABASE_NAME","value":"${DB_NAME}"}]}],"services":[{"name":"database","description":"Standalone PostgreSQL 9.2 database service","labels":{"name":"database-service"},"deploymentConfig":{"deployment":{"podTemplate":{"containers":[{"name":"postgresql-1","image":{"name":"postgres","tag":"9.2"},"env":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"FOO","value":"1"}],"ports":[{"containerPort":5432,"hostPort":5432}]}],"replicas":0}}}}],"Seed":{}} } From d5cb3b3475f36c9f442e97192b36baa3b95073e8 Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Wed, 20 Aug 2014 13:28:45 +0200 Subject: [PATCH 4/4] xxx --- pkg/template/example/project.json | 163 +++++++++++++++++------------- pkg/template/parameters.go | 69 +++++++++++++ pkg/template/template.go | 151 ++++++--------------------- pkg/template/template_test.go | 57 ++--------- pkg/template/types.go | 77 ++++++-------- 5 files changed, 234 insertions(+), 283 deletions(-) create mode 100644 pkg/template/parameters.go diff --git a/pkg/template/example/project.json b/pkg/template/example/project.json index 105c44b4b1cf..c1774e513183 100644 --- a/pkg/template/example/project.json +++ b/pkg/template/example/project.json @@ -2,21 +2,21 @@ "id": "example1", "name": "my-awesome-php-app", "description": "Example PHP application with PostgreSQL database", - "buildConfig": [ + "buildConfigs": [ { "name": "mfojtik/nginx-php-app", "type": "docker", "sourceUri": "https://raw.githubusercontent.com/mfojtik/phpapp/master/Dockerfile", - "imageRepository": "int.registry.com:5000/mfojtik/phpapp" + "imageRepository": "mfojtik/nginx-php-app" }, { "name": "postgres", "type": "docker", - "imageRepository": "registry.hub.docker.com/postgres", - "sourceUri": "https://raw.githubusercontent.com/docker-library/postgres/docker/9.2/Dockerfile" + "sourceUri": "https://raw.githubusercontent.com/docker-library/postgres/docker/9.2/Dockerfile", + "imageRepository": "postgres" } ], - "imageRepository": [ + "imageRepositories": [ { "name": "mfojtik/nginx-php-app", "url": "internal.registry.com:5000/mfojtik/phpapp" @@ -43,63 +43,69 @@ "name": "DB_NAME", "description": "PostgreSQL database name", "type": "string", - "generate": "[GET:http://broken.url/test]" + "value": "mydb" }, { - "name": "SAMPLE_VAR", - "description": "Sample", + "name": "REMOTE_KEY", + "description": "Example of remote key", "type": "string", - "value": "foo" + "value": "[GET:http://custom.url.int]" } ], - "serviceLinks": [ + "services": [ { - "from": "database", - "export": [ - { - "name": "POSTGRES_ADMIN_USERNAME", - "value": "${DB_USER}" - }, - { - "name": "POSTGRES_ADMIN_PASSWORD", - "value": "${DB_PASSWORD}" - }, - { - "name": "POSTGRES_DATABASE_NAME", - "value": "${DB_NAME}" - } - ], - "to": "frontend" + "id": "database", + "kind": "Service", + "apiVersion": "v1beta1", + "port": 5432, + "selector": { + "name": "database" + } + }, + { + "id": "frontend", + "kind": "Service", + "apiVersion": "v1beta1", + "port": 8080, + "selector": { + "name": "frontend" + } } ], - "services": [ + "deploymentConfigs": [ { - "name": "database", - "description": "Standalone PostgreSQL 9.2 database service", + "kind": "DeploymentConfig", + "apiVersion": "v1beta1", "labels": { - "name": "database-service" + "name": "database" }, - "deploymentConfig": { - "deployment": { - "podTemplate": { - "containers": [ - { - "name": "postgresql-1", - "image": { - "name": "postgres", - "tag": "9.2" - }, + "desiredState": { + "replicas": 2, + "replicaSelector": { + "name": "database" + }, + "podTemplate": { + "kind": "Pod", + "apiVersion": "v1beta1", + "id": "database", + "desiredState": { + "manifest": { + "version": "v1beta1", + "id": "database", + "containers": [{ + "name": "postgresql", + "image": "postgres", "env": [ { - "name": "POSTGRES_ADMIN_USERNAME", - "value": "${DB_USER}" + "name": "PGPASSWORD", + "value": "${DB_PASSWORD}" }, { - "name": "POSTGRES_ADMIN_PASSWORD", - "value": "${DB_PASSWORD}" + "name": "PGUSER", + "value": "${DB_USER}" }, { - "name": "POSTGRES_DATABASE_NAME", + "name": "PGDATABASE", "value": "${DB_NAME}" }, { @@ -109,46 +115,67 @@ ], "ports": [ { - "containerPort": 5432, - "hostPort": 5432 + "containerPort": 5432 } ] } - ] + ] + } + }, + "labels": { + "name": "database" } } } }, { - "name": "frontend", - "description": "Sample PHP 5.2 application served by NGINX", + "kind": "DeploymentConfig", + "apiVersion": "v1beta1", "labels": { - "name": "frontend-service" + "name": "frontend" }, - "deploymentConfig": { - "deployment": { - "podTemplate": { - "containers": [ - { - "name": "nginx-php-app", - "hooks": { - "prestart": { - "cmd": "import_database.sh" + "desiredState": { + "replicas": 2, + "replicaSelector": { + "name": "frontend" + }, + "podTemplate": { + "kind": "Pod", + "apiVersion": "v1beta1", + "id": "frontend", + "desiredState": { + "manifest": { + "version": "v1beta1", + "id": "frontend", + "containers": [{ + "name": "frontend", + "image": "mfojtik/nginx-php-app", + "env": [ + { + "name": "PGPASSWORD", + "value": "${DB_PASSWORD}" + }, + { + "name": "PGUSER", + "value": "${DB_USER}" }, - "url": "git://github.com/user/myapp-hooks.git" - }, - "image": { - "name": "mfojtik/nginx-php-app", - "tag": "latest" - }, + { + "name": "PGDATABASE", + "value": "${DB_NAME}" + } + ], "ports": [ { - "containerPort": 8080, + "containerPort": 9292, "hostPort": 8080 } ] } - ] + ] + } + }, + "labels": { + "name": "frontend" } } } diff --git a/pkg/template/parameters.go b/pkg/template/parameters.go new file mode 100644 index 000000000000..73a301f3993e --- /dev/null +++ b/pkg/template/parameters.go @@ -0,0 +1,69 @@ +package template + +import ( + "fmt" + + "github.com/openshift/origin/pkg/template/generator" +) + +type ParamMap map[string]Parameter + +// Generate the value for the Parameter if the default Value is not set and the +// Generator field is specified. Otherwise, just return the default Value +func (p *Parameter) GenerateValue() error { + if p.Value != "" || p.Generate == "" { + return nil + } + + if p.Seed == nil { + return fmt.Errorf("The random seed is not initialized.") + } + + g := new(generator.Generator) + g.SetSeed(p.Seed) + generatedValue, err := g.Generate(p.Generate).Value() + + if err != nil { + return err + } + p.Value = generatedValue + + return nil +} + +// The string representation of PValue +// +func (s PValue) String() string { + return string(s) +} + +// Generate Value field for defined Parameters. +// If the Parameter define Generate, then the Value is generated based +// on that template. The template is a pseudo-regexp formatted string. +// +// Example: +// +// s := generate.Template("[a-zA-Z0-9]{4}") +// // s: "Ga0b" +// +// s := generate.Template("[GET:http://example.com/new]") +// // s: +func (p *Template) ProcessParameters(customParams []Parameter) { + p.Parameters = append(p.Parameters, customParams...) + for i, _ := range p.Parameters { + p.Parameters[i].Seed = p.Seed + if err := p.Parameters[i].GenerateValue(); err != nil { + fmt.Printf("ERROR: Unable to process parameter %s: %v\n", p.Parameters[i].Name, err) + p.Parameters[i].Value = p.Parameters[i].Generate + } + } +} + +// Convert Parameter slice to more effective data structure +func (p *Template) CreateParameterMap() ParamMap { + ParamMap := make(ParamMap) + for _, p := range p.Parameters { + ParamMap[p.Name] = p + } + return ParamMap +} diff --git a/pkg/template/template.go b/pkg/template/template.go index 882839ff109d..8b761840fa69 100644 --- a/pkg/template/template.go +++ b/pkg/template/template.go @@ -2,148 +2,53 @@ package template import ( "encoding/json" - "fmt" "io/ioutil" "math/rand" "regexp" "strings" "time" - - "github.com/openshift/origin/pkg/template/generator" ) var valueExp = regexp.MustCompile(`(\$\{([a-zA-Z0-9\_]+)\})`) -type ParamMap map[string]Parameter - -// Generate the value for the Parameter if the default Value is not set and the -// Generator field is specified. Otherwise, just return the default Value -func (p *Parameter) GenerateValue() error { - if p.Value != "" || p.Generate == "" { - return nil - } - - if p.Seed == nil { - return fmt.Errorf("The random seed is not initialized.") - } - - g := new(generator.Generator) - g.SetSeed(p.Seed) - generatedValue, err := g.Generate(p.Generate).Value() - - if err != nil { - return err - } - p.Value = generatedValue - - return nil -} - -// The string representation of PValue -// -func (s PValue) String() string { - return string(s) -} - -// Replace references to parameters in PValue with their values. -// The format is specified in the `valueExp` constant ${PARAM_NAME}. -// -// If the referenced parameter is not defined, then the substitution is ignored. -func (s *PValue) Substitute(params ParamMap) { - newValue := *s - - for _, match := range valueExp.FindAllStringSubmatch(string(newValue), -1) { - // If the Parameter is not defined, then leave the value as it is - if params[match[2]].Value == "" { - continue +func ProcessContainerEnvs(source, target *DeploymentConfig, params ParamMap) { + *target = *source + for _, c := range target.DesiredState.PodTemplate.DesiredState.Manifest.Containers { + for v, e := range c.Env { + newValue := e.Value + for _, match := range valueExp.FindAllStringSubmatch(string(newValue), -1) { + if params[match[2]].Value == "" { + continue + } + newValue = strings.Replace(string(newValue), match[1], params[match[2]].Value, 1) + } + + c.Env[v].Value = newValue } - newValue = PValue(strings.Replace(string(newValue), match[1], params[match[2]].Value, 1)) } - - *s = newValue } -// Generate Value field for defined Parameters. -// If the Parameter define Generate, then the Value is generated based -// on that template. The template is a pseudo-regexp formatted string. -// -// Example: -// -// s := generate.Template("[a-zA-Z0-9]{4}") -// // s: "Ga0b" -// -// s := generate.Template("[GET:http://example.com/new]") -// // s: -func (p *Template) ProcessParameters(customParams []Parameter) { - p.Parameters = append(p.Parameters, customParams...) - for i, _ := range p.Parameters { - p.Parameters[i].Seed = p.Seed - if err := p.Parameters[i].GenerateValue(); err != nil { - fmt.Printf("ERROR: Unable to process parameter %s: %v\n", p.Parameters[i].Name, err) - p.Parameters[i].Value = p.Parameters[i].Generate - } - } +func TemplateToJSON(t Template) ([]byte, error) { + return json.Marshal(t) } -// Return the list of containers associated with the service -func (s *Service) Containers() []*Container { - result := make([]*Container, len((*s).DeploymentConfig.Deployment.PodTemplate.Containers)) +// Transform the source Template to target template, substituting the parameters +// referenced in podTemplate->containers using the parameter values. +// You might add more parameters using the third argument. +func TransformTemplate(source, target *Template, params []Parameter) { + *target = *source - for i, _ := range s.DeploymentConfig.Deployment.PodTemplate.Containers { - result[i] = &s.DeploymentConfig.Deployment.PodTemplate.Containers[i] - } - - return result -} + target.ProcessParameters(params) + paramMap := target.CreateParameterMap() -// A shorthand method to get list of *all* container defined in the Template -// template -func (p *Template) Containers() []*Container { - var result []*Container - for _, s := range p.Services { - result = append(result, s.Containers()...) + for i, d := range target.DeploymentConfigs { + newDeploymentConfig := new(DeploymentConfig) + ProcessContainerEnvs(&d, newDeploymentConfig, paramMap) + target.DeploymentConfigs[i] = *newDeploymentConfig } - return result -} - -// Convert Parameter slice to more effective data structure -func (p *Template) CreateParameterMap() ParamMap { - ParamMap := make(ParamMap) - for _, p := range p.Parameters { - ParamMap[p.Name] = p - } - return ParamMap -} - -// Process all Env variables in the Project template and replace parameters -// referenced in their values with the Parameter values. -// -// The replacement is done in Containers and ServiceLinks. -func (p *Template) Process() { - params := p.CreateParameterMap() - - for _, container := range p.Containers() { - (*container).Env.Process(params) - } - - for s, _ := range p.ServiceLinks { - p.ServiceLinks[s].Export.Process(params) - } -} - -// Substitute referenced parameters in Env values with parameter values. -func (e *Env) Process(params ParamMap) { - for i, _ := range *e { - (*e)[i].Value.Substitute(params) - } -} - -func (t *Template) Transform(customParams []Parameter) ([]byte, error) { - t.ProcessParameters(customParams) - t.Process() - return json.Marshal(*t) } +// Make a new Template from the JSON data and assign a random seed for it func NewTemplate(jsonData []byte) (*Template, error) { var template Template if err := json.Unmarshal(jsonData, &template); err != nil { @@ -153,6 +58,8 @@ func NewTemplate(jsonData []byte) (*Template, error) { return &template, nil } +// A helper function that reads the JSON file and return new Template with +// random seed assigned func NewTemplateFromFile(filename string) (*Template, error) { jsonData, err := ioutil.ReadFile(filename) if err != nil { diff --git a/pkg/template/template_test.go b/pkg/template/template_test.go index 5e6eade4824e..aa66763d4f87 100644 --- a/pkg/template/template_test.go +++ b/pkg/template/template_test.go @@ -3,7 +3,6 @@ package template import ( "fmt" "math/rand" - "strings" "testing" ) @@ -15,63 +14,29 @@ func TestNewTemplate(t *testing.T) { if err != nil { t.Errorf("Unable to process the JSON template: %v", err) } -} - -func TestProcessParameters(t *testing.T) { - sampleTemplate.ProcessParameters([]Parameter{}) - for _, p := range sampleTemplate.Parameters { - if p.Value == "" { - t.Errorf("Failed to process '%s' parameter", p.Name) - } - fmt.Printf("%s -> %s = %s\n", p.Name, p.Generate, p.Value) - } } -func TestProcessTemplate(t *testing.T) { - sampleTemplate.Process() +func ExampleTransformTemplate() { + var resultTemplate Template - for _, c := range sampleTemplate.Containers() { - for _, e := range c.Env { - if strings.Contains(string(e.Value), "${") { - if e.Name != "FOO" { - t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) - } - } - fmt.Printf("%s=%s\n", e.Name, e.Value) - } - } - - for _, s := range sampleTemplate.ServiceLinks { - for _, e := range s.Export { - if strings.Contains(string(e.Value), "${") { - if e.Name != "FOO" { - t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) - } - } - fmt.Printf("%s=%s\n", e.Name, e.Value) - } - } -} - -func ExampleTemplate_Transform() { - template, err := NewTemplateFromFile("example/simple.json") + template, err := NewTemplateFromFile("example/project.json") if err != nil { fmt.Printf("Unable to process example/simple.json template: %v", err) } - customParams := make([]Parameter, 3) - customParams[0] = Parameter{Name: "CUSTOM_PARAM1", Value: "1"} - customParams[1] = Parameter{Name: "CUSTOM_PARAM2", Value: "2"} - customParams[2] = Parameter{Name: "CUSTOM_PARAM3", Value: "3"} - // In this example, we want always produce the same result: - // template.Seed = rand.New(rand.NewSource(1337)) - result, _ := template.Transform(customParams) + // Define custom parameter for transformation: + customParams := make([]Parameter, 1) + customParams[0] = Parameter{Name: "CUSTOM_PARAM1", Value: "1"} + + TransformTemplate(template, &resultTemplate, customParams) + result, _ := TemplateToJSON(resultTemplate) fmt.Println(string(result)) // Output: - // {"id":"example2","buildConfig":null,"imageRepository":null,"parameters":[{"name":"DB_PASSWORD","description":"PostgreSQL admin user password","type":"string","generate":"[a-zA-Z0-9]{8}","value":"bQPdwNJi","Seed":{}},{"name":"DB_USER","description":"PostgreSQL username","type":"string","generate":"admin[a-zA-Z0-9]{4}","value":"adminJwWP","Seed":{}},{"name":"SAMPLE_VAR","description":"Sample","type":"string","generate":"","value":"foo","Seed":{}},{"name":"CUSTOM_PARAM1","description":"","type":"","generate":"","value":"1","Seed":{}},{"name":"CUSTOM_PARAM2","description":"","type":"","generate":"","value":"2","Seed":{}},{"name":"CUSTOM_PARAM3","description":"","type":"","generate":"","value":"3","Seed":{}}],"serviceLinks":[{"from":"database","to":"frontend","export":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"POSTGRES_DATABASE_NAME","value":"${DB_NAME}"}]}],"services":[{"name":"database","description":"Standalone PostgreSQL 9.2 database service","labels":{"name":"database-service"},"deploymentConfig":{"deployment":{"podTemplate":{"containers":[{"name":"postgresql-1","image":{"name":"postgres","tag":"9.2"},"env":[{"name":"POSTGRES_ADMIN_USERNAME","value":"adminJwWP"},{"name":"POSTGRES_ADMIN_PASSWORD","value":"bQPdwNJi"},{"name":"FOO","value":"1"}],"ports":[{"containerPort":5432,"hostPort":5432}]}],"replicas":0}}}}],"Seed":{}} + // {"id":"example1","buildConfigs":[{"name":"mfojtik/nginx-php-app","type":"docker","sourceUri":"https://raw.githubusercontent.com/mfojtik/phpapp/master/Dockerfile","imageRepository":"mfojtik/nginx-php-app"},{"name":"postgres","type":"docker","sourceUri":"https://raw.githubusercontent.com/docker-library/postgres/docker/9.2/Dockerfile","imageRepository":"postgres"}],"imageRepositories":[{"name":"mfojtik/nginx-php-app","url":"internal.registry.com:5000/mfojtik/phpapp"},{"name":"postgres","url":"registry.hub.docker.com/postgres"}],"parameters":[{"name":"DB_PASSWORD","description":"PostgreSQL admin user password","type":"string","generate":"[a-zA-Z0-9]{8}","value":"bQPdwNJi","Seed":{}},{"name":"DB_USER","description":"PostgreSQL username","type":"string","generate":"admin[a-zA-Z0-9]{4}","value":"adminJwWP","Seed":{}},{"name":"DB_NAME","description":"PostgreSQL database name","type":"string","generate":"","value":"mydb","Seed":{}},{"name":"REMOTE_KEY","description":"Example of remote key","type":"string","generate":"","value":"[GET:http://custom.url.int]","Seed":{}},{"name":"CUSTOM_PARAM1","description":"","type":"","generate":"","value":"1","Seed":{}}],"services":[{"kind":"Service","id":"database","apiVersion":"v1beta1","port":5432,"selector":{"name":"database"},"containerPort":0},{"kind":"Service","id":"frontend","apiVersion":"v1beta1","port":8080,"selector":{"name":"frontend"},"containerPort":0}],"deploymentConfigs":[{"kind":"DeploymentConfig","apiVersion":"v1beta1","labels":{"name":"database"},"desiredState":{"replicas":2,"replicaSelector":{"name":"database"},"podTemplate":{"desiredState":{"manifest":{"version":"v1beta1","id":"database","volumes":null,"containers":[{"name":"postgresql","image":"postgres","ports":[{"containerPort":5432}],"env":[{"name":"PGPASSWORD","value":"bQPdwNJi"},{"name":"PGUSER","value":"adminJwWP"},{"name":"PGDATABASE","value":"mydb"},{"name":"FOO","value":"${BAR}"}]}]},"restartpolicy":{}},"labels":{"name":"database"}}}},{"kind":"DeploymentConfig","apiVersion":"v1beta1","labels":{"name":"frontend"},"desiredState":{"replicas":2,"replicaSelector":{"name":"frontend"},"podTemplate":{"desiredState":{"manifest":{"version":"v1beta1","id":"frontend","volumes":null,"containers":[{"name":"frontend","image":"mfojtik/nginx-php-app","ports":[{"hostPort":8080,"containerPort":9292}],"env":[{"name":"PGPASSWORD","value":"bQPdwNJi"},{"name":"PGUSER","value":"adminJwWP"},{"name":"PGDATABASE","value":"mydb"}]}]},"restartpolicy":{}},"labels":{"name":"frontend"}}}}],"Seed":{}} + } diff --git a/pkg/template/types.go b/pkg/template/types.go index 97f057a1703c..ef7957490f8c 100644 --- a/pkg/template/types.go +++ b/pkg/template/types.go @@ -12,19 +12,29 @@ type ( ) type Template struct { - api.JSONBase `json:",inline" yaml:",inline"` - BuildConfig []BuildConfig `json:"buildConfig" yaml:"buildConfig"` - ImageRepository []ImageRepository `json:"imageRepository" yaml:"imageRepository"` - Parameters []Parameter `json:"parameters" yaml:"parameters"` - ServiceLinks []ServiceLink `json:"serviceLinks" yaml:"serviceLinks"` - Services []Service `json:"services" yaml:"services"` + api.JSONBase `json:",inline" yaml:",inline"` + BuildConfig []BuildConfig `json:"buildConfigs" yaml:"buildConfigs"` + ImageRepositories []ImageRepository `json:"imageRepositories" yaml:"imageRepositories"` + Parameters []Parameter `json:"parameters" yaml:"parameters"` + Services []api.Service `json:"services" yaml:"services"` + DeploymentConfigs []DeploymentConfig `json:"deploymentConfigs" yaml:"deploymentConfigs"` + Seed *rand.Rand +} - Seed *rand.Rand +type ImageRepositoryList struct { + api.JSONBase `json:",inline" yaml:",inline"` + Items []ImageRepository `json:"items,omitempty" yaml:"items,omitempty"` } type ImageRepository struct { - Name string `json:"name" yaml:"name"` - Url Uri `json:"url" yaml:"url"` + api.JSONBase `json:",inline" yaml:",inline"` + Name string `json:"name" yaml:"name"` + Url Uri `json:"url" yaml:"url"` +} + +type BuildConfigList struct { + api.JSONBase `json:",inline" yaml:",inline"` + Items []BuildConfig `json:"items,omitempty" yaml:"items,omitempty"` } type BuildConfig struct { @@ -34,6 +44,11 @@ type BuildConfig struct { ImageRepository string `json:"imageRepository" yaml:"imageRepository"` } +type ParameterList struct { + api.JSONBase `json:",inline" yaml:",inline"` + Items []Parameter `json:"items,omitempty" yaml:"items,omitempty"` +} + type Parameter struct { Name string `json:"name" yaml:"name"` Description string `json:"description" yaml:"description"` @@ -49,45 +64,13 @@ type Env []struct { Value PValue `json:"value" yaml:"value"` } -type ServiceLink struct { - From string `json:"from" yaml:"from"` - To string `json:"to" yaml:"to"` - Export Env `json:"export" yaml:"export"` +type DeploymentConfigList struct { + api.JSONBase `json:",inline" yaml:",inline"` + Items []DeploymentConfig `json:"items,omitempty" yaml:"items,omitempty"` } type DeploymentConfig struct { - Deployment Deployment `json:"deployment" yaml:"deployment"` -} - -type Deployment struct { - PodTemplate PodTemplate `json:"podTemplate" yaml:"podTemplate"` -} - -type PodTemplate struct { - Containers []Container `json:"containers" yaml:"containers"` - Replicas int `json:"replicas" yaml:"replicas"` -} - -type Image struct { - Name string `json:"name" yaml:"name"` - Tag string `json:"tag" yaml:"tag"` -} - -type ContainerPort struct { - ContainerPort int `json:"containerPort" yaml:"containerPort"` - HostPort int `json:"hostPort" yaml:"hostPort"` -} - -type Container struct { - Name string `json:"name" yaml:"name"` - Image Image `json:"image" yaml:"image"` - Env Env `json:"env" yaml:"env"` - Ports []ContainerPort `json:"ports" yaml:"ports"` -} - -type Service struct { - Name string `json:"name" yaml:"name"` - Description string `json:"description" yaml:"description"` - Labels map[string]PValue `json:"labels" yaml:"labels"` - DeploymentConfig DeploymentConfig `json:"deploymentConfig" yaml:"deploymentConfig"` + api.JSONBase `json:",inline" yaml:",inline"` + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` + DesiredState api.ReplicationControllerState `json:"desiredState" yaml:"desiredState"` }