From 03650a2187e90488c768de00281aafb001296156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= Date: Tue, 24 Nov 2020 18:13:01 +0100 Subject: [PATCH 1/2] Shell out package This shell out package should be threated as a brigde solutions that will help Knative move out of using to much shell scripts. It will enable us using new Golang orchestrated tests, and build processes, and have ability to shell out to scripts if needed. --- shell/assertions_test.go | 45 ++++++++++ shell/executor.go | 189 +++++++++++++++++++++++++++++++++++++++ shell/executor_test.go | 160 +++++++++++++++++++++++++++++++++ shell/fail-example.sh | 17 ++++ shell/prefixer.go | 68 ++++++++++++++ shell/prefixer_test.go | 70 +++++++++++++++ shell/project.go | 52 +++++++++++ shell/project_test.go | 35 ++++++++ shell/types.go | 82 +++++++++++++++++ 9 files changed, 718 insertions(+) create mode 100644 shell/assertions_test.go create mode 100644 shell/executor.go create mode 100644 shell/executor_test.go create mode 100755 shell/fail-example.sh create mode 100644 shell/prefixer.go create mode 100644 shell/prefixer_test.go create mode 100644 shell/project.go create mode 100644 shell/project_test.go create mode 100644 shell/types.go diff --git a/shell/assertions_test.go b/shell/assertions_test.go new file mode 100644 index 00000000..81491057 --- /dev/null +++ b/shell/assertions_test.go @@ -0,0 +1,45 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell_test + +import ( + "strings" + "testing" +) + +type assertions struct { + t *testing.T +} + +func (a assertions) NoError(err error) { + if err != nil { + a.t.Error(err) + } +} + +func (a assertions) Contains(haystack, needle string) { + if !strings.Contains(haystack, needle) { + a.t.Errorf("wanted to \ncontain: %#v\n in: %#v", + needle, haystack) + } +} + +func (a assertions) Equal(want, got string) { + if got != want { + a.t.Errorf("want: %#v\n got:%#v", want, got) + } +} diff --git a/shell/executor.go b/shell/executor.go new file mode 100644 index 00000000..e6308a07 --- /dev/null +++ b/shell/executor.go @@ -0,0 +1,189 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "time" +) + +const ( + defaultLabelOut = "[OUT]" + defaultLabelErr = "[ERR]" + executeMode = 0700 +) + +// ErrNoProjectLocation is returned if user didnt provided the project location. +var ErrNoProjectLocation = errors.New("project location isn't provided") + +// NewExecutor creates a new executor from given config. +func NewExecutor(config ExecutorConfig) Executor { + configureDefaultValues(&config) + return &streamingExecutor{ + ExecutorConfig: config, + } +} + +// RunScript executes a shell script with args. +func (s *streamingExecutor) RunScript(script Script, args ...string) error { + err := validate(s.ExecutorConfig) + if err != nil { + return err + } + cnt := script.scriptContent(s.ProjectLocation, args) + return withTempScript(cnt, func(bin string) error { + return stream(bin, s.ExecutorConfig, script.Label) + }) +} + +// RunFunction executes a shell function with args. +func (s *streamingExecutor) RunFunction(fn Function, args ...string) error { + err := validate(s.ExecutorConfig) + if err != nil { + return err + } + cnt := fn.scriptContent(s.ProjectLocation, args) + return withTempScript(cnt, func(bin string) error { + return stream(bin, s.ExecutorConfig, fn.Label) + }) +} + +type streamingExecutor struct { + ExecutorConfig +} + +func validate(config ExecutorConfig) error { + if config.ProjectLocation == nil { + return ErrNoProjectLocation + } + return nil +} + +func configureDefaultValues(config *ExecutorConfig) { + if config.Out == nil { + config.Out = os.Stdout + } + if config.Err == nil { + config.Err = os.Stderr + } + if config.LabelOut == "" { + config.LabelOut = defaultLabelOut + } + if config.LabelErr == "" { + config.LabelErr = defaultLabelErr + } + if config.Environ == nil { + config.Environ = os.Environ() + } + if !config.SkipDate && config.DateFormat == "" { + config.DateFormat = time.StampMilli + } + if config.PrefixFunc == nil { + config.PrefixFunc = defaultPrefixFunc + } +} + +func stream(bin string, cfg ExecutorConfig, label string) error { + c := exec.Command(bin) + c.Env = cfg.Environ + c.Stdout = NewPrefixer(cfg.Out, prefixFunc(StreamTypeOut, label, cfg)) + c.Stderr = NewPrefixer(cfg.Err, prefixFunc(StreamTypeErr, label, cfg)) + return c.Run() +} + +func prefixFunc(st StreamType, label string, cfg ExecutorConfig) func() string { + return func() string { + return cfg.PrefixFunc(st, label, cfg) + } +} + +func defaultPrefixFunc(st StreamType, label string, cfg ExecutorConfig) string { + sep := " " + var buf []string + if !cfg.SkipDate { + dt := time.Now().Format(cfg.DateFormat) + buf = append(buf, dt) + } + buf = append(buf, label) + switch st { + case StreamTypeOut: + buf = append(buf, cfg.LabelOut) + case StreamTypeErr: + buf = append(buf, cfg.LabelErr) + } + return strings.Join(buf, sep) + sep +} + +func withTempScript(contents string, fn func(bin string) error) error { + tmpfile, err := ioutil.TempFile("", "shellout-*.sh") + if err != nil { + return err + } + _, err = tmpfile.WriteString(contents) + if err != nil { + return err + } + err = tmpfile.Chmod(executeMode) + if err != nil { + return err + } + err = tmpfile.Close() + if err != nil { + return err + } + defer func() { + // clean up + _ = os.Remove(tmpfile.Name()) + }() + + return fn(tmpfile.Name()) +} + +func (fn *Function) scriptContent(location ProjectLocation, args []string) string { + return fmt.Sprintf(`#!/usr/bin/env bash + +set -Eeuo pipefail + +cd "%s" +source %s + +%s %s +`, location.RootPath(), fn.ScriptPath, fn.FunctionName, quoteArgs(args)) +} + +func (sc *Script) scriptContent(location ProjectLocation, args []string) string { + return fmt.Sprintf(`#!/usr/bin/env bash + +set -Eeuo pipefail + +cd "%s" +%s %s +`, location.RootPath(), sc.ScriptPath, quoteArgs(args)) +} + +func quoteArgs(args []string) string { + quoted := make([]string, len(args)) + for i, arg := range args { + quoted[i] = "\"" + strings.ReplaceAll(arg, "\"", "\\\"") + "\"" + } + return strings.Join(quoted, " ") +} diff --git a/shell/executor_test.go b/shell/executor_test.go new file mode 100644 index 00000000..a49efb28 --- /dev/null +++ b/shell/executor_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell_test + +import ( + "bytes" + "testing" + + "knative.dev/hack/shell" +) + +func TestNewExecutor(t *testing.T) { + assert := assertions{t: t} + tests := []testcase{ + helloWorldTestCase(t), + abortTestCase(t), + failExampleCase(t), + missingProjectLocationCase(), + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var outB, errB bytes.Buffer + tt.config.Out = &outB + tt.config.Err = &errB + executor := shell.NewExecutor(tt.config) + err := tt.op(executor) + if err != nil && !tt.wants.failed { + t.Errorf("%s: \n got: %#v\nfailed: %#v", tt.name, err, tt.failed) + } + + for _, wantOut := range tt.wants.outs { + assert.Contains(outB.String(), wantOut) + } + for _, wantErr := range tt.wants.errs { + assert.Contains(errB.String(), wantErr) + } + }) + } +} + +func TestExecutorDefaults(t *testing.T) { + assert := assertions{t: t} + loc, err := shell.NewProjectLocation("..") + assert.NoError(err) + exec := shell.NewExecutor(shell.ExecutorConfig{ + ProjectLocation: loc, + }) + err = exec.RunFunction(fn("true")) + assert.NoError(err) +} + +func helloWorldTestCase(t *testing.T) testcase { + return testcase{ + "echo Hello, World!", + config(t, func(cfg *shell.ExecutorConfig) { + cfg.SkipDate = true + }), + func(exec shell.Executor) error { + return exec.RunFunction(fn("echo"), "Hello, World!") + }, + wants{ + outs: []string{ + "echo [OUT] Hello, World!", + }, + }, + } +} + +func abortTestCase(t *testing.T) testcase { + return testcase{ + "abort function", + config(t, func(cfg *shell.ExecutorConfig) {}), + func(exec shell.Executor) error { + return exec.RunFunction(fn("abort"), "reason") + }, + wants{ + failed: true, + }, + } +} + +func failExampleCase(t *testing.T) testcase { + return testcase{ + "fail-example.sh", + config(t, func(cfg *shell.ExecutorConfig) {}), + func(exec shell.Executor) error { + return exec.RunScript(shell.Script{ + Label: "fail-example.sh", + ScriptPath: "shell/fail-example.sh", + }, "expected err") + }, + wants{ + failed: true, + errs: []string{ + "expected err", + }, + }, + } +} + +func missingProjectLocationCase() testcase { + return testcase{ + "missing project location", + shell.ExecutorConfig{}, + func(exec shell.Executor) error { + return exec.RunFunction(fn("id")) + }, + wants{ + failed: true, + }, + } +} + +type wants struct { + failed bool + outs []string + errs []string +} + +type testcase struct { + name string + config shell.ExecutorConfig + op func(exec shell.Executor) error + wants +} + +func config(t *testing.T, customize func(cfg *shell.ExecutorConfig)) shell.ExecutorConfig { + assert := assertions{t: t} + loc, err := shell.NewProjectLocation("..") + assert.NoError(err) + cfg := shell.ExecutorConfig{ + ProjectLocation: loc, + } + customize(&cfg) + return cfg +} + +func fn(name string) shell.Function { + return shell.Function{ + Script: shell.Script{ + Label: name, + ScriptPath: "library.sh", + }, + FunctionName: name, + } +} diff --git a/shell/fail-example.sh b/shell/fail-example.sh new file mode 100755 index 00000000..551ce662 --- /dev/null +++ b/shell/fail-example.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "$*" >&2 diff --git a/shell/prefixer.go b/shell/prefixer.go new file mode 100644 index 00000000..273ee1cc --- /dev/null +++ b/shell/prefixer.go @@ -0,0 +1,68 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "bytes" + "io" +) + +// NewPrefixer creates a new prefixer that forwards all calls to Write() to +// writer.Write() with all lines prefixed with the value of prefix. Having a +// function instead of a static prefix allows to print timestamps or other +// changing information. +func NewPrefixer(writer io.Writer, prefix func() string) io.Writer { + return &prefixer{prefix: prefix, writer: writer, trailingNewline: true} +} + +type prefixer struct { + prefix func() string + writer io.Writer + trailingNewline bool + buf bytes.Buffer // reuse buffer to save allocations +} + +func (pf *prefixer) Write(payload []byte) (int, error) { + pf.buf.Reset() // clear the buffer + + for _, b := range payload { + if pf.trailingNewline { + pf.buf.WriteString(pf.prefix()) + pf.trailingNewline = false + } + + pf.buf.WriteByte(b) + + if b == '\n' { + // do not print the prefix right after the newline character as this might + // be the very last character of the stream and we want to avoid a trailing prefix. + pf.trailingNewline = true + } + } + + n, err := pf.writer.Write(pf.buf.Bytes()) + if err != nil { + // never return more than original length to satisfy io.Writer interface + if n > len(payload) { + n = len(payload) + } + return n, err + } + + // return original length to satisfy io.Writer interface + return len(payload), nil +} diff --git a/shell/prefixer_test.go b/shell/prefixer_test.go new file mode 100644 index 00000000..8411e64c --- /dev/null +++ b/shell/prefixer_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell_test + +import ( + "bytes" + "strconv" + "testing" + + "knative.dev/hack/shell" +) + +func TestNewPrefixer(t *testing.T) { + assert := assertions{t: t} + var lineno int64 = 0 + tests := []struct { + name string + prefix func() string + want string + }{{ + "static", + func() string { + return "[prefix] " + }, + `[prefix] test string 1 +[prefix] test string 2 +`, + }, { + "empty", + func() string { + return "" + }, + `test string 1 +test string 2 +`, + }, { + "dynamic", + func() string { + lineno++ + return strconv.FormatInt(lineno, 10) + ") " + }, + `1) test string 1 +2) test string 2 +`, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + writer := &bytes.Buffer{} + wr := shell.NewPrefixer(writer, tt.prefix) + _, err := wr.Write([]byte("test string 1\ntest string 2\n")) + assert.NoError(err) + got := writer.String() + assert.Equal(tt.want, got) + }) + } +} diff --git a/shell/project.go b/shell/project.go new file mode 100644 index 00000000..2d7ba973 --- /dev/null +++ b/shell/project.go @@ -0,0 +1,52 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "errors" + "path" + "runtime" +) + +// ErrCantGetCaller is raised when we can't calculate a caller of NewProjectLocation. +var ErrCantGetCaller = errors.New("can't get caller") + +// NewProjectLocation creates a ProjectLocation that is used to calculate +// relative paths within the project. +func NewProjectLocation(pathToRoot string) (ProjectLocation, error) { + _, filename, _, ok := runtime.Caller(1) + if !ok { + return nil, ErrCantGetCaller + } + return &callerLocation{ + caller: filename, + pathToRoot: pathToRoot, + }, nil +} + +// RootPath return a path to root of the project. +func (c *callerLocation) RootPath() string { + return path.Join(path.Dir(c.caller), c.pathToRoot) +} + +// callerLocation holds a caller Go file, and a relative location to a project +// root directory. This information can be used to calculate relative paths and +// properly source shell scripts. +type callerLocation struct { + caller string + pathToRoot string +} diff --git a/shell/project_test.go b/shell/project_test.go new file mode 100644 index 00000000..719f511d --- /dev/null +++ b/shell/project_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell_test + +import ( + "io/ioutil" + "path" + "testing" + + "knative.dev/hack/shell" +) + +func TestNewProjectLocation(t *testing.T) { + assert := assertions{t: t} + loc, err := shell.NewProjectLocation("..") + assert.NoError(err) + goModPath := path.Join(loc.RootPath(), "go.mod") + bytes, err := ioutil.ReadFile(goModPath) + assert.NoError(err) + assert.Contains(string(bytes), "module knative.dev/hack") +} diff --git a/shell/types.go b/shell/types.go new file mode 100644 index 00000000..8e345152 --- /dev/null +++ b/shell/types.go @@ -0,0 +1,82 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import "io" + +// ProjectLocation represents a project location on a file system. +type ProjectLocation interface { + RootPath() string +} + +// Script represents a script to be executed. +type Script struct { + Label string + ScriptPath string +} + +// Function represents a function, whom will be sourced from Script file, +// and executed. +type Function struct { + Script + FunctionName string +} + +// ExecutorConfig holds a executor configuration options. +type ExecutorConfig struct { + ProjectLocation + Streams + Labels + Environ []string +} + +// StreamType represets either output or error stream. +type StreamType int + +const ( + // StreamTypeOut represents process output stream. + StreamTypeOut StreamType = iota + // StreamTypeErr represents process error stream. + StreamTypeErr +) + +// PrefixFunc is used to build a prefix that will be added to each line of the +// script/function output or error stream. +type PrefixFunc func(st StreamType, label string, config ExecutorConfig) string + +// Labels holds a labels to be used to prefix Out and Err streams of executed +// shells scripts/functions. +type Labels struct { + LabelOut string + LabelErr string + SkipDate bool + DateFormat string + PrefixFunc +} + +// Streams holds a streams of a shell scripts/functions. +type Streams struct { + Out io.Writer + Err io.Writer +} + +// Executor represents a executor that can execute shell scripts and call +// functions directly. +type Executor interface { + RunScript(script Script, args ...string) error + RunFunction(fn Function, args ...string) error +} From d2e43fecc21cabcc777352f66325033eef2202f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= Date: Tue, 24 Nov 2020 22:54:46 +0100 Subject: [PATCH 2/2] Make presubmit-tests.sh run unit tests --- test/presubmit-tests.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index 1d193e72..71ae1dfc 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -36,10 +36,6 @@ function post_build_tests() { return ${failed} } -function unit_tests() { - subheader "Skipping running unit tests since we don't have actual Go code in this repo." -} - # Run our custom unit tests after the standard unit tests. function post_unit_tests() {