From 2c95d090237c534b9ddec59b6488386445c41da6 Mon Sep 17 00:00:00 2001 From: apostasie Date: Tue, 17 Sep 2024 20:39:56 -0700 Subject: [PATCH 1/3] Adding NG test tooling Our testing tools have served us well, and there are plenty of good things with them that we absolutely need to keep. We have grown a large test suite though, and the sheer size of it is putting pressure on it and it is starting to show cracks. Besides bugs per-se (double execution) - that we hopefully fixed - our design is reaching its limits. We do have rather impactful issues with regard to test isolation and test concurrency, leading to situations where it is hard or outright impossible to figure out which test is causing a cascading failure, or why the adding of a new test file working individually breaks everything. Furthermore, as we are not prescriptive on certain things, we do see a lot of negative patterns emerging from test authors: - defer being used instead of t.Cleanup - not calling cleanup routines before running tests - outright forgetting to cleanup resources - Cwd for binary calls is by default the current working directory of the test process - this is causing a variety of issues, as it could very well be read-only (lima), and should by default be a temp directory - manipulating the environment directly - which has side-effects for other tests - tests becoming big blurbs of mixed together setup, cleanup, and actual test routines - making them hard to read and figuring out what is actually being tested - subtests repetitiveness w. shadowing testutil.T leading to confusing code - ... or not dereferencing the current test in a loop - in-test homegrown abstractions being inherently repetitive, with the same boilerplate over and over again - structuring tests and subtests being left as an exercise to the developer - leading to a wide variety of approaches and complex boilerplate - hard to debug: a lot of the assert methods do not provide any feedback whatsoever on what was the command that actually failed, or what was the output, or environment of it - icmd.Expected showing its limits, and making assumptions that there is only one error you can test, or that you can only test the output if exitCode == 0 - running commands with other binaries than the current target (eg: not calling base.Cmd) being left as an exercise to the developer, leading to all of the issues above (Chdir, Env manipulation, no debugging output, etc) - no-parallel being the default - unless specified otherwise - which should be the other way around - very rarely testing stderr in case of error - partly because we do not use typed errors, but also because it is cumbersome / not obvious / limited This new tooling offers a set of abstractions that should address all of these and encourage more expressive, better structured, better isolated, more debuggable tests. Signed-off-by: apostasie --- pkg/testutil/nerdtest/test.go | 253 +++++++++++++++++++++++++++++++ pkg/testutil/test/case.go | 175 +++++++++++++++++++++ pkg/testutil/test/command.go | 191 +++++++++++++++++++++++ pkg/testutil/test/data.go | 147 ++++++++++++++++++ pkg/testutil/test/expected.go | 90 +++++++++++ pkg/testutil/test/helpers.go | 71 +++++++++ pkg/testutil/test/requirement.go | 115 ++++++++++++++ pkg/testutil/test/test.go | 119 +++++++++++++++ 8 files changed, 1161 insertions(+) create mode 100644 pkg/testutil/nerdtest/test.go create mode 100644 pkg/testutil/test/case.go create mode 100644 pkg/testutil/test/command.go create mode 100644 pkg/testutil/test/data.go create mode 100644 pkg/testutil/test/expected.go create mode 100644 pkg/testutil/test/helpers.go create mode 100644 pkg/testutil/test/requirement.go create mode 100644 pkg/testutil/test/test.go diff --git a/pkg/testutil/nerdtest/test.go b/pkg/testutil/nerdtest/test.go new file mode 100644 index 00000000000..a2f7a5bd3c2 --- /dev/null +++ b/pkg/testutil/nerdtest/test.go @@ -0,0 +1,253 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package nerdtest + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/v2/pkg/buildkitutil" + "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" + "github.com/containerd/nerdctl/v2/pkg/inspecttypes/native" + "github.com/containerd/nerdctl/v2/pkg/rootlessutil" + "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func Setup() { + test.CustomCommand(nerdctlSetup) +} + +// Nerdctl specific config key and values +var NerdctlToml test.ConfigKey = "NerdctlToml" +var HostsDir test.ConfigKey = "HostsDir" +var DataRoot test.ConfigKey = "DataRoot" +var Namespace test.ConfigKey = "Namespace" + +var Mode test.ConfigKey = "Mode" +var ModePrivate test.ConfigValue = "Private" +var IPv6 test.ConfigKey = "IPv6Test" +var Only test.ConfigValue = "Only" + +var OnlyIPv6 = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + ret = testutil.GetEnableIPv6() + if !ret { + mess = "runner skips IPv6 compatible tests in the non-IPv6 environment" + } + data.WithConfig(IPv6, Only) + return ret, mess +}) + +var Private = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + data.WithConfig(Mode, ModePrivate) + return true, "" +}) + +var Docker = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + ret = testutil.GetTarget() == testutil.Docker + if ret { + mess = "current target is docker" + } else { + mess = "current target is not docker" + } + return ret, mess +}) + +var Rootless = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + ret = rootlessutil.IsRootless() + if ret { + mess = "environment is rootless" + } else { + mess = "environment is rootful" + } + return ret, mess +}) + +var Build = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + // FIXME: shouldn't we run buildkitd in a container? At least for testing, that would be so much easier than + // against the host install + ret = true + mess = "" + if testutil.GetTarget() == testutil.Nerdctl { + _, err := buildkitutil.GetBuildkitHost(testutil.Namespace) + if err != nil { + ret = false + mess = fmt.Sprintf("test requires buildkitd: %+v", err) + } + } + return ret, mess +}) + +type NerdCommand struct { + test.GenericCommand + // FIXME: annoying - forces custom Clone, etc + Target string +} + +// Run does override the generic command run, as we are testing both docker and nerdctl +func (nc *NerdCommand) Run(expect *test.Expected) { + // We are not in the business of testing docker error output, so, spay expect for errors testing, if any + if expect != nil && nc.Target != testutil.Nerdctl { + expect.Errors = nil + } + + nc.GenericCommand.Run(expect) +} + +// Clone is overridden as well, as we need to pass along the target +func (nc *NerdCommand) Clone() test.Command { + return &NerdCommand{ + GenericCommand: *((nc.GenericCommand.Clone()).(*test.GenericCommand)), + Target: nc.Target, + } +} + +// InspectContainer is a helper that can be used inside custom commands or Setup +func InspectContainer(helpers test.Helpers, name string) dockercompat.Container { + var dc []dockercompat.Container + cmd := helpers.Command("container", "inspect", name) + cmd.Run(&test.Expected{ + ExitCode: 0, + Output: func(stdout string, info string, t *testing.T) { + err := json.Unmarshal([]byte(stdout), &dc) + assert.NilError(t, err, "Unable to unmarshal output\n"+info) + assert.Equal(t, 1, len(dc), "Unexpectedly got multiple results\n"+info) + }, + }) + return dc[0] +} + +func InspectVolume(helpers test.Helpers, name string, args ...string) native.Volume { + var dc []native.Volume + cmdArgs := append([]string{"volume", "inspect"}, args...) + cmdArgs = append(cmdArgs, name) + + cmd := helpers.Command(cmdArgs...) + cmd.Run(&test.Expected{ + ExitCode: 0, + Output: func(stdout string, info string, t *testing.T) { + err := json.Unmarshal([]byte(stdout), &dc) + assert.NilError(t, err, "Unable to unmarshal output\n"+info) + assert.Equal(t, 1, len(dc), "Unexpectedly got multiple results\n"+info) + }, + }) + return dc[0] +} + +func InspectNetwork(helpers test.Helpers, name string, args ...string) dockercompat.Network { + var dc []dockercompat.Network + cmdArgs := append([]string{"network", "inspect"}, args...) + cmdArgs = append(cmdArgs, name) + + cmd := helpers.Command(cmdArgs...) + cmd.Run(&test.Expected{ + ExitCode: 0, + Output: func(stdout string, info string, t *testing.T) { + err := json.Unmarshal([]byte(stdout), &dc) + assert.NilError(t, err, "Unable to unmarshal output\n"+info) + assert.Equal(t, 1, len(dc), "Unexpectedly got multiple results\n"+info) + }, + }) + return dc[0] +} + +func nerdctlSetup(testCase *test.Case, t *testing.T) test.Command { + t.Helper() + + var testUtilBase *testutil.Base + dt := testCase.Data + var pvNamespace string + inherited := false + + if dt.ReadConfig(IPv6) != Only && testutil.GetEnableIPv6() { + t.Skip("runner skips non-IPv6 compatible tests in the IPv6 environment") + } + + if dt.ReadConfig(Mode) == ModePrivate { + // If private was inherited, we already got a configured namespace + if dt.ReadConfig(Namespace) != "" { + pvNamespace = string(dt.ReadConfig(Namespace)) + inherited = true + } else { + // Otherwise, we need to set everything up + pvNamespace = testCase.Data.Identifier() + dt.WithConfig(Namespace, test.ConfigValue(pvNamespace)) + testCase.Env["DOCKER_CONFIG"] = testCase.Data.TempDir() + testCase.Env["NERDCTL_TOML"] = filepath.Join(testCase.Data.TempDir(), "nerdctl.toml") + dt.WithConfig(HostsDir, test.ConfigValue(testCase.Data.TempDir())) + dt.WithConfig(DataRoot, test.ConfigValue(testCase.Data.TempDir())) + } + testUtilBase = testutil.NewBaseWithNamespace(t, pvNamespace) + if testUtilBase.Target == testutil.Docker { + // For docker, just disable parallel + testCase.NoParallel = true + } + } else if dt.ReadConfig(Namespace) != "" { + pvNamespace = string(dt.ReadConfig(Namespace)) + testUtilBase = testutil.NewBaseWithNamespace(t, pvNamespace) + } else { + testUtilBase = testutil.NewBase(t) + } + + // If we were passed custom content for NerdctlToml, save it + // Not happening if this is not nerdctl of course + if testUtilBase.Target == testutil.Nerdctl && dt.ReadConfig(NerdctlToml) != "" { + dest := filepath.Join(testCase.Data.TempDir(), "nerdctl.toml") + testCase.Env["NERDCTL_TOML"] = dest + err := os.WriteFile(dest, []byte(dt.ReadConfig(NerdctlToml)), 0400) + assert.NilError(t, err, "failed to write custom nerdctl toml file for test") + } + + // Build the base + baseCommand := &NerdCommand{} + baseCommand.WithBinary(testUtilBase.Binary) + baseCommand.WithArgs(testUtilBase.Args...) + baseCommand.WithEnv(testCase.Env) + baseCommand.WithT(t) + baseCommand.WithTempDir(testCase.Data.TempDir()) + baseCommand.Target = testUtilBase.Target + + if testUtilBase.Target == testutil.Nerdctl { + if dt.ReadConfig(HostsDir) != "" { + baseCommand.GenericCommand.WithArgs("--hosts-dir=" + string(dt.ReadConfig(HostsDir))) + } + + if dt.ReadConfig(DataRoot) != "" { + baseCommand.GenericCommand.WithArgs("--data-root=" + string(dt.ReadConfig(DataRoot))) + } + } + + // If we were in a custom namespace, not inherited - make sure we clean up the namespace + // FIXME: this is broken, and custom namespaces are not cleaned properly + if testUtilBase.Target == testutil.Nerdctl && pvNamespace != "" && !inherited { + cleanup := func() { + cl := baseCommand.Clone() + cl.WithArgs("namespace", "remove", pvNamespace) + cl.Run(nil) + } + cleanup() + t.Cleanup(cleanup) + } + + // Attach the base command + return baseCommand +} diff --git a/pkg/testutil/test/case.go b/pkg/testutil/test/case.go new file mode 100644 index 00000000000..eed35a929f6 --- /dev/null +++ b/pkg/testutil/test/case.go @@ -0,0 +1,175 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +// Group informally describes a slice of tests +type Group []*Case + +func (tg *Group) Run(t *testing.T) { + t.Helper() + // If the group contains only one test, no need to create a subtest + sub := len(*tg) > 1 + if sub { + t.Parallel() + } + // Run each subtest + for _, tc := range *tg { + tc.subIt = sub + tc.Run(t) + } +} + +// Case describes an entire test-case, including data, setup and cleanup routines, command and expectations +type Case struct { + // Description contains a human-readable short desc, used as a seed for the identifier and as a title for the test + Description string + // NoParallel disables parallel execution if set to true + NoParallel bool + // Env contains a map of environment variables to use for commands run in Setup, Command and Cleanup + // Note that the environment is inherited by subtests + Env map[string]string + // Data contains test specific data, accessible to all operations, also inherited by subtests + Data Data + + // Setup + Setup Butler + // Expected + Expected Manager + // Command + Command Executor + // Cleanup + Cleanup Butler + // Requirement + Require Requirement + + // SubTests + SubTests []*Case + + // Private + helpers Helpers + t *testing.T + parent *Case + baseCommand Command + + subIt bool +} + +// Run prepares and executes the test, and any possible subtests +func (test *Case) Run(t *testing.T) { + t.Helper() + // Run the test + testRun := func(tt *testing.T) { + tt.Helper() + test.seal(tt) + + if registeredInit == nil { + bc := &GenericCommand{} + bc.WithEnv(test.Env) + bc.WithT(tt) + bc.WithTempDir(test.Data.TempDir()) + test.baseCommand = bc + } else { + test.baseCommand = registeredInit(test, test.t) + } + + test.exec(tt) + } + + if test.subIt { + t.Run(test.Description, testRun) + } else { + testRun(t) + } +} + +// seal is a private method to prepare the test +func (test *Case) seal(t *testing.T) { + t.Helper() + assert.Assert(t, test.t == nil, "You cannot run a test multiple times") + assert.Assert(t, test.Description != "", "A test description cannot be empty") + assert.Assert(t, test.Command == nil || test.Expected != nil, + "Expectations for a test command cannot be nil. You may want to use Setup instead.") + + // Ensure we have env + if test.Env == nil { + test.Env = map[string]string{} + } + + // If we have a parent, get parent env and data + var parentData Data + if test.parent != nil { + parentData = test.parent.Data + for k, v := range test.parent.Env { + if _, ok := test.Env[k]; !ok { + test.Env[k] = v + } + } + } + + // Attach testing.T + test.t = t + // Inherit and attach Data + test.Data = configureData(t, test.Data, parentData) + + // Check the requirements + if test.Require != nil { + test.Require(test.Data, t) + } +} + +// exec is a private method that will take care of the test setup, command and cleanup execution +func (test *Case) exec(t *testing.T) { + t.Helper() + test.helpers = &helpers{ + test.baseCommand, + } + + // Set parallel unless asked not to + if !test.NoParallel { + t.Parallel() + } + + // Register cleanup if there is any, and run it to collect any leftovers from previous runs + if test.Cleanup != nil { + test.Cleanup(test.Data, test.helpers) + t.Cleanup(func() { + test.Cleanup(test.Data, test.helpers) + }) + } + + // Run setup + if test.Setup != nil { + test.Setup(test.Data, test.helpers) + } + + // Run the command if any, with expectations + if test.Command != nil { + test.Command(test.Data, test.helpers).Run(test.Expected(test.Data, test.helpers)) + } + + for _, subTest := range test.SubTests { + subTest.parent = test + subTest.subIt = true + subTest.Run(t) + } +} diff --git a/pkg/testutil/test/command.go b/pkg/testutil/test/command.go new file mode 100644 index 00000000000..6fbb1779d52 --- /dev/null +++ b/pkg/testutil/test/command.go @@ -0,0 +1,191 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "fmt" + "io" + "os" + "strings" + "testing" + "time" + + "gotest.tools/v3/assert" + "gotest.tools/v3/icmd" +) + +// GenericCommand is a concrete Command implementation +type GenericCommand struct { + WorkingDir string + Env map[string]string + + t *testing.T + tempDir string + helperBinary string + helperArgs []string + mainBinary string + mainArgs []string + result *icmd.Result + stdin io.Reader + async bool + timeout time.Duration +} + +func (gc *GenericCommand) WithBinary(binary string) Command { + gc.mainBinary = binary + return gc +} + +func (gc *GenericCommand) WithArgs(args ...string) Command { + gc.mainArgs = append(gc.mainArgs, args...) + return gc +} + +// WithEnv will overload the command env with values from the passed map +func (gc *GenericCommand) WithEnv(env map[string]string) Command { + if gc.Env == nil { + gc.Env = map[string]string{} + } + for k, v := range env { + gc.Env[k] = v + } + return gc +} + +func (gc *GenericCommand) WithWrapper(binary string, args ...string) Command { + gc.helperBinary = binary + gc.helperArgs = args + return gc +} + +// WithStdin sets the standard input of Cmd to the specified reader +func (gc *GenericCommand) WithStdin(r io.Reader) Command { + gc.stdin = r + return gc +} + +func (gc *GenericCommand) Background(timeout time.Duration) Command { + // Run it + gc.async = true + i := gc.boot() + gc.result = icmd.StartCmd(i) + gc.timeout = timeout + return gc +} + +// TODO: it should be possible to: +// - timeout execution +func (gc *GenericCommand) Run(expect *Expected) { + var result *icmd.Result + var env []string + if gc.async { + result = icmd.WaitOnCmd(gc.timeout, gc.result) + env = gc.result.Cmd.Env + } else { + icmdCmd := gc.boot() + env = icmdCmd.Env + // Run it + result = icmd.RunCmd(icmdCmd) + } + + // Check our expectations, if any + if expect != nil { + // Build the debug string - additionally attach the env (which icmd does not do) + debug := result.String() + "Env:\n" + strings.Join(env, "\n") + // ExitCode goes first + if expect.ExitCode == -1 { + assert.Assert(gc.t, result.ExitCode != 0, + "Expected exit code to be different than 0"+debug) + } else { + assert.Assert(gc.t, expect.ExitCode == result.ExitCode, + fmt.Sprintf("Expected exit code: %d", expect.ExitCode)+debug) + } + // Range through the expected errors and confirm they are seen on stderr + for _, expectErr := range expect.Errors { + assert.Assert(gc.t, strings.Contains(result.Stderr(), expectErr.Error()), + fmt.Sprintf("Expected error: %q to be found in stderr", expectErr.Error())+debug) + } + // Finally, check the output if we are asked to + if expect.Output != nil { + expect.Output(result.Stdout(), debug, gc.t) + } + } +} + +func (gc *GenericCommand) boot() icmd.Cmd { + // This is a helper function, not to appear in the debugging output + gc.t.Helper() + + binary := gc.mainBinary + args := gc.mainArgs + if gc.helperBinary != "" { + args = append([]string{binary}, args...) + args = append(gc.helperArgs, args...) + binary = gc.helperBinary + } + + // Create the command and set the env + // TODO: do we really need icmd? + icmdCmd := icmd.Command(binary, args...) + icmdCmd.Env = []string{} + for _, v := range os.Environ() { + // Ignore LS_COLORS from the env, just too much noise + if !strings.HasPrefix(v, "LS_COLORS") { + icmdCmd.Env = append(icmdCmd.Env, v) + } + } + + // Ensure the subprocess gets executed in a temporary directory unless explicitly instructed otherwise + icmdCmd.Dir = gc.WorkingDir + if icmdCmd.Dir == "" { + icmdCmd.Dir = gc.tempDir + } + + // Attach any extra env we have + for k, v := range gc.Env { + icmdCmd.Env = append(icmdCmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + + return icmdCmd +} + +func (gc *GenericCommand) Clone() Command { + // Copy the command and return a new one - with WorkingDir, binary, args, etc + cc := *gc + // Clone Env + cc.Env = make(map[string]string, len(gc.Env)) + for k, v := range gc.Env { + cc.Env[k] = v + } + return &cc +} + +func (gc *GenericCommand) Clear() Command { + gc.mainBinary = "" + gc.helperBinary = "" + gc.mainArgs = []string{} + gc.helperArgs = []string{} + return gc +} + +func (gc *GenericCommand) WithT(t *testing.T) { + gc.t = t +} + +func (gc *GenericCommand) WithTempDir(tempDir string) { + gc.tempDir = tempDir +} diff --git a/pkg/testutil/test/data.go b/pkg/testutil/test/data.go new file mode 100644 index 00000000000..99f2aa041ad --- /dev/null +++ b/pkg/testutil/test/data.go @@ -0,0 +1,147 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "crypto/sha256" + "fmt" + "strings" + "testing" +) + +// Contains the implementation of the Data interface + +type data struct { + config map[ConfigKey]ConfigValue + + system map[SystemKey]SystemValue + + labels map[string]string + testID string + tempDir string +} + +func (dt *data) WithConfig(key ConfigKey, value ConfigValue) Data { + if dt.config == nil { + dt.config = make(map[ConfigKey]ConfigValue) + } + dt.config[key] = value + return dt +} + +func (dt *data) ReadConfig(key ConfigKey) ConfigValue { + if dt.config == nil { + dt.config = make(map[ConfigKey]ConfigValue) + } + if val, ok := dt.config[key]; ok { + return val + } + return "" +} + +func (dt *data) Get(key string) string { + if dt.labels == nil { + dt.labels = map[string]string{} + } + return dt.labels[key] +} + +func (dt *data) Set(key string, value string) Data { + if dt.labels == nil { + dt.labels = map[string]string{} + } + dt.labels[key] = value + return dt +} + +func (dt *data) Identifier() string { + return dt.testID +} + +func (dt *data) TempDir() string { + return dt.tempDir +} + +func (dt *data) adopt(parent Data) { + for k, v := range parent.getLabels() { + // Only copy keys that are not set already + if _, ok := dt.labels[k]; !ok { + dt.Set(k, v) + } + } + for k, v := range parent.getConfig() { + // Only copy keys that are not set already + if _, ok := dt.config[k]; !ok { + dt.WithConfig(k, v) + } + } +} + +func (dt *data) Sink(key SystemKey, value SystemValue) { + if _, ok := dt.system[key]; !ok { + dt.system[key] = value + } else { + // XXX should we really panic? + panic(fmt.Sprintf("Unable to set system key %s multiple times", key)) + } +} + +func (dt *data) Surface(key SystemKey) SystemValue { + if v, ok := dt.system[key]; ok { + return v + } + // XXX should we really panic? + panic(fmt.Sprintf("Unable to retrieve system key %s", key)) +} + +func (dt *data) getLabels() map[string]string { + return dt.labels +} + +func (dt *data) getConfig() map[ConfigKey]ConfigValue { + return dt.config +} + +func defaultIdentifierHashing(name string) string { + s := strings.ReplaceAll(name, " ", "_") + s = strings.ReplaceAll(s, "/", "_") + s = strings.ReplaceAll(s, "-", "_") + s = strings.ReplaceAll(s, ",", "_") + s = strings.ToLower(s) + if len(s) > 76 { + s = fmt.Sprintf("%x", sha256.Sum256([]byte(s))) + } + + return s +} + +// TODO: allow to pass custom hashing methods? +func configureData(t *testing.T, seedData Data, parent Data) Data { + if seedData == nil { + seedData = &data{} + } + dat := &data{ + config: seedData.getConfig(), + labels: seedData.getLabels(), + tempDir: t.TempDir(), + testID: defaultIdentifierHashing(t.Name()), + } + if parent != nil { + dat.adopt(parent) + } + return dat +} diff --git a/pkg/testutil/test/expected.go b/pkg/testutil/test/expected.go new file mode 100644 index 00000000000..81d617acfdf --- /dev/null +++ b/pkg/testutil/test/expected.go @@ -0,0 +1,90 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "fmt" + "strings" + "testing" + + "gotest.tools/v3/assert" +) + +func RunCommand(args ...string) Executor { + return func(data Data, helpers Helpers) Command { + return helpers.Command(args...) + } +} + +// WithData returns a data object with a certain key value set +func WithData(key string, value string) Data { + dat := &data{} + dat.Set(key, value) + return dat +} + +// WithConfig returns a data object with a certain config property set +func WithConfig(key ConfigKey, value ConfigValue) Data { + dat := &data{} + dat.WithConfig(key, value) + return dat +} + +// Expects is provided as a simple helper covering "expectations" for simple use-cases where access to the test data is not necessary +func Expects(exitCode int, errors []error, output Comparator) Manager { + return func(_ Data, _ Helpers) *Expected { + return &Expected{ + ExitCode: exitCode, + Errors: errors, + Output: output, + } + } +} + +// All can be used as a parameter for expected.Output and allow passing a collection of conditions to match +func All(comparators ...Comparator) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + for _, comparator := range comparators { + comparator(stdout, info, t) + } + } +} + +// Contains can be used as a parameter for expected.Output and ensures a comparison string is found contained in the output +func Contains(compare string) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Assert(t, strings.Contains(stdout, compare), fmt.Sprintf("Expected output to contain: %q", compare)+info) + } +} + +// DoesNotContain is to be used for expected.Output to ensure a comparison string is NOT found in the output +func DoesNotContain(compare string) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Assert(t, !strings.Contains(stdout, compare), fmt.Sprintf("Expected output to not contain: %q", compare)+info) + } +} + +// Equals is to be used for expected.Output to ensure it is exactly the output +func Equals(compare string) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Equal(t, compare, stdout, info) + } +} diff --git a/pkg/testutil/test/helpers.go b/pkg/testutil/test/helpers.go new file mode 100644 index 00000000000..64a734dd86c --- /dev/null +++ b/pkg/testutil/test/helpers.go @@ -0,0 +1,71 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import "testing" + +type Helpers interface { + Ensure(args ...string) + Anyhow(args ...string) + Fail(args ...string) + Capture(args ...string) string + + Command(args ...string) Command + CustomCommand(binary string, args ...string) Command +} + +type helpers struct { + cmd Command +} + +func (hel *helpers) Ensure(args ...string) { + hel.Command(args...).Run(&Expected{}) +} + +func (hel *helpers) Anyhow(args ...string) { + hel.Command(args...).Run(nil) +} + +func (hel *helpers) Fail(args ...string) { + hel.Command(args...).Run(&Expected{ + ExitCode: 1, + }) +} + +func (hel *helpers) Capture(args ...string) string { + var ret string + hel.Command(args...).Run(&Expected{ + Output: func(stdout string, info string, t *testing.T) { + ret = stdout + }, + }) + return ret +} + +func (hel *helpers) Command(args ...string) Command { + cc := hel.cmd.Clone() + cc.WithArgs(args...) + return cc +} + +func (hel *helpers) CustomCommand(binary string, args ...string) Command { + cc := hel.cmd.Clone() + cc.Clear() + cc.WithBinary(binary) + cc.WithArgs(args...) + return cc +} diff --git a/pkg/testutil/test/requirement.go b/pkg/testutil/test/requirement.go new file mode 100644 index 00000000000..1acad13af28 --- /dev/null +++ b/pkg/testutil/test/requirement.go @@ -0,0 +1,115 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "fmt" + "os/exec" + "runtime" + "testing" +) + +func MakeRequirement(fn func(data Data) (bool, string)) Requirement { + return func(data Data, t *testing.T) (bool, string) { + ret, mess := fn(data) + + if t != nil && !ret { + t.Helper() + t.Skipf("Test skipped as %s", mess) + } + + return ret, mess + } +} + +func Binary(name string) Requirement { + return MakeRequirement(func(data Data) (ret bool, mess string) { + mess = fmt.Sprintf("executable %q has been found in PATH", name) + ret = true + if _, err := exec.LookPath(name); err != nil { + ret = false + mess = fmt.Sprintf("executable %q doesn't exist in PATH", name) + } + + return ret, mess + }) +} + +func OS(os string) Requirement { + return MakeRequirement(func(data Data) (ret bool, mess string) { + mess = fmt.Sprintf("current operating is %q", runtime.GOOS) + ret = true + if runtime.GOOS != os { + ret = false + } + + return ret, mess + }) +} + +var Windows = MakeRequirement(func(data Data) (ret bool, mess string) { + ret = runtime.GOOS == "windows" + if ret { + mess = "operating system is Windows" + } else { + mess = "operating system is not Windows" + } + return ret, mess +}) + +var Linux = MakeRequirement(func(data Data) (ret bool, mess string) { + ret = runtime.GOOS == "linux" + if ret { + mess = "operating system is Linux" + } else { + mess = "operating system is not Linux" + } + return ret, mess +}) + +var Darwin = MakeRequirement(func(data Data) (ret bool, mess string) { + ret = runtime.GOOS == "darwin" + if ret { + mess = "operating system is Darwin" + } else { + mess = "operating system is not Darwin" + } + return ret, mess +}) + +func Not(requirement Requirement) Requirement { + return MakeRequirement(func(data Data) (ret bool, mess string) { + b, mess := requirement(data, nil) + return !b, mess + }) +} + +func Require(thing ...Requirement) Requirement { + return func(data Data, t *testing.T) (ret bool, mess string) { + for _, th := range thing { + b, m := th(data, nil) + if !b { + if t != nil { + t.Helper() + t.Skipf("Test skipped as %s", m) + } + return false, "" + } + } + return true, "" + } +} diff --git a/pkg/testutil/test/test.go b/pkg/testutil/test/test.go new file mode 100644 index 00000000000..2e6743be8e6 --- /dev/null +++ b/pkg/testutil/test/test.go @@ -0,0 +1,119 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "io" + "testing" + "time" +) + +// A Requirement is a function that can evaluate random requirement and possibly skip a test +// See test.MakeRequirement to make your own +type Requirement func(data Data, t *testing.T) (bool, string) + +// A Butler is the function signature meant to be attached to a Setup or Cleanup routine for a test.Case +type Butler func(data Data, helpers Helpers) + +// An Executor is the function signature meant to be attached to a test.Case Command +type Executor func(data Data, helpers Helpers) Command + +// A Manager is the function signature to be run to produce expectations to be fed to a command +type Manager func(data Data, helpers Helpers) *Expected + +// The Command interface represents a low-level command to execute, typically to be compared with an Expected +// A Command can be used as a Case Command obviously, but also as part of a Setup or Cleanup routine, +// and as the basis of any type of helper. +// A Command can be cloned, in which case, the subcommand inherits a copy of all of its Env and parameters. +// Typically, a Case has a base-command, from which all commands involved in the test are derived. +type Command interface { + // WithBinary specifies what binary to execute + WithBinary(binary string) Command + // WithArgs specifies the args to pass to the binary. Note that WithArgs is additive. + WithArgs(args ...string) Command + // WithEnv adds the passed map to the environment of the command to be executed + WithEnv(env map[string]string) Command + // WithWrapper allows wrapping a command with another command (for example: `time`, `unbuffer`) + WithWrapper(binary string, args ...string) Command + // WithStdin allows passing a reader to be used for stdin for the command + WithStdin(r io.Reader) Command + // Run does execute the command, and compare the output with the provided expectation. + // Passing nil for `Expected` will just run the command regardless of outcome. + // An empty `&Expected{}` is (of course) equivalent to &Expected{Exit: 0}, meaning the command is verified to be + // successful + Run(expect *Expected) + // Clone returns a copy of the command + Clone() Command + // Clear will clear binary and arguments, but retain the env, or any other custom properties + Clear() Command + // Allow starting a command in the background + Background(timeout time.Duration) Command +} + +type Comparator func(stdout string, info string, t *testing.T) + +// Expected expresses the expected output of a command +type Expected struct { + // ExitCode to expect + ExitCode int + // Errors contains any error that (once serialized) should be seen in stderr + Errors []error + // Output function to match against stdout + Output Comparator +} + +type ConfigKey string +type ConfigValue string + +type SystemKey string +type SystemValue string + +// Data is meant to hold information about a test: +// - first, any random key value data that the test implementer wants to carry / modify - this is test data +// - second, configuration specific to the binary being tested - typically defined by the specialized command being tested +// - third, immutable "system" info (unique identifier, tempdir, or other SystemKey/Value pairs) +type Data interface { + // Get returns the value of a certain key for custom data + Get(key string) string + // Set will save `value` for `key` + Set(key string, value string) Data + + // Identifier returns the test identifier that can be used to name resources + Identifier() string + // TempDir returns the test temporary directory + TempDir() string + // Sink allows to define ONCE a certain system property + Sink(key SystemKey, value SystemValue) + // Surface allows retrieving a certain system property + Surface(key SystemKey) SystemValue + + // WithConfig allows setting a declared ConfigKey to a ConfigValue + WithConfig(key ConfigKey, value ConfigValue) Data + ReadConfig(key ConfigKey) ConfigValue + + // Private methods + getLabels() map[string]string + getConfig() map[ConfigKey]ConfigValue +} + +var ( + registeredInit func(test *Case, t *testing.T) Command +) + +func CustomCommand(custom func(test *Case, t *testing.T) Command) { + registeredInit = custom +} From cc56a0931f86069e9ffbe3822730f58da1f0d758 Mon Sep 17 00:00:00 2001 From: apostasie Date: Tue, 17 Sep 2024 20:41:22 -0700 Subject: [PATCH 2/3] Move root and volume tests to new test tooling Signed-off-by: apostasie --- cmd/nerdctl/main_linux_test.go | 38 +- cmd/nerdctl/main_test.go | 149 ++-- cmd/nerdctl/main_test_test.go | 147 ++++ cmd/nerdctl/volume/volume_create_test.go | 162 ++-- cmd/nerdctl/volume/volume_inspect_test.go | 389 ++++----- cmd/nerdctl/volume/volume_list_test.go | 742 ++++++++---------- cmd/nerdctl/volume/volume_namespace_test.go | 178 ++--- cmd/nerdctl/volume/volume_prune_linux_test.go | 172 ++-- .../volume/volume_remove_linux_test.go | 319 +++----- 9 files changed, 1063 insertions(+), 1233 deletions(-) create mode 100644 cmd/nerdctl/main_test_test.go diff --git a/cmd/nerdctl/main_linux_test.go b/cmd/nerdctl/main_linux_test.go index a0e82861a84..274604e27b9 100644 --- a/cmd/nerdctl/main_linux_test.go +++ b/cmd/nerdctl/main_linux_test.go @@ -20,17 +20,39 @@ import ( "testing" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) // TestIssue108 tests https://github.com/containerd/nerdctl/issues/108 // ("`nerdctl run --net=host -it` fails while `nerdctl run -it --net=host` works") func TestIssue108(t *testing.T) { - base := testutil.NewBase(t) - // unbuffer(1) emulates tty, which is required by `nerdctl run -t`. - // unbuffer(1) can be installed with `apt-get install expect`. - unbuffer := []string{"unbuffer"} - base.CmdWithHelper(unbuffer, "run", "-it", "--rm", "--net=host", testutil.AlpineImage, - "echo", "this was always working").AssertOK() - base.CmdWithHelper(unbuffer, "run", "--rm", "--net=host", "-it", testutil.AlpineImage, - "echo", "this was not working due to issue #108").AssertOK() + nerdtest.Setup() + + testGroup := &test.Group{ + { + Description: "-it --net=host", + Require: test.Binary("unbuffer"), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers. + Command("run", "-it", "--rm", "--net=host", testutil.AlpineImage, "echo", "this was always working"). + WithWrapper("unbuffer") + }, + // Note: unbuffer will merge stdout and stderr, preventing exact match here + Expected: test.Expects(0, nil, test.Contains("this was always working")), + }, + { + Description: "--net=host -it", + Require: test.Binary("unbuffer"), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers. + Command("run", "--rm", "--net=host", "-it", testutil.AlpineImage, "echo", "this was not working due to issue #108"). + WithWrapper("unbuffer") + }, + // Note: unbuffer will merge stdout and stderr, preventing exact match here + Expected: test.Expects(0, nil, test.Contains("this was not working due to issue #108")), + }, + } + + testGroup.Run(t) } diff --git a/cmd/nerdctl/main_test.go b/cmd/nerdctl/main_test.go index a450c963bf5..c1e3caf94fd 100644 --- a/cmd/nerdctl/main_test.go +++ b/cmd/nerdctl/main_test.go @@ -17,15 +17,14 @@ package main import ( - "os" - "path/filepath" + "errors" "testing" - "gotest.tools/v3/assert" - "github.com/containerd/containerd/v2/defaults" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestMain(m *testing.M) { @@ -34,56 +33,100 @@ func TestMain(m *testing.M) { // TestUnknownCommand tests https://github.com/containerd/nerdctl/issues/487 func TestUnknownCommand(t *testing.T) { - t.Parallel() - base := testutil.NewBase(t) - base.Cmd("non-existent-command").AssertFail() - base.Cmd("non-existent-command", "info").AssertFail() - base.Cmd("system", "non-existent-command").AssertFail() - base.Cmd("system", "non-existent-command", "info").AssertFail() - base.Cmd("system").AssertOK() // show help without error - base.Cmd("system", "info").AssertOutContains("Kernel Version:") - base.Cmd("info").AssertOutContains("Kernel Version:") + nerdtest.Setup() + + var unknownSubCommand = errors.New("unknown subcommand") + + testGroup := &test.Group{ + { + Description: "non-existent-command", + Command: test.RunCommand("non-existent-command"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "non-existent-command info", + Command: test.RunCommand("non-existent-command", "info"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "system non-existent-command", + Command: test.RunCommand("system", "non-existent-command"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "system non-existent-command info", + Command: test.RunCommand("system", "non-existent-command", "info"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "system", + Command: test.RunCommand("system"), + Expected: test.Expects(0, nil, nil), + }, + { + Description: "system info", + Command: test.RunCommand("system", "info"), + Expected: test.Expects(0, nil, test.Contains("Kernel Version:")), + }, + { + Description: "info", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, test.Contains("Kernel Version:")), + }, + } + + testGroup.Run(t) } -// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default]. +// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default] and broken config rejection func TestNerdctlConfig(t *testing.T) { - testutil.DockerIncompatible(t) - t.Parallel() - tomlPath := filepath.Join(t.TempDir(), "nerdctl.toml") - err := os.WriteFile(tomlPath, []byte(` -snapshotter = "dummy-snapshotter-via-toml" -`), 0400) - assert.NilError(t, err) - base := testutil.NewBase(t) - - // [Default] - base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly(defaults.DefaultSnapshotter + "\n") - - // [TOML, Default] - base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath) - base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-toml\n") - - // [CLI, TOML, Default] - base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n") - - // [Env, TOML, Default] - base.Env = append(base.Env, "CONTAINERD_SNAPSHOTTER=dummy-snapshotter-via-env") - base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-env\n") - - // [CLI, Env, TOML, Default] - base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n") -} - -func TestNerdctlConfigBad(t *testing.T) { - testutil.DockerIncompatible(t) - t.Parallel() - tomlPath := filepath.Join(t.TempDir(), "config.toml") - err := os.WriteFile(tomlPath, []byte(` -# containerd config, not nerdctl config -version = 2 -`), 0400) - assert.NilError(t, err) - base := testutil.NewBase(t) - base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath) - base.Cmd("info").AssertFail() + nerdtest.Setup() + + tc := &test.Case{ + Description: "Nerdctl configuration", + // Docker does not support nerdctl.toml obviously + Require: test.Not(nerdtest.Docker), + SubTests: []*test.Case{ + { + Description: "Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}"), + Expected: test.Expects(0, nil, test.Equals(defaults.DefaultSnapshotter+"\n")), + }, + { + Description: "TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}"), + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-toml\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Cli > TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"), + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Env > TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}"), + Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"}, + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-env\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Cli > Env > TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"), + Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"}, + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Broken config", + Command: test.RunCommand("info"), + Expected: test.Expects(1, []error{errors.New("failed to load nerdctl config")}, nil), + Data: test.WithConfig(nerdtest.NerdctlToml, `# containerd config, not nerdctl config +version = 2`), + }, + }, + } + + tc.Run(t) } diff --git a/cmd/nerdctl/main_test_test.go b/cmd/nerdctl/main_test_test.go new file mode 100644 index 00000000000..a515df48536 --- /dev/null +++ b/cmd/nerdctl/main_test_test.go @@ -0,0 +1,147 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "errors" + "log" + "testing" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +// TestTest is testing the test tooling itself +func TestTest(t *testing.T) { + nerdtest.Setup() + + tg := &test.Group{ + { + Description: "failure", + Command: test.RunCommand("undefinedcommand"), + Expected: test.Expects(1, nil, nil), + }, + { + Description: "success", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, nil), + }, + { + Description: "failure with single error testing", + Command: test.RunCommand("undefinedcommand"), + Expected: test.Expects(1, []error{errors.New("unknown subcommand")}, nil), + }, + { + Description: "success with contains output testing", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, test.Contains("Kernel")), + }, + { + Description: "success with negative output testing", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, test.DoesNotContain("foobar")), + }, + // Note that docker annoyingly returns 125 in a few conditions like this + { + Description: "failure with multiple error testing", + Command: test.RunCommand("-fail"), + Expected: test.Expects(-1, []error{errors.New("unknown"), errors.New("shorthand")}, nil), + }, + { + Description: "success with exact output testing", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.CustomCommand("echo", "foobar") + }, + Expected: test.Expects(0, nil, test.Equals("foobar\n")), + }, + { + Description: "data propagation", + Data: test.WithData("status", "uninitialized"), + Setup: func(data test.Data, helpers test.Helpers) { + data.Set("status", data.Get("status")+"-setup") + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + cmd := helpers.CustomCommand("printf", data.Get("status")) + data.Set("status", data.Get("status")+"-command") + return cmd + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Get("first-run") == "" { + data.Set("first-run", "first cleanup") + return + } + if data.Get("status") != "uninitialized-setup-command" { + log.Fatalf("unexpected status label %q", data.Get("status")) + } + data.Set("status", data.Get("status")+"-cleanup") + }, + SubTests: []*test.Case{ + { + Description: "Subtest data propagation", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.CustomCommand("printf", data.Get("status")) + }, + Expected: test.Expects(0, nil, test.Equals("uninitialized-setup-command")), + }, + }, + Expected: test.Expects(0, nil, test.Equals("uninitialized-setup")), + }, + { + Description: "env propagation and isolation", + Env: map[string]string{ + "GLOBAL_ENV": "in this test", + }, + Setup: func(data test.Data, helpers test.Helpers) { + cmd := helpers.CustomCommand("sh", "-c", "--", "printf \"$GLOBAL_ENV\"") + cmd.Run(&test.Expected{ + Output: test.Equals("in this test"), + }) + cmd.WithEnv(map[string]string{ + "GLOBAL_ENV": "overridden in setup", + }) + cmd.Run(&test.Expected{ + Output: test.Equals("overridden in setup"), + }) + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + cmd := helpers.CustomCommand("sh", "-c", "--", "printf \"$GLOBAL_ENV\"") + cmd.Run(&test.Expected{ + Output: test.Equals("in this test"), + }) + cmd.WithEnv(map[string]string{ + "GLOBAL_ENV": "overridden in command", + }) + return cmd + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + cmd := helpers.CustomCommand("sh", "-c", "--", "printf \"$GLOBAL_ENV\"") + cmd.Run(&test.Expected{ + Output: test.Equals("in this test"), + }) + cmd.WithEnv(map[string]string{ + "GLOBAL_ENV": "overridden in cleanup", + }) + cmd.Run(&test.Expected{ + Output: test.Equals("overridden in cleanup"), + }) + }, + Expected: test.Expects(0, nil, test.Equals("overridden in command")), + }, + } + + tg.Run(t) +} diff --git a/cmd/nerdctl/volume/volume_create_test.go b/cmd/nerdctl/volume/volume_create_test.go index a9126f704f7..767f7ac12be 100644 --- a/cmd/nerdctl/volume/volume_create_test.go +++ b/cmd/nerdctl/volume/volume_create_test.go @@ -17,155 +17,97 @@ package volume import ( + "errors" "testing" - "gotest.tools/v3/icmd" - "github.com/containerd/errdefs" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestVolumeCreate(t *testing.T) { - t.Parallel() - - base := testutil.NewBase(t) + nerdtest.Setup() - malformed := errdefs.ErrInvalidArgument.Error() - atMost := "at most 1 arg" - exitCodeVariant := 1 - if base.Target == testutil.Docker { - malformed = "invalid" - exitCodeVariant = 125 - } - - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ + tg := &test.Group{ { - description: "arg missing should create anonymous volume", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - } - }, + Description: "arg missing should create anonymous volume", + Command: test.RunCommand("volume", "create"), + Expected: test.Expects(0, nil, nil), }, { - description: "invalid identifier should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", "∞") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: malformed, - } - }, + Description: "invalid identifier should fail", + Command: test.RunCommand("volume", "create", "∞"), + Expected: test.Expects(1, []error{errdefs.ErrInvalidArgument}, nil), }, { - description: "too many args should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", "too", "many") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: atMost, - } - }, + Description: "too many args should fail", + Command: test.RunCommand("volume", "create", "too", "many"), + Expected: test.Expects(1, []error{errors.New("at most 1 arg")}, nil), }, { - description: "success", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", tID) + Description: "success", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, { - description: "success with labels", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", "--label", "foo1=baz1", "--label", "foo2=baz2", tID) + Description: "success with labels", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", "--label", "foo1=baz1", "--label", "foo2=baz2", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, { - description: "invalid labels", - command: func(tID string) *testutil.Cmd { + Description: "invalid labels", + Command: func(data test.Data, helpers test.Helpers) test.Command { // See https://github.com/containerd/nerdctl/issues/3126 - return base.Cmd("volume", "create", "--label", "a", "--label", "", tID) + return helpers.Command("volume", "create", "--label", "a", "--label", "", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: exitCodeVariant, - Err: malformed, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + // NOTE: docker returns 125 on this + ExitCode: -1, + Errors: []error{errdefs.ErrInvalidArgument}, } }, }, { - description: "creating already existing volume should succeed", - command: func(tID string) *testutil.Cmd { - base.Cmd("volume", "create", tID).AssertOK() - return base.Cmd("volume", "create", tID) + Description: "creating already existing volume should succeed", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - tt.Parallel() - - tID := testutil.Identifier(tt) - - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - cmd := currentTest.command(tID) - cmd.Assert(currentTest.expected(tID)) - }) - } + tg.Run(t) } diff --git a/cmd/nerdctl/volume/volume_inspect_test.go b/cmd/nerdctl/volume/volume_inspect_test.go index 0d4ce4e6d05..edee98be906 100644 --- a/cmd/nerdctl/volume/volume_inspect_test.go +++ b/cmd/nerdctl/volume/volume_inspect_test.go @@ -19,266 +19,185 @@ package volume import ( "crypto/rand" "encoding/json" + "errors" "fmt" "os" "path/filepath" - "strings" "testing" "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" "github.com/containerd/errdefs" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/native" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) -func createFileWithSize(base *testutil.Base, vol string, size int64) { - v := base.InspectVolume(vol) +func createFileWithSize(mountPoint string, size int64) error { token := make([]byte, size) _, _ = rand.Read(token) - err := os.WriteFile(filepath.Join(v.Mountpoint, "test-file"), token, 0644) - assert.NilError(base.T, err) + err := os.WriteFile(filepath.Join(mountPoint, "test-file"), token, 0644) + return err } func TestVolumeInspect(t *testing.T) { - t.Parallel() - - base := testutil.NewBase(t) - tID := testutil.Identifier(t) + nerdtest.Setup() var size int64 = 1028 - malformed := errdefs.ErrInvalidArgument.Error() - notFound := errdefs.ErrNotFound.Error() - requireArg := "requires at least 1 arg" - if base.Target == testutil.Docker { - malformed = "no such volume" - notFound = "no such volume" - } - - tearUp := func(t *testing.T) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("volume", "create", "--label", "foo=fooval", "--label", "bar=barval", tID+"-second").AssertOK() - - // Obviously note here that if inspect code gets totally hosed, this entire suite will - // probably fail right here on the tearUp instead of actually testing something - createFileWithSize(base, tID, size) - } - - tearDown := func(t *testing.T) { - base.Cmd("volume", "rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID+"-second").Run() - } - - tearDown(t) - t.Cleanup(func() { - tearDown(t) - }) - tearUp(t) - - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ - { - description: "arg missing should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: requireArg, - } - }, - }, - { - description: "invalid identifier should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "∞") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: malformed, - } - }, - }, - { - description: "non existent volume should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "doesnotexist") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: notFound, - } - }, - }, - { - description: "success", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", tID) - }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))) - assert.Assert(t, dc[0].Name == tID, fmt.Sprintf("expected name to be %q (was %q)", tID, dc[0].Name)) - assert.Assert(t, dc[0].Labels == nil, fmt.Sprintf("expected labels to be nil and were %v", dc[0].Labels)) - }, - }, - { - description: "inspect labels", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", tID+"-second") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - - labels := *dc[0].Labels - assert.Assert(t, len(labels) == 2, fmt.Sprintf("two results, not %d", len(labels))) - assert.Assert(t, labels["foo"] == "fooval", fmt.Sprintf("label foo should be fooval, not %s", labels["foo"])) - assert.Assert(t, labels["bar"] == "barval", fmt.Sprintf("label bar should be barval, not %s", labels["bar"])) - }, + tc := &test.Case{ + Description: "Volume inspect", + Setup: func(data test.Data, helpers test.Helpers) { + data.Set("volprefix", data.Identifier()) + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("volume", "create", "--label", "foo=fooval", "--label", "bar=barval", data.Identifier()+"-second") + // Obviously note here that if inspect code gets totally hosed, this entire suite will + // probably fail right here on the Setup instead of actually testing something + vol := nerdtest.InspectVolume(helpers, data.Identifier()) + err := createFileWithSize(vol.Mountpoint, size) + assert.NilError(t, err, "File creation failed") }, - { - description: "inspect size", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "--size", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, dc[0].Size == size, fmt.Sprintf("expected size to be %d (was %d)", size, dc[0].Size)) - }, - dockerIncompatible: true, - }, - { - description: "multi success", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", tID, tID+"-second") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, len(dc) == 2, fmt.Sprintf("two results, not %d", len(dc))) - assert.Assert(t, dc[0].Name == tID, fmt.Sprintf("expected name to be %q (was %q)", tID, dc[0].Name)) - assert.Assert(t, dc[1].Name == tID+"-second", fmt.Sprintf("expected name to be %q (was %q)", tID+"-second", dc[1].Name)) - }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-second") }, - { - description: "part success multi", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "invalid∞", "nonexistent", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Out: tID, - Err: notFound, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, malformed)) - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))) - assert.Assert(t, dc[0].Name == tID, fmt.Sprintf("expected name to be %q (was %q)", tID, dc[0].Name)) - }, - }, - { - description: "multi failure", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "invalid∞", "nonexistent") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, malformed)) + SubTests: []*test.Case{ + { + Description: "arg missing should fail", + Command: test.RunCommand("volume", "inspect"), + Expected: test.Expects(1, []error{errors.New("requires at least 1 arg")}, nil), + }, + { + Description: "invalid identifier should fail", + Command: test.RunCommand("volume", "inspect", "∞"), + Expected: test.Expects(1, []error{errdefs.ErrInvalidArgument}, nil), + }, + { + Description: "non existent volume should fail", + Command: test.RunCommand("volume", "inspect", "doesnotexist"), + Expected: test.Expects(1, []error{errdefs.ErrNotFound}, nil), + }, + { + Description: "success", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("volprefix")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))+info) + assert.Assert(t, dc[0].Name == data.Get("volprefix"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix"), dc[0].Name)+info) + assert.Assert(t, dc[0].Labels == nil, fmt.Sprintf("expected labels to be nil and were %v", dc[0].Labels)+info) + }, + ), + } + }, + }, + { + Description: "inspect labels", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("volprefix")+"-second") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + labels := *dc[0].Labels + assert.Assert(t, len(labels) == 2, fmt.Sprintf("two results, not %d", len(labels))) + assert.Assert(t, labels["foo"] == "fooval", fmt.Sprintf("label foo should be fooval, not %s", labels["foo"])) + assert.Assert(t, labels["bar"] == "barval", fmt.Sprintf("label bar should be barval, not %s", labels["bar"])) + }, + ), + } + }, + }, + { + Description: "inspect size", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", "--size", data.Get("volprefix")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, dc[0].Size == size, fmt.Sprintf("expected size to be %d (was %d)", size, dc[0].Size)) + }, + ), + } + }, + }, + { + Description: "multi success", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("volprefix"), data.Get("volprefix")+"-second") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + test.Contains(data.Get("volprefix")+"-second"), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, len(dc) == 2, fmt.Sprintf("two results, not %d", len(dc))) + assert.Assert(t, dc[0].Name == data.Get("volprefix"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix"), dc[0].Name)) + assert.Assert(t, dc[1].Name == data.Get("volprefix")+"-second", fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix")+"-second", dc[1].Name)) + }, + ), + } + }, + }, + { + Description: "part success multi", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", "invalid∞", "nonexistent", data.Get("volprefix")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{errdefs.ErrNotFound, errdefs.ErrInvalidArgument}, + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))) + assert.Assert(t, dc[0].Name == data.Get("volprefix"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix"), dc[0].Name)) + }, + ), + } + }, + }, + { + Description: "multi failure", + Command: test.RunCommand("volume", "inspect", "invalid∞", "nonexistent"), + Expected: test.Expects(1, []error{errdefs.ErrNotFound, errdefs.ErrInvalidArgument}, nil), }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - tt.Parallel() - - // We use the main test tID here - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(tID) - res := cmd.Run() - cmd.Assert(currentTest.expected(tID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + tc.Run(t) } diff --git a/cmd/nerdctl/volume/volume_list_test.go b/cmd/nerdctl/volume/volume_list_test.go index 6195db2eb13..d0ad6d78463 100644 --- a/cmd/nerdctl/volume/volume_list_test.go +++ b/cmd/nerdctl/volume/volume_list_test.go @@ -17,464 +17,376 @@ package volume import ( - "errors" "fmt" "strings" "testing" + "gotest.tools/v3/assert" + "github.com/containerd/nerdctl/v2/pkg/tabutil" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) -func TestVolumeLs(t *testing.T) { - t.Parallel() - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - testutil.DockerIncompatible(t) - - var vol1, vol2, vol3 = tID + "vol-1", tID + "vol-2", tID + "empty" - - tearDown := func() { - base.Cmd("volume", "rm", "-f", vol1).Run() - base.Cmd("volume", "rm", "-f", vol2).Run() - base.Cmd("volume", "rm", "-f", vol3).Run() - } - - tearUp := func() { - base.Cmd("volume", "create", vol1).AssertOK() - base.Cmd("volume", "create", vol2).AssertOK() - base.Cmd("volume", "create", vol3).AssertOK() - createFileWithSize(base, vol1, 102400) - createFileWithSize(base, vol2, 204800) - } - - tearDown() - t.Cleanup(func() { - tearDown() - }) - tearUp() - - base.Cmd("volume", "ls", "--size").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 4 { - return errors.New("expected at least 4 lines") - } - volSizes := map[string]string{ - vol1: "100.0 KiB", - vol2: "200.0 KiB", - vol3: "0.0 B", - } - - var numMatches = 0 - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err - } - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - size, _ := tab.ReadRow(line, "SIZE") - expectSize, ok := volSizes[name] - if !ok { - continue - } - if size != expectSize { - return fmt.Errorf("expected size %s for volume %s, got %s", expectSize, name, size) - } - numMatches++ - } - if len(volSizes) != numMatches { - return fmt.Errorf("expected %d volumes, got: %d", len(volSizes), numMatches) - } - return nil - }) - -} - -func TestVolumeLsFilter(t *testing.T) { - t.Parallel() - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - - var vol1, vol2, vol3, vol4 = tID + "vol-1", tID + "vol-2", tID + "vol-3", tID + "vol-4" - var label1, label2, label3, label4 = tID + "=label-1", tID + "=label-2", tID + "=label-3", tID + "-group=label-4" - - tearDown := func() { - base.Cmd("volume", "rm", "-f", vol1).Run() - base.Cmd("volume", "rm", "-f", vol2).Run() - base.Cmd("volume", "rm", "-f", vol3).Run() - base.Cmd("volume", "rm", "-f", vol4).Run() - } - - tearUp := func() { - base.Cmd("volume", "create", "--label="+label1, "--label="+label4, vol1).AssertOK() - base.Cmd("volume", "create", "--label="+label2, "--label="+label4, vol2).AssertOK() - base.Cmd("volume", "create", "--label="+label3, vol3).AssertOK() - base.Cmd("volume", "create", vol4).AssertOK() - } - - tearDown() - t.Cleanup(func() { - tearDown() - }) - tearUp() - - testCases := []struct { - description string - command func(tID string) - }{ - { - description: "no filter", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet").AssertOutWithFunc(func(stdout string) error { +func TestVolumeLsSize(t *testing.T) { + nerdtest.Setup() + + tc := &test.Case{ + Description: "Volume ls --size", + Require: test.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()+"-1") + helpers.Ensure("volume", "create", data.Identifier()+"-2") + helpers.Ensure("volume", "create", data.Identifier()+"-empty") + vol1 := nerdtest.InspectVolume(helpers, data.Identifier()+"-1") + vol2 := nerdtest.InspectVolume(helpers, data.Identifier()+"-2") + + err := createFileWithSize(vol1.Mountpoint, 102400) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(vol2.Mountpoint, 204800) + assert.NilError(t, err, "File creation failed") + }, + Command: test.RunCommand("volume", "ls", "--size"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 4 { - return errors.New("expected at least 4 lines") - } - volNames := map[string]struct{}{ - vol1: {}, - vol2: {}, - vol3: {}, - vol4: {}, + assert.Assert(t, len(lines) >= 4, "expected at least 4 lines"+info) + volSizes := map[string]string{ + data.Identifier() + "-1": "100.0 KiB", + data.Identifier() + "-2": "200.0 KiB", + data.Identifier() + "-empty": "0.0 B", } var numMatches = 0 - for _, name := range lines { - _, ok := volNames[name] + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, info) + + for _, line := range lines { + name, _ := tab.ReadRow(line, "VOLUME NAME") + size, _ := tab.ReadRow(line, "SIZE") + expectSize, ok := volSizes[name] if !ok { continue } + assert.Assert(t, size == expectSize, fmt.Sprintf("expected size %s for volume %s, got %s", expectSize, name, size)+info) numMatches++ } - if len(volNames) != numMatches { - return fmt.Errorf("expected %d volumes, got: %d", len(volNames), numMatches) - } - return nil - }) - }, + assert.Assert(t, numMatches == len(volSizes), fmt.Sprintf("expected %d volumes, got: %d", len(volSizes), numMatches)+info) + }, + } }, - { - description: "label=" + tID, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+tID).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") - } - volNames := map[string]struct{}{ - vol1: {}, - vol2: {}, - vol3: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } - } - return nil - }) - }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-1") + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-2") + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-empty") }, - { - description: "label=" + label2, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+label2).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 1 { - return errors.New("expected at least 1 lines") - } - volNames := map[string]struct{}{ - vol2: {}, - } + } + + tc.Run(t) +} + +func TestVolumeLsFilter(t *testing.T) { + nerdtest.Setup() + + tc := &test.Case{ + Description: "Volume ls", + Setup: func(data test.Data, helpers test.Helpers) { + var vol1, vol2, vol3, vol4 = data.Identifier() + "-1", data.Identifier() + "-2", data.Identifier() + "-3", data.Identifier() + "-4" + var label1, label2, label3, label4 = data.Identifier() + "=label-1", data.Identifier() + "=label-2", data.Identifier() + "=label-3", data.Identifier() + "-group=label-4" + + helpers.Ensure("volume", "create", "--label="+label1, "--label="+label4, vol1) + helpers.Ensure("volume", "create", "--label="+label2, "--label="+label4, vol2) + helpers.Ensure("volume", "create", "--label="+label3, vol3) + helpers.Ensure("volume", "create", vol4) + + err := createFileWithSize(nerdtest.InspectVolume(helpers, vol1).Mountpoint, 409600) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(nerdtest.InspectVolume(helpers, vol2).Mountpoint, 1024000) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(nerdtest.InspectVolume(helpers, vol3).Mountpoint, 409600) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(nerdtest.InspectVolume(helpers, vol4).Mountpoint, 1024000) + assert.NilError(t, err, "File creation failed") + + data.Set("vol1", vol1) + data.Set("vol2", vol2) + data.Set("vol3", vol3) + data.Set("vol4", vol4) + data.Set("mainlabel", data.Identifier()) + data.Set("label1", label1) + data.Set("label2", label2) + data.Set("label3", label3) + data.Set("label4", label4) - for _, name := range lines { - if name == "" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } - } - return nil - }) - }, }, - { - description: "label=" + tID + "=", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+tID+"=").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) > 0 { - for _, name := range lines { - if name != "" { - return fmt.Errorf("unexpected volumes %d found", len(lines)) + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Get("vol1")) + helpers.Anyhow("volume", "rm", "-f", data.Get("vol2")) + helpers.Anyhow("volume", "rm", "-f", data.Get("vol3")) + helpers.Anyhow("volume", "rm", "-f", data.Get("vol4")) + }, + SubTests: []*test.Case{ + { + Description: "No filter", + Command: test.RunCommand("volume", "ls", "--quiet"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 4, "expected at least 4 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, + data.Get("vol3"): {}, + data.Get("vol4"): {}, } - } + var numMatches = 0 + for _, name := range lines { + _, ok := volNames[name] + if !ok { + continue + } + numMatches++ + } + assert.Assert(t, len(volNames) == numMatches, fmt.Sprintf("expected %d volumes, got: %d", len(volNames), numMatches)) + }, } - return nil - }) + }, }, - }, - { - description: "label=" + label1 + " label=" + label2, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+label1, "--filter", "label="+label2).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) > 0 { - for _, name := range lines { - if name != "" { - return fmt.Errorf("unexpected volumes %d found", len(lines)) + { + Description: "Retrieving label=mainlabel", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("mainlabel")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, + data.Get("vol3"): {}, } - } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) - + }, }, - }, - { - description: "label=" + tID + " label=" + label4, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+tID, "--filter", "label="+label4).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 2 { - return errors.New("expected at least 2 lines") - } - volNames := map[string]struct{}{ - vol1: {}, - vol2: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + { + Description: "Retrieving label=mainlabel=label2", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("label2")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 1, "expected at least 1 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol2"): {}, + } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) + }, }, - }, - { - description: "name=" + vol1, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "name="+vol1).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 1 { - return errors.New("expected at least 1 lines") + { + Description: "Retrieving label=mainlabel=", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("mainlabel")+"=") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, strings.TrimSpace(stdout) == "", "expected no result"+info) + }, } - volNames := map[string]struct{}{ - vol1: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + }, + }, + { + Description: "Retrieving label=mainlabel=label1 and label=mainlabel=label2", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("label1"), "--filter", "label="+data.Get("label2")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, strings.TrimSpace(stdout) == "", "expected no result"+info) + }, } - return nil - }) + }, }, - }, - { - description: "name=vol-3", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "name=vol-3"). - AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 1 { - return errors.New("expected at least 1 lines") - } - volNames := map[string]struct{}{ - vol3: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) + { + Description: "Retrieving label=mainlabel and label=grouplabel=label4", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("mainlabel"), "--filter", "label="+data.Get("label4")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 2, "expected at least 2 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, } - } - return nil - }) - }, - }, - { - description: "name=vol2 name=vol1", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "name=vol2", "--filter", "name=vol1"). - AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) > 0 { for _, name := range lines { - if name != "" { - return fmt.Errorf("unexpected volumes %d found", len(lines)) - } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) } - } - return nil - }) - }, - }, - } - - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - tt.Parallel() - currentTest.command(tID) - }) - } - -} - -func TestVolumeLsFilterSize(t *testing.T) { - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - testutil.DockerIncompatible(t) - - var vol1, vol2, vol3, vol4 = tID + "volsize-1", tID + "volsize-2", tID + "volsize-3", tID + "volsize-4" - var label1, label2, label3, label4 = tID + "=label-1", tID + "=label-2", tID + "=label-3", tID + "-group=label-4" - - tearDown := func() { - base.Cmd("volume", "rm", "-f", vol1).Run() - base.Cmd("volume", "rm", "-f", vol2).Run() - base.Cmd("volume", "rm", "-f", vol3).Run() - base.Cmd("volume", "rm", "-f", vol4).Run() - } - - tearUp := func() { - base.Cmd("volume", "create", "--label="+label1, "--label="+label4, vol1).AssertOK() - base.Cmd("volume", "create", "--label="+label2, "--label="+label4, vol2).AssertOK() - base.Cmd("volume", "create", "--label="+label3, vol3).AssertOK() - base.Cmd("volume", "create", vol4).AssertOK() - - createFileWithSize(base, vol1, 409600) - createFileWithSize(base, vol2, 1024000) - createFileWithSize(base, vol3, 409600) - createFileWithSize(base, vol4, 1024000) - } - - tearDown() - t.Cleanup(func() { - tearDown() - }) - tearUp() - - testCases := []struct { - description string - command func(tID string) - }{ - { - description: "size=1024000", - command: func(tID string) { - base.Cmd("volume", "ls", "--size", "--filter", "size=1024000").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") + }, } - - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err - } - volNames := map[string]struct{}{ - vol2: {}, - vol4: {}, - } - - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - if name == "VOLUME NAME" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } - } - return nil - }) + }, }, - }, - { - description: "size>=1024000 size<=2048000", - command: func(tID string) { - base.Cmd("volume", "ls", "--size", "--filter", "size>=1024000", "--filter", "size<=2048000").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") - } - - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err - } - volNames := map[string]struct{}{ - vol2: {}, - vol4: {}, - } - - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - if name == "VOLUME NAME" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + { + Description: "Retrieving name=volume1", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "name="+data.Get("vol1")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 1, "expected at least 1 line"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) + }, }, - }, - { - description: "size>204800 size<1024000", - command: func(tID string) { - base.Cmd("volume", "ls", "--size", "--filter", "size>204800", "--filter", "size<1024000").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") + { + Description: "Retrieving name=volume1 and name=volume2", + // FIXME: https://github.com/containerd/nerdctl/issues/3452 + // Nerdctl filter behavior is broken + Require: nerdtest.Docker, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "name="+data.Get("vol1"), "--filter", "name="+data.Get("vol2")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 2, "expected at least 2 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, + } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err + }, + }, + { + Description: "Retrieving size=1024000", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--size", "--filter", "size=1024000") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol2"): {}, + data.Get("vol4"): {}, + } + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, "Tab reader failed") + for _, line := range lines { + + name, _ := tab.ReadRow(line, "VOLUME NAME") + if name == "VOLUME NAME" { + continue + } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - volNames := map[string]struct{}{ - vol1: {}, - vol3: {}, + }, + }, + { + Description: "Retrieving size>=1024000 size<=2048000", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--size", "--filter", "size>=1024000", "--filter", "size<=2048000") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol2"): {}, + data.Get("vol4"): {}, + } + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, "Tab reader failed") + for _, line := range lines { + + name, _ := tab.ReadRow(line, "VOLUME NAME") + if name == "VOLUME NAME" { + continue + } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - if name == "VOLUME NAME" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + }, + }, + { + Description: "Retrieving size>204800 size<1024000", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--size", "--filter", "size>204800", "--filter", "size<1024000") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol3"): {}, + } + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, "Tab reader failed") + for _, line := range lines { + + name, _ := tab.ReadRow(line, "VOLUME NAME") + if name == "VOLUME NAME" { + continue + } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) + }, }, }, } - - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - tt.Parallel() - currentTest.command(tID) - }) - } + tc.Run(t) } diff --git a/cmd/nerdctl/volume/volume_namespace_test.go b/cmd/nerdctl/volume/volume_namespace_test.go index dcda29a3648..b20f64984dc 100644 --- a/cmd/nerdctl/volume/volume_namespace_test.go +++ b/cmd/nerdctl/volume/volume_namespace_test.go @@ -19,134 +19,78 @@ package volume import ( "testing" - "gotest.tools/v3/icmd" - "github.com/containerd/errdefs" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestVolumeNamespace(t *testing.T) { - testutil.DockerIncompatible(t) - - t.Parallel() - - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - otherBase := testutil.NewBaseWithNamespace(t, tID+"-1") - thirdBase := testutil.NewBaseWithNamespace(t, tID+"-2") - - tearUp := func(t *testing.T) { - base.Cmd("volume", "create", tID).AssertOK() - } - - tearDown := func(t *testing.T) { - base.Cmd("volume", "rm", "-f", tID).Run() - otherBase.Cmd("namespace", "rm", "-f", tID+"-1").Run() - thirdBase.Cmd("namespace", "rm", "-f", tID+"-2").Run() - } - - tearDown(t) - t.Cleanup(func() { - tearDown(t) - }) - tearUp(t) - - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ - { - description: "inspect another namespace volume should fail", - command: func(tID string) *testutil.Cmd { - return otherBase.Cmd("volume", "inspect", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: errdefs.ErrNotFound.Error(), - } - }, + nerdtest.Setup() + + tg := &test.Case{ + Description: "Namespaces", + Require: test.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + data.Set("root_namespace", data.Identifier()) + data.Set("root_volume", data.Identifier()) + helpers.Ensure("--namespace", data.Identifier(), "volume", "create", data.Identifier()) }, - { - description: "remove another namespace volume should fail", - command: func(tID string) *testutil.Cmd { - return otherBase.Cmd("volume", "remove", tID) + SubTests: []*test.Case{ + { + Description: "inspect another namespace volume should fail", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("root_volume")) + }, + Expected: test.Expects(1, []error{ + errdefs.ErrNotFound, + }, nil), }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: errdefs.ErrNotFound.Error(), - } + { + Description: "removing another namespace volume should fail", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "remove", data.Get("root_volume")) + }, + Expected: test.Expects(1, []error{ + errdefs.ErrNotFound, + }, nil), }, - }, - { - description: "prune should leave other namespace untouched", - command: func(tID string) *testutil.Cmd { - return otherBase.Cmd("volume", "prune", "-a", "-f") - }, - tearDown: func(tID string) { - // Assert that the volume is here in the base namespace - // both before and after the prune command - base.Cmd("volume", "inspect", tID).AssertOK() - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - } - }, - }, - { - description: "create with namespace should work", - command: func(tID string) *testutil.Cmd { - return thirdBase.Cmd("volume", "create", tID) + { + Description: "prune should leave another namespace volume untouched", + NoParallel: true, + Command: test.RunCommand("volume", "prune", "-a", "-f"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.DoesNotContain(data.Get("root_volume")), + func(stdout string, info string, t *testing.T) { + helpers.Ensure("--namespace", data.Get("root_namespace"), "volume", "inspect", data.Get("root_volume")) + }, + ), + } + }, }, - tearDown: func(tID string) { - thirdBase.Cmd("volume", "remove", "-f", tID).Run() - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } + { + Description: "create with the same name should work, then delete it", + NoParallel: true, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", data.Get("root_volume")) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", data.Get("root_volume")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + helpers.Ensure("volume", "inspect", data.Get("root_volume")) + helpers.Ensure("volume", "rm", data.Get("root_volume")) + helpers.Ensure("--namespace", data.Get("root_namespace"), "volume", "inspect", data.Get("root_volume")) + }, + } + }, }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - tt.Parallel() - - // Note that here we are using the main test tID - // since we are testing against the volume created in it - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(tID) - res := cmd.Run() - cmd.Assert(currentTest.expected(tID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + tg.Run(t) } diff --git a/cmd/nerdctl/volume/volume_prune_linux_test.go b/cmd/nerdctl/volume/volume_prune_linux_test.go index 19632a73992..8898ad30ab4 100644 --- a/cmd/nerdctl/volume/volume_prune_linux_test.go +++ b/cmd/nerdctl/volume/volume_prune_linux_test.go @@ -20,130 +20,90 @@ import ( "strings" "testing" - "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestVolumePrune(t *testing.T) { - // Volume pruning cannot be parallelized for Docker, since we need namespaces to do that in a way that does interact with other tests - if testutil.GetTarget() != testutil.Docker { - t.Parallel() - } + nerdtest.Setup() + + var setup = func(data test.Data, helpers test.Helpers) { + anonIDBusy := strings.TrimSpace(helpers.Capture("volume", "create")) + anonIDDangling := strings.TrimSpace(helpers.Capture("volume", "create")) + + namedBusy := data.Identifier() + "-busy" + namedDangling := data.Identifier() + "-free" - // FIXME: for an unknown reason, when testing ipv6, calling NewBaseWithNamespace per sub-test, in the tearDown/tearUp methods - // will actually panic the test (also happens with target=docker) - // Calling base here *first* so that it can skip NOW - does seem to workaround the problem - // If you have any idea how to properly do this, feel free to remove the following line and fix the underlying issue - testutil.NewBase(t) + helpers.Ensure("volume", "create", namedBusy) + helpers.Ensure("volume", "create", namedDangling) + helpers.Ensure("run", "--name", data.Identifier(), + "-v", namedBusy+":/whatever", + "-v", anonIDBusy+":/other", testutil.CommonImage) - subTearUp := func(tID string) { - base := testutil.NewBaseWithNamespace(t, tID) - res := base.Cmd("volume", "create").Run() - anonIDBusy := res.Stdout() - base.Cmd("volume", "create").Run() - base.Cmd("volume", "create", tID+"-busy").AssertOK() - base.Cmd("volume", "create", tID+"-free").AssertOK() - base.Cmd("run", "--name", tID, - "-v", tID+"-busy:/whatever", - "-v", anonIDBusy, testutil.CommonImage).AssertOK() + data.Set("anonIDBusy", anonIDBusy) + data.Set("anonIDDangling", anonIDDangling) + data.Set("namedBusy", namedBusy) + data.Set("namedDangling", namedDangling) } - subTearDown := func(tID string) { - base := testutil.NewBaseWithNamespace(t, tID) - base.Cmd("rm", "-f", tID).Run() - base.Cmd("namespace", "remove", "-f", tID).Run() + var cleanup = func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("rm", "-f", data.Get("anonIDBusy")) + helpers.Anyhow("rm", "-f", data.Get("anonIDDangling")) + helpers.Anyhow("rm", "-f", data.Get("namedBusy")) + helpers.Anyhow("rm", "-f", data.Get("namedDangling")) } - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ + // This set must be marked as private, since we cannot prune without interacting with other tests. + testGroup := &test.Group{ { - description: "prune anonymous only", - command: func(tID string) *testutil.Cmd { - base := testutil.NewBaseWithNamespace(t, tID) - return base.Cmd("volume", "prune", "-f") - }, - tearUp: subTearUp, - tearDown: subTearDown, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, + Description: "prune anonymous only", + Require: nerdtest.Private, + Command: test.RunCommand("volume", "prune", "-f"), + Setup: setup, + Cleanup: cleanup, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.DoesNotContain(data.Get("anonIDBusy")), + test.Contains(data.Get("anonIDDangling")), + test.DoesNotContain(data.Get("namedBusy")), + test.DoesNotContain(data.Get("namedDangling")), + func(stdout string, info string, t *testing.T) { + helpers.Ensure("volume", "inspect", data.Get("anonIDBusy")) + helpers.Fail("volume", "inspect", data.Get("anonIDDangling")) + helpers.Ensure("volume", "inspect", data.Get("namedBusy")) + helpers.Ensure("volume", "inspect", data.Get("namedDangling")) + }, + ), } }, - inspect: func(t *testing.T, stdout string, stderr string) { - tID := testutil.Identifier(t) - base := testutil.NewBaseWithNamespace(t, tID) - assert.Assert(base.T, !strings.Contains(stdout, tID+"-free")) - base.Cmd("volume", "inspect", tID+"-free").AssertOK() - assert.Assert(base.T, !strings.Contains(stdout, tID+"-busy")) - base.Cmd("volume", "inspect", tID+"-busy").AssertOK() - // TODO verify the anonymous volumes status - }, }, { - description: "prune all", - command: func(tID string) *testutil.Cmd { - base := testutil.NewBaseWithNamespace(t, tID) - return base.Cmd("volume", "prune", "-f", "--all") - }, - tearUp: subTearUp, - tearDown: subTearDown, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, + Description: "prune all", + Require: nerdtest.Private, + Command: test.RunCommand("volume", "prune", "-f", "--all"), + Setup: setup, + Cleanup: cleanup, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.DoesNotContain(data.Get("anonIDBusy")), + test.Contains(data.Get("anonIDDangling")), + test.DoesNotContain(data.Get("namedBusy")), + test.Contains(data.Get("namedDangling")), + func(stdout string, info string, t *testing.T) { + helpers.Ensure("volume", "inspect", data.Get("anonIDBusy")) + helpers.Fail("volume", "inspect", data.Get("anonIDDangling")) + helpers.Ensure("volume", "inspect", data.Get("namedBusy")) + helpers.Fail("volume", "inspect", data.Get("namedDangling")) + }, + ), } }, - inspect: func(t *testing.T, stdout string, stderr string) { - tID := testutil.Identifier(t) - base := testutil.NewBaseWithNamespace(t, tID) - assert.Assert(t, !strings.Contains(stdout, tID+"-busy")) - base.Cmd("volume", "inspect", tID+"-busy").AssertOK() - assert.Assert(t, strings.Contains(stdout, tID+"-free")) - base.Cmd("volume", "inspect", tID+"-free").AssertFail() - // TODO verify the anonymous volumes status - }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - if testutil.GetTarget() != testutil.Docker { - tt.Parallel() - } - - subTID := testutil.Identifier(tt) - - if currentTest.tearDown != nil { - currentTest.tearDown(subTID) - tt.Cleanup(func() { - currentTest.tearDown(subTID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(subTID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(subTID) - res := cmd.Run() - cmd.Assert(currentTest.expected(subTID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + testGroup.Run(t) } diff --git a/cmd/nerdctl/volume/volume_remove_linux_test.go b/cmd/nerdctl/volume/volume_remove_linux_test.go index 04c67a131aa..ba775614766 100644 --- a/cmd/nerdctl/volume/volume_remove_linux_test.go +++ b/cmd/nerdctl/volume/volume_remove_linux_test.go @@ -17,16 +17,17 @@ package volume import ( + "errors" "fmt" - "strings" "testing" "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" "github.com/containerd/errdefs" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) // TestVolumeRemove does test a large variety of volume remove situations, albeit none of them being @@ -34,91 +35,51 @@ import ( // Behavior in such cases is largely unspecified, as there is no easy way to compare with Docker. // Anyhow, borked filesystem conditions is not something we should be expected to deal with in a smart way. func TestVolumeRemove(t *testing.T) { - t.Parallel() - - base := testutil.NewBase(t) - - inUse := errdefs.ErrFailedPrecondition.Error() - malformed := errdefs.ErrInvalidArgument.Error() - notFound := errdefs.ErrNotFound.Error() - requireArg := "requires at least 1 arg" - if base.Target == testutil.Docker { - malformed = "no such volume" - notFound = "no such volume" - inUse = "volume is in use" - } + nerdtest.Setup() - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ + testGroup := &test.Group{ { - description: "arg missing should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: requireArg, - } - }, + Description: "arg missing should fail", + Command: test.RunCommand("volume", "rm"), + Expected: test.Expects(1, []error{errors.New("requires at least 1 arg")}, nil), }, { - description: "invalid identifier should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "∞") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: malformed, - } - }, + Description: "invalid identifier should fail", + Command: test.RunCommand("volume", "rm", "∞"), + Expected: test.Expects(1, []error{errdefs.ErrInvalidArgument}, nil), }, { - description: "non existent volume should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "doesnotexist") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: notFound, - } - }, + Description: "non existent volume should fail", + Command: test.RunCommand("volume", "rm", "doesnotexist"), + Expected: test.Expects(1, []error{errdefs.ErrNotFound}, nil), }, { - description: "busy volume should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID) - }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID), "--name", tID, testutil.CommonImage).AssertOK() + Description: "busy volume should fail", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()), + "--name", data.Identifier(), testutil.CommonImage) }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID).Run() + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: inUse, - } + Expected: test.Expects(1, []error{errdefs.ErrFailedPrecondition}, nil), + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, }, { - description: "busy anonymous volume should fail", - command: func(tID string) *testutil.Cmd { + Description: "busy anonymous volume should fail", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()), "--name", data.Identifier(), testutil.CommonImage) // Inspect the container and find the anonymous volume id - inspect := base.InspectContainer(tID) + inspect := nerdtest.InspectContainer(helpers, data.Identifier()) var anonName string for _, v := range inspect.Mounts { if v.Destination == "/volume" { @@ -127,162 +88,142 @@ func TestVolumeRemove(t *testing.T) { } } assert.Assert(t, anonName != "", "Failed to find anonymous volume id") + data.Set("anonName", anonName) + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { // Try to remove that anon volume - return base.Cmd("volume", "rm", anonName) + return helpers.Command("volume", "rm", data.Get("anonName")) }, - tearUp: func(tID string) { - // base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID), "--name", tID, testutil.CommonImage).AssertOK() - }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: inUse, - } + Expected: test.Expects(1, []error{errdefs.ErrFailedPrecondition}, nil), + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) }, }, { - description: "freed volume should succeed", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID) - }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID), "--name", tID, testutil.CommonImage).AssertOK() - base.Cmd("rm", "-f", tID).AssertOK() + Description: "freed volume should succeed", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()), "--name", data.Identifier(), testutil.CommonImage) + helpers.Ensure("rm", "-f", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID).Run() + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - Out: tID, + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) + }, }, { - description: "dangling volume should succeed", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID) + Description: "dangling volume should succeed", + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()) }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - Out: tID, + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, { - description: "part success multi-remove", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "invalid∞", "nonexistent", tID+"-busy", tID) + Description: "part success multi-remove", + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", "invalid∞", "nonexistent", data.Identifier()+"-busy", data.Identifier()) }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("volume", "create", tID+"-busy").AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID+"-busy"), "--name", tID, testutil.CommonImage).AssertOK() + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("volume", "create", data.Identifier()+"-busy") + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()+"-busy"), "--name", data.Identifier(), testutil.CommonImage) }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID+"-busy").Run() + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-busy") }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ ExitCode: 1, - Out: tID, + Errors: []error{ + errdefs.ErrNotFound, + errdefs.ErrFailedPrecondition, + errdefs.ErrInvalidArgument, + }, + Output: test.Equals(data.Identifier() + "\n"), } }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, inUse)) - assert.Assert(t, strings.Contains(stderr, malformed)) - }, }, { - description: "success multi-remove", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID+"-1", tID+"-2") + Description: "success multi-remove", + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()+"-1", data.Identifier()+"-2") }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID+"-1").AssertOK() - base.Cmd("volume", "create", tID+"-2").AssertOK() + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()+"-1") + helpers.Ensure("volume", "create", data.Identifier()+"-2") }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID+"-1", tID+"-2").Run() + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-1", data.Identifier()+"-2") }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - Out: tID + "-1\n" + tID + "-2", + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "-1\n" + data.Identifier() + "-2" + "\n"), } }, }, { - description: "failing multi-remove", - tearUp: func(tID string) { - base.Cmd("volume", "create", tID+"-busy").AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID+"-busy"), "--name", tID, testutil.CommonImage).AssertOK() - }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID+"-busy").Run() - }, - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "invalid∞", "nonexistent", tID+"-busy") + Description: "failing multi-remove", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()+"-busy") + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()+"-busy"), "--name", data.Identifier(), testutil.CommonImage) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - } + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-busy") }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, inUse)) - assert.Assert(t, strings.Contains(stderr, malformed)) + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", "invalid∞", "nonexistent", data.Identifier()+"-busy") }, + + Expected: test.Expects(1, []error{ + errdefs.ErrNotFound, + errdefs.ErrFailedPrecondition, + errdefs.ErrInvalidArgument, + }, nil), }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - tt.Parallel() - - tID := testutil.Identifier(tt) - - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(tID) - res := cmd.Run() - cmd.Assert(currentTest.expected(tID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + testGroup.Run(t) } From a74f0f3024748fb8a8263efd615970d97f67c13e Mon Sep 17 00:00:00 2001 From: apostasie Date: Tue, 17 Sep 2024 21:45:09 -0700 Subject: [PATCH 3/3] Testtool initial documentation Signed-off-by: apostasie --- .github/workflows/test-canary.yml | 2 +- .github/workflows/test-kube.yml | 2 +- .github/workflows/test.yml | 6 +- README.md | 2 +- docs/{dev/testing.md => testing/README.md} | 5 + docs/testing/tools.md | 364 +++++++++++++++++++++ 6 files changed, 375 insertions(+), 6 deletions(-) rename docs/{dev/testing.md => testing/README.md} (94%) create mode 100644 docs/testing/tools.md diff --git a/.github/workflows/test-canary.yml b/.github/workflows/test-canary.yml index 7c385edb236..d6ea18306fc 100644 --- a/.github/workflows/test-canary.yml +++ b/.github/workflows/test-canary.yml @@ -112,5 +112,5 @@ jobs: ctrdVersion: ${{ env.CONTAINERD_VERSION }} run: powershell hack/configure-windows-ci.ps1 - name: "Run integration tests" - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization run: go test -p 1 -v ./cmd/nerdctl/... diff --git a/.github/workflows/test-kube.yml b/.github/workflows/test-kube.yml index cb487898173..f42bb7945a0 100644 --- a/.github/workflows/test-kube.yml +++ b/.github/workflows/test-kube.yml @@ -22,7 +22,7 @@ jobs: with: fetch-depth: 1 - name: "Run Kubernetes integration tests" - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization run: | ./hack/build-integration-kubernetes.sh sudo ./_output/nerdctl exec nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test -p 1 ./cmd/nerdctl/... -test.only-kubernetes' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f00e7c85a8d..1501b0fcebf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -290,7 +290,7 @@ jobs: timeout_minutes: 30 max_attempts: 2 retry_on: error - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon - name: "Ensure that the IPv6 integration test suite is compatible with Docker" uses: nick-fields/retry@v3 @@ -298,7 +298,7 @@ jobs: timeout_minutes: 30 max_attempts: 2 retry_on: error - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6 test-integration-windows: @@ -332,7 +332,7 @@ jobs: run: powershell hack/configure-windows-ci.ps1 # TODO: Run unit tests - name: "Run integration tests" - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization run: go test -p 1 -v ./cmd/nerdctl/... test-integration-freebsd: diff --git a/README.md b/README.md index ef85dbd54f5..8076cd67446 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,7 @@ Using `go install github.com/containerd/nerdctl/v2/cmd/nerdctl` is possible, but ### Testing -See [testing nerdctl](docs/dev/testing.md). +See [testing nerdctl](docs/testing/README.md). ### Contributing to nerdctl diff --git a/docs/dev/testing.md b/docs/testing/README.md similarity index 94% rename from docs/dev/testing.md rename to docs/testing/README.md index 190733cb59d..c5bddc99285 100644 --- a/docs/dev/testing.md +++ b/docs/testing/README.md @@ -1,5 +1,10 @@ # Testing nerdctl +This document covers basic usage of nerdctl testing tasks, and generic recommendations +and principles about writing tests. + +For more comprehensive information about nerdctl test tools, see [tools.md](tools.md). + ## Lint ``` diff --git a/docs/testing/tools.md b/docs/testing/tools.md new file mode 100644 index 00000000000..655306dc48b --- /dev/null +++ b/docs/testing/tools.md @@ -0,0 +1,364 @@ +# Nerdctl testing tools + +## Preamble + +The integration test suite in nerdctl is meant to apply to both nerdctl and docker, +and further support additional test properties to target specific contexts (ipv6, kube). + +Basic _usage_ is covered in the [testing docs](testing.md). + +This here covers how to write tests, leveraging nerdctl `pkg/testutil/test` +which has been specifically developed to take care of repetitive tasks, +protect the developer against unintended side effects across tests, and generally +encourage clear testing structure with good debug-ability and a relatively simple API for +most cases. + +## Using `test.Case` + +Starting from scratch, the simplest, basic structure of a new test is: + +```go +package main + +import ( + "testing" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A first test", + // This is going to run `nerdctl info` (or `docker info`) + Command: test.RunCommand("info"), + // Verify the command exits with 0, and stdout contains the word `Kernel` + Expected: test.Expects(0, nil, test.Contains("Kernel")), + } + + // Run it + myTest.Run(t) +} +``` + +## Expectations + +There are a handful of helpers for "expectations". + +You already saw two (`test.Expects` and `test.Contains`): + +First, `test.Expects(exitCode int, errors []error, outputCompare Comparator)`, which is +convenient to quickly describe what you expect overall. + +`exitCode` is obvious. + +`errors` is a slice of go `error`, that allows you to compare what is seen on stderr +with existing errors (for example: `errdefs.ErrNotFound`), or more generally +any string you want to match. + +`outputCompare` can be either your own comparison function, or +one of the comparison helper. + +Secondly, `test.Contains`, is a `Comparator`. + +### Comparators + +Besides `test.Contains(string)`, there are a few more: +- `test.DoesNotContain(string)` +- `test.Equals(string)` +- `test.All(comparators ...Comparator)`, which allows you to bundle together a bunch of other comparators + +The following example shows how to implement your own custom `Comparator` +(this is actually the `Equals` comparator). + +```go +package whatever + +import ( + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func MyComparator(compare string) test.Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Assert(t, stdout == compare, info) + } +} +``` + +Note that you have access to an opaque `info` string. +It contains relevant debugging information in case your comparator is going to fail, +and you should make sure it is displayed. + +### Advanced expectations + +You may want to have expectations that contain a certain piece of data +that is being used in the command or at other stages of your test (Setup). + +For example, creating a container with a certain name, you might want to verify +that this name is then visible in the list of containers. + +To achieve that, you should write your own `Expecter`, leveraging test `Data`. + +Here is an example, where we are using `data.Get("sometestdata")`. + +```go +package main + +import ( + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/errdefs" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A subtest with custom data, manager, and comparator", + Data: test.WithData("sometestdata", "blah"), + Command: test.RunCommand("info"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{ + errors.New("foobla"), + errdefs.ErrNotFound, + }, + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, stdout == data.Get("sometestdata"), info) + }, + } + }, + } + + myTest.Run(t) +} +``` + +## On `Data` + +`Data` is provided to first allow storing mutable key-value information that pertain to the test. + +While it can be provided through `WithData(key string, value string)` (or `WithConfig`, see below), +inside the testcase definition, it can also be dynamically manipulated inside `Setup`, or `Command`. + +Note that `Data` additionally exposes the following functions: +- `Identifier()` which returns the unique id associated with the _current_ test (or subtest) +- `TempDir()` which returns the private, temporary directory associated with the test + +... along with the `Get(key)` and `Set(key, value)` methods. + +Secondly, `Data` allows defining and manipulating "configuration" data. + +In the case of nerdctl here, the following configuration options are defined: +- `WithConfig(Docker, NotCompatible)` to flag a test as not compatible +- `WithConfig(Mode, Private)` will entirely isolate the test using a different +namespace, data root, nerdctl config, etc +- `WithConfig(NerdctlToml, "foo")` which allows specifying a custom config +- `WithConfig(DataRoot, "foo")` allowing to point to a custom data-root +- `WithConfig(HostsDir, "foo")` to point to a specific hosts directory +- `WithConfig(Namespace, "foo")` allows passing a specific namespace (otherwise defaults to `nerdctl-test`) + +## Commands + +For simple cases, `test.RunCommand(args ...string)` is the way to go. + +It will execute the binary to test (nerdctl or docker), with the provided arguments, +and will by default get cwd inside the temporary directory associated with the test. + +### Environment + +You can attach custom environment variables for your test in the `Env` property of your +test. + +These will be automatically added to the environment for your command, and also +your setup and cleanup routines (see below). + +If you would like to override the environment specifically for a command, but not for +others (eg: in `Setup` or `Cleanup`), you can do so with custom commands (see below). + +Note that environment as defined statically in the test will be inherited by subtests, +unless explicitly overridden. + +### Working directory + +By default, the working directory of the command will be set to the temporary directory +of the test. + +This behavior can be overridden using custom commands. + +### Custom commands + +Custom commands allow you to manipulate test `Data`, override important aspects +of the command to execute (`Env`, `WorkingDir`), or otherwise give you full control +on what the command does. + +You just need to implement an `Executor`: + +```go +package main + +import ( + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/errdefs" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A subtest with custom data, manager, and comparator", + Data: test.WithData("sometestdata", "blah"), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("run", "--name", data.Identifier()+data.Get("sometestdata")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{ + errors.New("foobla"), + errdefs.ErrNotFound, + }, + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, stdout == data.Get("sometestdata"), info) + }, + } + }, + } + + myTest.Run(t) +} +``` + +### On `helpers` + +Inside a custom `Executor`, `Manager`, or `Butler`, you have access to a collection of +`helpers` to simplify command execution: + +- `helpers.Ensure(args ...string)` will run a command and ensure it exits succesfully +- `helpers.Fail(args ...string)` will run a command and ensure it fails +- `helpers.Anyhow(args ...string)` will run a command but does not care if it succeeds or fails +- `helpers.Capture(args ...string)` will run a command, ensure it is successful, and return the output +- `helpers.Command(args ...string)` will return a command that can then be tested against expectations +- `helpers.CustomCommand(binary string, args ...string)` will do the same for any arbitrary command (not limited to nerdctl) + +## Setup and Cleanup + +Tests routinely require a set of actions to be performed _before_ you can run the +command you want to test. +A setup routine will get executed before your `Command`, and have access to and can +manipulate your test `Data`. + +Conversely, you very likely want to clean things up once your test is done. +While temporary directories are cleaned for you with no action needed on your part, +the app you are testing might have stateful data you may want to remove. +Note that a `Cleanup` routine will get executed twice - after your `Command` has run +its course evidently - but also, pre-emptively, before your `Setup`, so that possible leftovers from +previous runs are taken care of. + +```go +package main + +import ( + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/errdefs" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A subtest with custom data, manager, and comparator", + Data: test.WithData("sometestdata", "blah"), + Setup: func(data *test.Data, helpers test.Helpers){ + helpers.Ensure("volume", "create", "foo") + helpers.Ensure("volume", "create", "bar") + }, + Cleanup: func(data *test.Data, helpers test.Helpers){ + helpers.Anyhow("volume", "rm", "foo") + helpers.Anyhow("volume", "rm", "bar") + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("run", "--name", data.Identifier()+data.Get("sometestdata")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{ + errors.New("foobla"), + errdefs.ErrNotFound, + }, + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, stdout == data.Get("sometestdata"), info) + }, + } + }, + } + + myTest.Run(t) +} +``` + +## Subtests + +Subtests are just regular tests, attached to the `SubTests` slice of a test. + +Note that a subtest will inherit its parent `Data` and `Env`, in the state they are at +after the parent test has run its `Setup` and `Command` routines (but before `Cleanup`). +This does _not_ apply to `Identifier()` and `TempDir()`, which are unique to the sub-test. + +Also note that a test does not have to have a `Command`. +This is a convenient pattern if you just need a common `Setup` for a bunch of subtests. + +## Groups + +A `test.Group` is just a convenient way to represent a slice of tests. + +Note that unlike a `test.Case`, a group cannot define properties inherited by +subtests, nor `Setup` or `Cleanup` routines. + +- if you just have a bunch of subtests you want to run, put them in a `test.Group` +- if you want to have a global setup, or otherwise set a common property first for your subtests, use a `test.Case` with `SubTests` + +## Parallelism + +All tests (and subtests) are assumed to be parallelizable. + +You can force a specific `test.Case` to not be run in parallel though, +by setting its `NoParallel` property to `true`. + +Note that if you want better isolation, it is usually better to use +`WithConfig(nerdtest.Mode, nerdtest.Private)` instead. +This will keep the test parallel (for nerdctl), but isolate it in a different context. +For Docker (which does not support namespaces), it is equivalent to passing `NoParallel: true`.