From 96d187652489019ceedd848720b9089e9c2f685d Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Thu, 24 Jan 2019 11:22:17 +0100 Subject: [PATCH 1/7] virtcontainers: hypervisor: Minimal package This is the minimal package to help us define the hypervisor interface into its own package. Fixes: #1229 Signed-off-by: Samuel Ortiz --- cli/factory_test.go | 7 +- cli/kata-check.go | 4 +- cli/kata-check_amd64.go | 4 +- cli/kata-env.go | 3 +- cli/kata-env_test.go | 3 +- cli/list.go | 4 +- cli/main_test.go | 9 +- containerd-shim-v2/utils_test.go | 9 +- pkg/katautils/config.go | 55 ++-- pkg/katautils/config_test.go | 16 +- pkg/katautils/create.go | 9 +- pkg/katautils/create_test.go | 15 +- virtcontainers/agent.go | 3 +- virtcontainers/api_test.go | 65 ++--- virtcontainers/bridgedmacvlan_endpoint.go | 9 +- virtcontainers/cgroups.go | 8 +- virtcontainers/container.go | 4 +- virtcontainers/container_test.go | 3 +- virtcontainers/endpoint.go | 8 +- virtcontainers/example_pod_run_test.go | 5 +- virtcontainers/factory/cache/cache_test.go | 3 +- virtcontainers/factory/direct/direct_test.go | 3 +- virtcontainers/factory/factory_test.go | 21 +- .../factory/template/template_test.go | 3 +- virtcontainers/fc.go | 25 +- virtcontainers/hack/virtc/main.go | 9 +- virtcontainers/hyperstart_agent.go | 9 +- virtcontainers/{ => hypervisor}/hypervisor.go | 249 ++++++++++-------- .../{ => hypervisor}/hypervisor_test.go | 154 ++++++----- virtcontainers/iostream_test.go | 3 +- virtcontainers/ipvlan_endpoint.go | 9 +- virtcontainers/kata_agent.go | 15 +- virtcontainers/kata_agent_test.go | 7 +- virtcontainers/macvtap_endpoint.go | 16 +- virtcontainers/mock_hypervisor.go | 53 ++-- virtcontainers/mock_hypervisor_test.go | 22 +- virtcontainers/monitor_test.go | 5 +- virtcontainers/network.go | 19 +- virtcontainers/noop_agent.go | 3 +- virtcontainers/physical_endpoint.go | 9 +- virtcontainers/pkg/oci/utils.go | 7 +- virtcontainers/pkg/oci/utils_test.go | 9 +- virtcontainers/pkg/vcmock/mock_test.go | 5 +- virtcontainers/qemu.go | 155 ++++++----- virtcontainers/qemu_amd64.go | 7 +- virtcontainers/qemu_amd64_test.go | 3 +- virtcontainers/qemu_arch_base.go | 23 +- virtcontainers/qemu_arch_base_test.go | 19 +- virtcontainers/qemu_test.go | 51 ++-- virtcontainers/sandbox.go | 51 ++-- virtcontainers/sandbox_test.go | 67 ++--- virtcontainers/tap_endpoint.go | 13 +- virtcontainers/veth_endpoint.go | 13 +- virtcontainers/vhostuser_endpoint.go | 9 +- virtcontainers/vm.go | 40 +-- virtcontainers/vm_test.go | 9 +- 56 files changed, 722 insertions(+), 639 deletions(-) rename virtcontainers/{ => hypervisor}/hypervisor.go (70%) rename virtcontainers/{ => hypervisor}/hypervisor_test.go (71%) diff --git a/cli/factory_test.go b/cli/factory_test.go index faf5028d73..cfe194e0d1 100644 --- a/cli/factory_test.go +++ b/cli/factory_test.go @@ -15,6 +15,7 @@ import ( "github.com/urfave/cli" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestFactoryCLIFunctionNoRuntimeConfig(t *testing.T) { @@ -64,7 +65,7 @@ func TestFactoryCLIFunctionInit(t *testing.T) { // With template runtimeConfig.FactoryConfig.Template = true - runtimeConfig.HypervisorType = vc.MockHypervisor + runtimeConfig.HypervisorType = hypervisor.Mock runtimeConfig.AgentType = vc.NoopAgentType ctx.App.Metadata["runtimeConfig"] = runtimeConfig fn, ok = initFactoryCommand.Action.(func(context *cli.Context) error) @@ -99,7 +100,7 @@ func TestFactoryCLIFunctionDestroy(t *testing.T) { // With template runtimeConfig.FactoryConfig.Template = true - runtimeConfig.HypervisorType = vc.MockHypervisor + runtimeConfig.HypervisorType = hypervisor.Mock runtimeConfig.AgentType = vc.NoopAgentType ctx.App.Metadata["runtimeConfig"] = runtimeConfig fn, ok = destroyFactoryCommand.Action.(func(context *cli.Context) error) @@ -135,7 +136,7 @@ func TestFactoryCLIFunctionStatus(t *testing.T) { // With template runtimeConfig.FactoryConfig.Template = true - runtimeConfig.HypervisorType = vc.MockHypervisor + runtimeConfig.HypervisorType = hypervisor.Mock runtimeConfig.AgentType = vc.NoopAgentType ctx.App.Metadata["runtimeConfig"] = runtimeConfig err = fn(ctx) diff --git a/cli/kata-check.go b/cli/kata-check.go index d47328867c..187c3253c8 100644 --- a/cli/kata-check.go +++ b/cli/kata-check.go @@ -24,7 +24,7 @@ import ( "syscall" "github.com/kata-containers/runtime/pkg/katautils" - vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -186,7 +186,7 @@ type kernelParamHandler func(onVMM bool, fields logrus.Fields, msg string) bool // the number of module errors (all of which are logged by this // function). Only fatal errors result in an error return. func checkKernelModules(modules map[string]kernelModule, handler kernelParamHandler) (count uint32, err error) { - onVMM, err := vc.RunningOnVMM(procCPUInfo) + onVMM, err := vcHypervisor.RunningOnVMM(procCPUInfo) if err != nil { return 0, err } diff --git a/cli/kata-check_amd64.go b/cli/kata-check_amd64.go index 2aa34f34e9..f8cfec2805 100644 --- a/cli/kata-check_amd64.go +++ b/cli/kata-check_amd64.go @@ -11,7 +11,7 @@ import ( "io/ioutil" "strings" - vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) const ( @@ -55,7 +55,7 @@ func setCPUtype() error { return fmt.Errorf("Unknow CPU Type") } else if cpuType == cpuTypeIntel { var kvmIntelParams map[string]string - onVMM, err := vc.RunningOnVMM(procCPUInfo) + onVMM, err := hypervisor.RunningOnVMM(procCPUInfo) if err != nil && !onVMM { kvmIntelParams = map[string]string{ // "VMX Unrestricted mode support". This is used diff --git a/cli/kata-env.go b/cli/kata-env.go index 8bf2d05760..0e5d436af8 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -16,6 +16,7 @@ import ( "github.com/BurntSushi/toml" "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" vcUtils "github.com/kata-containers/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -368,7 +369,7 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e kernel := KernelInfo{ Path: config.HypervisorConfig.KernelPath, - Parameters: strings.Join(vc.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), + Parameters: strings.Join(vcHypervisor.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), } initrd := InitrdInfo{ diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go index a3b6a5970a..47672ae824 100644 --- a/cli/kata-env_test.go +++ b/cli/kata-env_test.go @@ -20,6 +20,7 @@ import ( "github.com/BurntSushi/toml" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcUtils "github.com/kata-containers/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/urfave/cli" @@ -344,7 +345,7 @@ func getExpectedImage(config oci.RuntimeConfig) ImageInfo { func getExpectedKernel(config oci.RuntimeConfig) KernelInfo { return KernelInfo{ Path: config.HypervisorConfig.KernelPath, - Parameters: strings.Join(vc.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), + Parameters: strings.Join(hypervisor.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), } } diff --git a/cli/list.go b/cli/list.go index 26dc7391b3..5f6a4a26bb 100644 --- a/cli/list.go +++ b/cli/list.go @@ -20,7 +20,7 @@ import ( "github.com/urfave/cli" "github.com/kata-containers/runtime/pkg/katautils" - vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" oci "github.com/kata-containers/runtime/virtcontainers/pkg/oci" ) @@ -360,7 +360,7 @@ func getContainers(ctx context.Context, context *cli.Context) ([]fullContainerSt // getHypervisorDetails returns details of the latest version of the // hypervisor and the associated assets. -func getHypervisorDetails(hypervisorConfig *vc.HypervisorConfig) hypervisorDetails { +func getHypervisorDetails(hypervisorConfig *vcHypervisor.Config) hypervisorDetails { hypervisorPath, err := hypervisorConfig.HypervisorAssetPath() if err != nil { hypervisorPath = hypervisorConfig.HypervisorPath diff --git a/cli/main_test.go b/cli/main_test.go index f9a65fce81..56d6a941fc 100644 --- a/cli/main_test.go +++ b/cli/main_test.go @@ -24,6 +24,7 @@ import ( "github.com/dlespiau/covertool/pkg/cover" "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" "github.com/kata-containers/runtime/virtcontainers/types" @@ -206,7 +207,7 @@ func grep(pattern, file string) error { // // Note: no parameter validation in case caller wishes to create an invalid // object. -func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) { +func newTestHypervisorConfig(dir string, create bool) (hypervisor.Config, error) { kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") hypervisorPath := path.Join(dir, "hypervisor") @@ -215,12 +216,12 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro for _, file := range []string{kernelPath, imagePath, hypervisorPath} { err := createEmptyFile(file) if err != nil { - return vc.HypervisorConfig{}, err + return hypervisor.Config{}, err } } } - return vc.HypervisorConfig{ + return hypervisor.Config{ KernelPath: kernelPath, ImagePath: imagePath, HypervisorPath: hypervisorPath, @@ -240,7 +241,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf } return oci.RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.KataContainersAgent, ProxyType: vc.CCProxyType, diff --git a/containerd-shim-v2/utils_test.go b/containerd-shim-v2/utils_test.go index 4ae413ef93..cf0f75a8c5 100644 --- a/containerd-shim-v2/utils_test.go +++ b/containerd-shim-v2/utils_test.go @@ -19,6 +19,7 @@ import ( "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" ) @@ -160,7 +161,7 @@ func createEmptyFile(path string) (err error) { // // Note: no parameter validation in case caller wishes to create an invalid // object. -func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) { +func newTestHypervisorConfig(dir string, create bool) (hypervisor.Config, error) { kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") hypervisorPath := path.Join(dir, "hypervisor") @@ -169,12 +170,12 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro for _, file := range []string{kernelPath, imagePath, hypervisorPath} { err := createEmptyFile(file) if err != nil { - return vc.HypervisorConfig{}, err + return hypervisor.Config{}, err } } } - return vc.HypervisorConfig{ + return hypervisor.Config{ KernelPath: kernelPath, ImagePath: imagePath, HypervisorPath: hypervisorPath, @@ -194,7 +195,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf } return oci.RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.KataContainersAgent, ProxyType: vc.KataBuiltInProxyType, diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index b286ef1fe2..f4ab1206cf 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -16,13 +16,14 @@ import ( "github.com/BurntSushi/toml" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/config" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" ) const ( - defaultHypervisor = vc.QemuHypervisor + defaultHypervisor = vcHypervisor.Qemu defaultAgent = vc.KataContainersAgent ) @@ -401,45 +402,45 @@ func (n netmon) debug() bool { return n.Debug } -func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { +func newFirecrackerHypervisorConfig(h hypervisor) (vcHypervisor.Config, error) { hypervisor, err := h.path() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } kernel, err := h.kernel() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } initrd, image, err := h.getInitrdAndImage() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } firmware, err := h.firmware() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } kernelParams := h.kernelParams() blockDriver, err := h.blockDeviceDriver() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } if !utils.SupportsVsocks() { - return vc.HypervisorConfig{}, errors.New("No vsock support, firecracker cannot be used") + return vcHypervisor.Config{}, errors.New("No vsock support, firecracker cannot be used") } - return vc.HypervisorConfig{ + return vcHypervisor.Config{ HypervisorPath: hypervisor, KernelPath: kernel, InitrdPath: initrd, ImagePath: image, FirmwarePath: firmware, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + KernelParams: vcHypervisor.DeserializeParams(strings.Fields(kernelParams)), NumVCPUs: h.defaultVCPUs(), DefaultMaxVCPUs: h.defaultMaxVCPUs(), MemorySize: h.defaultMemSz(), @@ -458,35 +459,35 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } -func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { +func newQemuHypervisorConfig(h hypervisor) (vcHypervisor.Config, error) { hypervisor, err := h.path() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } kernel, err := h.kernel() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } initrd, image, err := h.getInitrdAndImage() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } if image != "" && initrd != "" { - return vc.HypervisorConfig{}, + return vcHypervisor.Config{}, errors.New("having both an image and an initrd defined in the configuration file is not supported") } if image == "" && initrd == "" { - return vc.HypervisorConfig{}, + return vcHypervisor.Config{}, errors.New("either image or initrd must be defined in the configuration file") } firmware, err := h.firmware() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } machineAccelerators := h.machineAccelerators() @@ -495,7 +496,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { blockDriver, err := h.blockDeviceDriver() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } useVSock := false @@ -508,14 +509,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { } } - return vc.HypervisorConfig{ + return vcHypervisor.Config{ HypervisorPath: hypervisor, KernelPath: kernel, InitrdPath: initrd, ImagePath: image, FirmwarePath: firmware, MachineAccelerators: machineAccelerators, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + KernelParams: vcHypervisor.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, NumVCPUs: h.defaultVCPUs(), DefaultMaxVCPUs: h.defaultMaxVCPUs(), @@ -563,14 +564,14 @@ func newShimConfig(s shim) (vc.ShimConfig, error) { func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig) error { for k, hypervisor := range tomlConf.Hypervisor { var err error - var hConfig vc.HypervisorConfig + var hConfig vcHypervisor.Config switch k { case firecrackerHypervisorTableType: - config.HypervisorType = vc.FirecrackerHypervisor + config.HypervisorType = vcHypervisor.Firecracker hConfig, err = newFirecrackerHypervisorConfig(hypervisor) case qemuHypervisorTableType: - config.HypervisorType = vc.QemuHypervisor + config.HypervisorType = vcHypervisor.Qemu hConfig, err = newQemuHypervisorConfig(hypervisor) } @@ -645,7 +646,7 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error { defaultKernelParams := GetKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig)) if runtimeConfig.HypervisorConfig.Debug { - strParams := vc.SerializeParams(defaultKernelParams, "=") + strParams := vcHypervisor.SerializeParams(defaultKernelParams, "=") formatted := strings.Join(strParams, " ") kataUtilsLogger.WithField("default-kernel-parameters", formatted).Debug() @@ -655,7 +656,7 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error { userKernelParams := runtimeConfig.HypervisorConfig.KernelParams // reset - runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{} + runtimeConfig.HypervisorConfig.KernelParams = []vcHypervisor.Param{} // first, add default values for _, p := range defaultKernelParams { @@ -714,7 +715,7 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run func initConfig() (config oci.RuntimeConfig, err error) { var defaultAgentConfig interface{} - defaultHypervisorConfig := vc.HypervisorConfig{ + defaultHypervisorConfig := vcHypervisor.Config{ HypervisorPath: defaultHypervisorPath, KernelPath: defaultKernelPath, ImagePath: defaultImagePath, @@ -923,7 +924,7 @@ func checkFactoryConfig(config oci.RuntimeConfig) error { // checkHypervisorConfig performs basic "sanity checks" on the hypervisor // config. -func checkHypervisorConfig(config vc.HypervisorConfig) error { +func checkHypervisorConfig(config vcHypervisor.Config) error { type image struct { path string initrd bool diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index 12bac15979..ac82b950eb 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -21,6 +21,7 @@ import ( "testing" vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/stretchr/testify/assert" @@ -84,7 +85,6 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath } func createConfig(configPath string, fileData string) error { - err := ioutil.WriteFile(configPath, []byte(fileData), testFileMode) if err != nil { fmt.Fprintf(os.Stderr, "Unable to create config file %s %v\n", configPath, err) @@ -147,11 +147,11 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf } } - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := vcHypervisor.Config{ HypervisorPath: hypervisorPath, KernelPath: kernelPath, ImagePath: imagePath, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + KernelParams: vcHypervisor.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, NumVCPUs: defaultVCPUCount, DefaultMaxVCPUs: uint32(goruntime.NumCPU()), @@ -593,7 +593,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { t.Fatal(err) } - expectedHypervisorConfig := vc.HypervisorConfig{ + expectedHypervisorConfig := vcHypervisor.Config{ HypervisorPath: defaultHypervisorPath, KernelPath: defaultKernelPath, ImagePath: defaultImagePath, @@ -1401,8 +1401,8 @@ func TestUpdateRuntimeConfigurationInvalidKernelParams(t *testing.T) { GetKernelParamsFunc = savedFunc }() - GetKernelParamsFunc = func(needSystemd bool) []vc.Param { - return []vc.Param{ + GetKernelParamsFunc = func(needSystemd bool) []vcHypervisor.Param { + return []vcHypervisor.Param{ { Key: "", Value: "", @@ -1493,7 +1493,7 @@ func TestCheckHypervisorConfig(t *testing.T) { // capture output to buffer kataUtilsLogger.Logger.Out = logBuf - config := vc.HypervisorConfig{ + config := vcHypervisor.Config{ ImagePath: d.imagePath, InitrdPath: d.initrdPath, MemorySize: d.memBytes, @@ -1559,7 +1559,7 @@ func TestCheckFactoryConfig(t *testing.T) { for i, d := range data { config := oci.RuntimeConfig{ - HypervisorConfig: vc.HypervisorConfig{ + HypervisorConfig: vcHypervisor.Config{ ImagePath: d.imagePath, InitrdPath: d.initrdPath, }, diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go index 0e07e8003c..ad29676ca2 100644 --- a/pkg/katautils/create.go +++ b/pkg/katautils/create.go @@ -12,13 +12,14 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" vf "github.com/kata-containers/runtime/virtcontainers/factory" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" ) // GetKernelParamsFunc use a variable to allow tests to modify its value var GetKernelParamsFunc = getKernelParams -var systemdKernelParam = []vc.Param{ +var systemdKernelParam = []vcHypervisor.Param{ { Key: "init", Value: "/usr/lib/systemd/systemd", @@ -37,8 +38,8 @@ var systemdKernelParam = []vc.Param{ }, } -func getKernelParams(needSystemd bool) []vc.Param { - p := []vc.Param{} +func getKernelParams(needSystemd bool) []vcHypervisor.Param { + p := []vcHypervisor.Param{} if needSystemd { p = append(p, systemdKernelParam...) @@ -47,7 +48,7 @@ func getKernelParams(needSystemd bool) []vc.Param { return p } -func needSystemd(config vc.HypervisorConfig) bool { +func needSystemd(config vcHypervisor.Config) bool { return config.ImagePath != "" } diff --git a/pkg/katautils/create_test.go b/pkg/katautils/create_test.go index 77701838e9..e72d286833 100644 --- a/pkg/katautils/create_test.go +++ b/pkg/katautils/create_test.go @@ -19,6 +19,7 @@ import ( "testing" vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" "github.com/opencontainers/runtime-spec/specs-go" @@ -117,7 +118,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf } return oci.RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: vcHypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.KataContainersAgent, ProxyType: vc.CCProxyType, @@ -132,7 +133,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf // // Note: no parameter validation in case caller wishes to create an invalid // object. -func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) { +func newTestHypervisorConfig(dir string, create bool) (vcHypervisor.Config, error) { kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") hypervisorPath := path.Join(dir, "hypervisor") @@ -141,12 +142,12 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro for _, file := range []string{kernelPath, imagePath, hypervisorPath} { err := createEmptyFile(file) if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } } } - return vc.HypervisorConfig{ + return vcHypervisor.Config{ KernelPath: kernelPath, ImagePath: imagePath, HypervisorPath: hypervisorPath, @@ -155,7 +156,7 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro } // return the value of the *last* param with the specified key -func findLastParam(key string, params []vc.Param) (string, error) { +func findLastParam(key string, params []vcHypervisor.Param) (string, error) { if key == "" { return "", errors.New("ERROR: need non-nil key") } @@ -218,12 +219,12 @@ func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) { ipName := "ip" ipValue := "127.0.0.1" - params := []vc.Param{ + params := []vcHypervisor.Param{ {Key: initName, Value: initValue}, {Key: ipName, Value: ipValue}, } - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := vcHypervisor.Config{ KernelParams: params, } diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go index 9cfce0f349..8964af18af 100644 --- a/virtcontainers/agent.go +++ b/virtcontainers/agent.go @@ -11,6 +11,7 @@ import ( "time" "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/mitchellh/mapstructure" @@ -223,7 +224,7 @@ type agent interface { resumeContainer(sandbox *Sandbox, c Container) error // configure will update agent settings based on provided arguments - configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error + configure(h hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error // getVMPath will return the agent vm socket's directory path getVMPath(id string) string diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go index ec19e72acc..8b869a5803 100644 --- a/virtcontainers/api_test.go +++ b/virtcontainers/api_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/mock" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/store" @@ -67,7 +68,7 @@ func newTestSandboxConfigNoop() SandboxConfig { } // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -75,7 +76,7 @@ func newTestSandboxConfigNoop() SandboxConfig { sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: NoopAgentType, @@ -98,7 +99,7 @@ func newTestSandboxConfigHyperstartAgent() SandboxConfig { } // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -111,7 +112,7 @@ func newTestSandboxConfigHyperstartAgent() SandboxConfig { sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: HyperstartAgent, @@ -134,7 +135,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { } // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -150,7 +151,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: HyperstartAgent, @@ -167,7 +168,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { func newTestSandboxConfigKataAgent() SandboxConfig { // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -175,7 +176,7 @@ func newTestSandboxConfigKataAgent() SandboxConfig { sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: KataContainersAgent, @@ -851,16 +852,16 @@ func TestStatusSandboxSuccessfulStateReady(t *testing.T) { cleanUp() config := newTestSandboxConfigNoop() - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: hypervisor.DefaultVCPUs, + MemorySize: hypervisor.DefaultMemSzMiB, + DefaultBridges: hypervisor.DefaultBridges, + BlockDeviceDriver: hypervisor.DefaultBlockDriver, + DefaultMaxVCPUs: hypervisor.DefaultMaxQemuVCPUs, + Msize9p: hypervisor.DefaultMsize9p, } expectedStatus := SandboxStatus{ @@ -868,7 +869,7 @@ func TestStatusSandboxSuccessfulStateReady(t *testing.T) { State: types.State{ State: types.StateReady, }, - Hypervisor: MockHypervisor, + Hypervisor: hypervisor.Mock, HypervisorConfig: hypervisorConfig, Agent: NoopAgentType, Annotations: sandboxAnnotations, @@ -909,16 +910,16 @@ func TestStatusSandboxSuccessfulStateRunning(t *testing.T) { cleanUp() config := newTestSandboxConfigNoop() - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: hypervisor.DefaultVCPUs, + MemorySize: hypervisor.DefaultMemSzMiB, + DefaultBridges: hypervisor.DefaultBridges, + BlockDeviceDriver: hypervisor.DefaultBlockDriver, + DefaultMaxVCPUs: hypervisor.DefaultMaxQemuVCPUs, + Msize9p: hypervisor.DefaultMsize9p, } expectedStatus := SandboxStatus{ @@ -926,7 +927,7 @@ func TestStatusSandboxSuccessfulStateRunning(t *testing.T) { State: types.State{ State: types.StateRunning, }, - Hypervisor: MockHypervisor, + Hypervisor: hypervisor.Mock, HypervisorConfig: hypervisorConfig, Agent: NoopAgentType, Annotations: sandboxAnnotations, @@ -1993,8 +1994,8 @@ func TestProcessListContainer(t *testing.T) { * Benchmarks */ -func createNewSandboxConfig(hType HypervisorType, aType AgentType, aConfig interface{}) SandboxConfig { - hypervisorConfig := HypervisorConfig{ +func createNewSandboxConfig(hType hypervisor.Type, aType AgentType, aConfig interface{}) SandboxConfig { + hypervisorConfig := hypervisor.Config{ KernelPath: "/usr/share/kata-containers/vmlinux.container", ImagePath: "/usr/share/kata-containers/kata-containers.img", HypervisorPath: "/usr/bin/qemu-system-x86_64", @@ -2159,7 +2160,7 @@ func createStartStopDeleteContainers(b *testing.B, sandboxConfig SandboxConfig, func BenchmarkCreateStartStopDeleteSandboxQemuHypervisorHyperstartAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, HyperstartAgent, HyperConfig{}) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, HyperstartAgent, HyperConfig{}) sockDir, err := testGenerateCCProxySockDir() if err != nil { @@ -2180,21 +2181,21 @@ func BenchmarkCreateStartStopDeleteSandboxQemuHypervisorHyperstartAgentNetworkNo func BenchmarkCreateStartStopDeleteSandboxQemuHypervisorNoopAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, NoopAgentType, nil) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, NoopAgentType, nil) createStartStopDeleteSandbox(b, sandboxConfig) } } -func BenchmarkCreateStartStopDeleteSandboxMockHypervisorNoopAgentNetworkNoop(b *testing.B) { +func BenchmarkCreateStartStopDeleteSandboxHypervisorMockNoopAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(MockHypervisor, NoopAgentType, nil) + sandboxConfig := createNewSandboxConfig(hypervisor.Mock, NoopAgentType, nil) createStartStopDeleteSandbox(b, sandboxConfig) } } func BenchmarkStartStop1ContainerQemuHypervisorHyperstartAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, HyperstartAgent, HyperConfig{}) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, HyperstartAgent, HyperConfig{}) contConfigs := createNewContainerConfigs(1) sockDir, err := testGenerateCCProxySockDir() @@ -2216,7 +2217,7 @@ func BenchmarkStartStop1ContainerQemuHypervisorHyperstartAgentNetworkNoop(b *tes func BenchmarkStartStop10ContainerQemuHypervisorHyperstartAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, HyperstartAgent, HyperConfig{}) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, HyperstartAgent, HyperConfig{}) contConfigs := createNewContainerConfigs(10) sockDir, err := testGenerateCCProxySockDir() diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go index d2efd56f53..aa7e862ef3 100644 --- a/virtcontainers/bridgedmacvlan_endpoint.go +++ b/virtcontainers/bridgedmacvlan_endpoint.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) // BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM @@ -83,13 +84,13 @@ func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair { // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error { +func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor.Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, hypervisor.NetDev) } // Detach for the virtual endpoint tears down the tap and bridge @@ -107,11 +108,11 @@ func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath stri } // HotAttach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error { +func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor.Hypervisor) error { return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") } diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go index 20fa6b07bc..fd11a9a8f8 100644 --- a/virtcontainers/cgroups.go +++ b/virtcontainers/cgroups.go @@ -99,7 +99,7 @@ func (s *Sandbox) applyCPUCgroup(rc *specs.LinuxResources) error { // when new container joins, new CPU could be hotplugged, so we // have to query fresh vcpu info from hypervisor for every time. - tids, err := s.hypervisor.getThreadIDs() + tids, err := s.hypervisor.GetThreadIDs() if err != nil { return fmt.Errorf("failed to get thread ids from hypervisor: %v", err) } @@ -113,15 +113,15 @@ func (s *Sandbox) applyCPUCgroup(rc *specs.LinuxResources) error { // use Add() to add vcpu thread to s.cgroup, it will write thread id to // `cgroup.procs` which will move all threads in qemu process to this cgroup // immediately as default behaviour. - if len(tids.vcpus) > 0 { + if len(tids.VCPUs) > 0 { if err := s.cgroup.sandboxSub.Add(cgroups.Process{ - Pid: tids.vcpus[0], + Pid: tids.VCPUs[0], }); err != nil { return err } } - for _, i := range tids.vcpus { + for _, i := range tids.VCPUs { if i <= 0 { continue } diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 7952dfbd61..887ede7ad3 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -431,7 +431,7 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s // copy file to contaier's rootfs if filesystem sharing is not supported, otherwise // bind mount it in the shared directory. - caps := c.sandbox.hypervisor.capabilities() + caps := c.sandbox.hypervisor.Capabilities() if !caps.IsFsSharingSupported() { c.Logger().Debug("filesystem sharing is not supported, files will be copied") @@ -713,7 +713,7 @@ func (c *Container) rollbackFailingContainerCreation() { func (c *Container) checkBlockDeviceSupport() bool { if !c.sandbox.config.HypervisorConfig.DisableBlockDeviceUse { agentCaps := c.sandbox.agent.capabilities() - hypervisorCaps := c.sandbox.hypervisor.capabilities() + hypervisorCaps := c.sandbox.hypervisor.Capabilities() if agentCaps.IsBlockDeviceSupported() && hypervisorCaps.IsBlockDeviceHotplugSupported() { return true diff --git a/virtcontainers/container_test.go b/virtcontainers/container_test.go index e97d699215..badaed9fce 100644 --- a/virtcontainers/container_test.go +++ b/virtcontainers/container_test.go @@ -20,6 +20,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" @@ -220,7 +221,7 @@ func TestContainerAddDriveDir(t *testing.T) { hypervisor: &mockHypervisor{}, agent: &noopAgent{}, config: &SandboxConfig{ - HypervisorConfig: HypervisorConfig{ + HypervisorConfig: hypervisor.Config{ DisableBlockDeviceUse: false, }, }, diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go index 978da2dc10..2819832cf6 100644 --- a/virtcontainers/endpoint.go +++ b/virtcontainers/endpoint.go @@ -7,6 +7,8 @@ package virtcontainers import ( "fmt" + + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) // Endpoint represents a physical or virtual network interface. @@ -20,10 +22,10 @@ type Endpoint interface { SetProperties(NetworkInfo) SetPciAddr(string) - Attach(hypervisor) error + Attach(hypervisor.Hypervisor) error Detach(netNsCreated bool, netNsPath string) error - HotAttach(h hypervisor) error - HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error + HotAttach(hypervisor.Hypervisor) error + HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error } // EndpointType identifies the type of the network endpoint. diff --git a/virtcontainers/example_pod_run_test.go b/virtcontainers/example_pod_run_test.go index 5b93479079..d3844605dd 100644 --- a/virtcontainers/example_pod_run_test.go +++ b/virtcontainers/example_pod_run_test.go @@ -11,6 +11,7 @@ import ( "strings" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -40,7 +41,7 @@ func Example_createAndStartSandbox() { } // Sets the hypervisor configuration. - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: "/usr/share/kata-containers/vmlinux.container", ImagePath: "/usr/share/kata-containers/kata-containers.img", HypervisorPath: "/usr/bin/qemu-system-x86_64", @@ -55,7 +56,7 @@ func Example_createAndStartSandbox() { // - Hypervisor is QEMU // - Agent is hyperstart sandboxConfig := vc.SandboxConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.HyperstartAgent, diff --git a/virtcontainers/factory/cache/cache_test.go b/virtcontainers/factory/cache/cache_test.go index 2189360d8e..18bff0e9d9 100644 --- a/virtcontainers/factory/cache/cache_test.go +++ b/virtcontainers/factory/cache/cache_test.go @@ -14,13 +14,14 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory/direct" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestTemplateFactory(t *testing.T) { assert := assert.New(t) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } diff --git a/virtcontainers/factory/direct/direct_test.go b/virtcontainers/factory/direct/direct_test.go index cca51cea97..0396dea734 100644 --- a/virtcontainers/factory/direct/direct_test.go +++ b/virtcontainers/factory/direct/direct_test.go @@ -13,13 +13,14 @@ import ( "github.com/stretchr/testify/assert" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestTemplateFactory(t *testing.T) { assert := assert.New(t) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } diff --git a/virtcontainers/factory/factory_test.go b/virtcontainers/factory/factory_test.go index deaa02c20e..9a43965dd1 100644 --- a/virtcontainers/factory/factory_test.go +++ b/virtcontainers/factory/factory_test.go @@ -15,6 +15,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory/base" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestNewFactory(t *testing.T) { @@ -39,7 +40,7 @@ func TestNewFactory(t *testing.T) { testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + config.VMConfig.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } @@ -84,7 +85,7 @@ func TestFactorySetLogger(t *testing.T) { SetLogger(context.Background(), testLog) var config Config - config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + config.VMConfig.HypervisorConfig = hypervisor.Config{ KernelPath: "foo", ImagePath: "bar", } @@ -105,7 +106,7 @@ func TestVMConfigValid(t *testing.T) { config := vc.VMConfig{ HypervisorType: vc.MockHypervisor, - HypervisorConfig: vc.HypervisorConfig{ + HypervisorConfig: hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, }, @@ -151,14 +152,14 @@ func TestCheckVMConfig(t *testing.T) { assert.Nil(err) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config1.HypervisorConfig = vc.HypervisorConfig{ + config1.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } err = checkVMConfig(config1, config2) assert.Error(err) - config2.HypervisorConfig = vc.HypervisorConfig{ + config2.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } @@ -170,7 +171,7 @@ func TestFactoryGetVM(t *testing.T) { assert := assert.New(t) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } @@ -287,11 +288,11 @@ func TestDeepCompare(t *testing.T) { assert.True(deepCompare(foo, bar)) // slice - foo.HypervisorConfig.KernelParams = []vc.Param{} + foo.HypervisorConfig.KernelParams = []hypervisor.Param{} assert.True(deepCompare(foo, bar)) - foo.HypervisorConfig.KernelParams = append(foo.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"}) + foo.HypervisorConfig.KernelParams = append(foo.HypervisorConfig.KernelParams, hypervisor.Param{Key: "key", Value: "value"}) assert.False(deepCompare(foo, bar)) - bar.HypervisorConfig.KernelParams = append(bar.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"}) + bar.HypervisorConfig.KernelParams = append(bar.HypervisorConfig.KernelParams, hypervisor.Param{Key: "key", Value: "value"}) assert.True(deepCompare(foo, bar)) // map @@ -325,7 +326,7 @@ func TestDeepCompare(t *testing.T) { ProxyType: vc.NoopProxyType, } testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + config.VMConfig.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } diff --git a/virtcontainers/factory/template/template_test.go b/virtcontainers/factory/template/template_test.go index d08a2ab69e..d042aa700b 100644 --- a/virtcontainers/factory/template/template_test.go +++ b/virtcontainers/factory/template/template_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestTemplateFactory(t *testing.T) { @@ -23,7 +24,7 @@ func TestTemplateFactory(t *testing.T) { templateWaitForAgent = 1 * time.Microsecond testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 9111fff653..12058c422e 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -22,6 +22,7 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -100,14 +101,14 @@ type firecracker struct { socketPath string store *store.VCStore - config HypervisorConfig + config hypervisor.Config pendingDevices []firecrackerDevice // Devices to be added when the FC API is ready ctx context.Context } type firecrackerDevice struct { dev interface{} - devType deviceType + devType hypervisor.Device } // Logger returns a logrus logger appropriate for logging firecracker messages @@ -131,7 +132,7 @@ func (fc *firecracker) trace(name string) (opentracing.Span, context.Context) { // For firecracker this call only sets the internal structure up. // The sandbox will be created and started through startSandbox(). -func (fc *firecracker) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (fc *firecracker) createSandbox(ctx context.Context, id string, hypervisorConfig *hypervisor.Config, vcStore *store.VCStore) error { fc.ctx = ctx span, _ := fc.trace("createSandbox") @@ -352,7 +353,7 @@ func (fc *firecracker) startSandbox(timeout int) error { return err } - strParams := SerializeParams(fc.config.KernelParams, "=") + strParams := hypervisor.SerializeParams(fc.config.KernelParams, "=") formattedParams := strings.Join(strParams, " ") fc.fcSetBootSource(kernelPath, formattedParams) @@ -594,7 +595,7 @@ func (fc *firecracker) fcUpdateBlockDrive(drive config.BlockDrive) error { // addDevice will add extra devices to firecracker. Limited to configure before the // virtual machine starts. Devices include drivers and network interfaces only. -func (fc *firecracker) addDevice(devInfo interface{}, devType deviceType) error { +func (fc *firecracker) addDevice(devInfo interface{}, devType hypervisor.Device) error { span, _ := fc.trace("addDevice") defer span.Finish() @@ -630,24 +631,24 @@ func (fc *firecracker) addDevice(devInfo interface{}, devType deviceType) error } // hotplugAddDevice supported in Firecracker VMM -func (fc *firecracker) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (fc *firecracker) hotplugAddDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { span, _ := fc.trace("hotplugAddDevice") defer span.Finish() switch devType { - case blockDev: + case hypervisor.BlockDev: //The drive placeholder has to exist prior to Update return nil, fc.fcUpdateBlockDrive(*devInfo.(*config.BlockDrive)) default: fc.Logger().WithFields(logrus.Fields{"devInfo": devInfo, - "deviceType": devType}).Warn("hotplugAddDevice: unsupported device") - return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, deviceType%v", + "hypervisor.Device": devType}).Warn("hotplugAddDevice: unsupported device") + return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, hypervisor.Device%v", devInfo, devType) } } // hotplugRemoveDevice supported in Firecracker VMM, but no-op -func (fc *firecracker) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (fc *firecracker) hotplugRemoveDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { return nil, nil } @@ -675,7 +676,7 @@ func (fc *firecracker) capabilities() types.Capabilities { return caps } -func (fc *firecracker) hypervisorConfig() HypervisorConfig { +func (fc *firecracker) hypervisorConfig() hypervisor.Config { return fc.config } @@ -693,7 +694,7 @@ func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCP // Need to see if there's an easy way to ask firecracker for thread ids associated with // the vCPUs. Issue opened to ask for per vCPU thread IDs: // https://github.com/firecracker-microvm/firecracker/issues/718 -func (fc *firecracker) getThreadIDs() (*threadIDs, error) { +func (fc *firecracker) getThreadIDs() (*hypervisor.ThreadIDs, error) { //TODO: this may not be exactly supported in Firecracker. Closest is cpu-template as part // of get /machine-config return nil, nil diff --git a/virtcontainers/hack/virtc/main.go b/virtcontainers/hack/virtc/main.go index 2c25d43031..846c922855 100644 --- a/virtcontainers/hack/virtc/main.go +++ b/virtcontainers/hack/virtc/main.go @@ -13,6 +13,7 @@ import ( "strings" "text/tabwriter" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/sirupsen/logrus" @@ -101,7 +102,7 @@ var sandboxConfigFlags = []cli.Flag{ }, } -var ccKernelParams = []vc.Param{ +var ccKernelParams = []hypervisor.Param{ { Key: "init", Value: "/usr/lib/systemd/systemd", @@ -120,7 +121,7 @@ var ccKernelParams = []vc.Param{ }, } -func buildKernelParams(config *vc.HypervisorConfig) error { +func buildKernelParams(config *hypervisor.Config) error { for _, p := range ccKernelParams { if err := config.AddKernelParam(p); err != nil { return err @@ -159,7 +160,7 @@ func buildSandboxConfig(context *cli.Context) (vc.SandboxConfig, error) { kernelPath = "/usr/share/clear-containers/vmlinux.container" } - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: kernelPath, ImagePath: "/usr/share/clear-containers/clear-containers.img", HypervisorMachineType: machineType, @@ -195,7 +196,7 @@ func buildSandboxConfig(context *cli.Context) (vc.SandboxConfig, error) { sandboxConfig := vc.SandboxConfig{ ID: id, - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: *agentType, diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index da6b9ecb69..14c00a24c7 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -20,6 +20,7 @@ import ( proxyClient "github.com/clearcontainers/proxy/client" "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" @@ -305,9 +306,9 @@ func (h *hyper) getSharePath(id string) string { return filepath.Join(defaultSharedDir, id) } -func (h *hyper) configure(hv hypervisor, id, sharePath string, builtin bool, config interface{}) error { +func (h *hyper) configure(hv hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error { for _, socket := range h.sockets { - err := hv.addDevice(socket, serialPortDev) + err := hv.AddDevice(socket, hypervisor.SerialPortDev) if err != nil { return err } @@ -324,7 +325,7 @@ func (h *hyper) configure(hv hypervisor, id, sharePath string, builtin bool, con return err } - return hv.addDevice(sharedVolume, fsDev) + return hv.AddDevice(sharedVolume, hypervisor.FsDev) } func (h *hyper) createSandbox(sandbox *Sandbox) (err error) { @@ -815,7 +816,7 @@ func (h *hyper) register() error { } defer h.disconnect() - console, err := h.sandbox.hypervisor.getSandboxConsole(h.sandbox.id) + console, err := h.sandbox.hypervisor.GetSandboxConsole(h.sandbox.id) if err != nil { return err } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor/hypervisor.go similarity index 70% rename from virtcontainers/hypervisor.go rename to virtcontainers/hypervisor/hypervisor.go index dd37f8706d..416a7d2964 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor/hypervisor.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "bufio" @@ -17,99 +17,128 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/sirupsen/logrus" ) -// HypervisorType describes an hypervisor type. -type HypervisorType string +var hLog = logrus.WithField("source", "virtcontainers/hypervisor") -type operation int +// Type describes an hypervisor type. +type Type string const ( - // FirecrackerHypervisor is the FC hypervisor. - FirecrackerHypervisor HypervisorType = "firecracker" + // Firecracker is the Firecracker hypervisor. + Firecracker Type = "firecracker" - // QemuHypervisor is the QEMU hypervisor. - QemuHypervisor HypervisorType = "qemu" + // Qemu is the QEMU hypervisor. + Qemu Type = "qemu" - // MockHypervisor is a mock hypervisor for testing purposes - MockHypervisor HypervisorType = "mock" + // Mock is a mock hypervisor for testing purposes + Mock Type = "mock" +) + +// Operation represents a hypervisor device operation +type Operation int + +const ( + // AddDevice adds a device to a guest. + AddDevice Operation = iota + + // RemoveDevice removes a device from a guest. + RemoveDevice ) const ( - procMemInfo = "/proc/meminfo" - procCPUInfo = "/proc/cpuinfo" + // ProcMemInfo is the /proc/meminfo path + ProcMemInfo = "/proc/meminfo" + + // ProcCPUInfo is the /proc/cpuinfo path + ProcCPUInfo = "/proc/cpuinfo" ) const ( - defaultVCPUs = 1 - // 2 GiB - defaultMemSzMiB = 2048 + // DefaultVCPUs is the number of vCPUs a virtcontainers VM will run with by default. + DefaultVCPUs = 1 - defaultBridges = 1 + // DefaultMemSzMiB is the amount of memory a virtcontainers VM will run with by default. + DefaultMemSzMiB = 2048 - defaultBlockDriver = config.VirtioSCSI + // DefaultBridges is the number of PCI bridges a virtcontainers VM will run with by default. + DefaultBridges = 1 + + // DefaultBlockDriver is the default virtio block based driver. + DefaultBlockDriver = config.VirtioSCSI + + // DefaultMaxQemuVCPUs is the maximum number of vCPUs a virtcontainers VM will run with by default. + DefaultMaxQemuVCPUs uint32 = 4 + + // DefaultMsize9p is the default 9pfs msize value. + DefaultMsize9p = 8192 ) // In some architectures the maximum number of vCPUs depends on the number of physical cores. -var defaultMaxQemuVCPUs = MaxQemuVCPUs() +//var defaultMaxQemuVCPUs = MaxQemuVCPUs() -// deviceType describes a virtualized device type. -type deviceType int +// Device describes a virtualized device. +type Device int const ( // ImgDev is the image device type. - imgDev deviceType = iota + ImgDev Device = iota // FsDev is the filesystem device type. - fsDev + FsDev // NetDev is the network device type. - netDev + NetDev // SerialDev is the serial device type. - serialDev // nolint: varcheck,unused + SerialDev // nolint: varcheck,unused // BlockDev is the block device type. - blockDev + BlockDev // ConsoleDev is the console device type. - consoleDev // nolint: varcheck,unused + ConsoleDev // nolint: varcheck,unused // SerialPortDev is the serial port device type. - serialPortDev + SerialPortDev - // vSockPCIDev is the vhost vsock PCI device type. - vSockPCIDev + // VSockPCIDev is the vhost vsock PCI device type. + VSockPCIDev - // VFIODevice is VFIO device type - vfioDev + // VfioDev is VFIO device type + VfioDev - // vhostuserDev is a Vhost-user device type - vhostuserDev + // VhostuserDev is a Vhost-user device type + VhostuserDev - // CPUDevice is CPU device type - cpuDev + // CPUDev is CPU device type + CPUDev - // memoryDevice is memory device type - memoryDev + // MemoryDev is memory device type + MemoryDev ) -type memoryDevice struct { - slot int - sizeMB int +// MemoryDevice represents a memory slot +type MemoryDevice struct { + // Slot is the memory slot ID + Slot int + + // SizeMB is the memory slot size in MBytes. + SizeMB int } // Set sets an hypervisor type based on the input string. -func (hType *HypervisorType) Set(value string) error { +func (t *Type) Set(value string) error { switch value { case "qemu": - *hType = QemuHypervisor + *t = Qemu return nil case "firecracker": - *hType = FirecrackerHypervisor + *t = Firecracker return nil case "mock": - *hType = MockHypervisor + *t = Mock return nil default: return fmt.Errorf("Unknown hypervisor type %s", value) @@ -117,31 +146,22 @@ func (hType *HypervisorType) Set(value string) error { } // String converts an hypervisor type to a string. -func (hType *HypervisorType) String() string { - switch *hType { - case QemuHypervisor: - return string(QemuHypervisor) - case FirecrackerHypervisor: - return string(FirecrackerHypervisor) - case MockHypervisor: - return string(MockHypervisor) +func (t *Type) String() string { + switch *t { + case Qemu: + return string(Qemu) + case Firecracker: + return string(Firecracker) + case Mock: + return string(Mock) default: return "" } } -// newHypervisor returns an hypervisor from and hypervisor type. -func newHypervisor(hType HypervisorType) (hypervisor, error) { - switch hType { - case QemuHypervisor: - return &qemu{}, nil - case FirecrackerHypervisor: - return &firecracker{}, nil - case MockHypervisor: - return &mockHypervisor{}, nil - default: - return nil, fmt.Errorf("Unknown hypervisor type %s", hType) - } +// New returns an hypervisor from and hypervisor type. +func New(t Type) (Hypervisor, error) { + return nil, fmt.Errorf("Unknown hypervisor type %s", t) } // Param is a key/value representation for hypervisor and kernel parameters. @@ -150,8 +170,8 @@ type Param struct { Value string } -// HypervisorConfig is the hypervisor configuration. -type HypervisorConfig struct { +// Config is the hypervisor configuration. +type Config struct { // NumVCPUs specifies default number of vCPUs for the VM. NumVCPUs uint32 @@ -285,11 +305,13 @@ type HypervisorConfig struct { GuestHookPath string } -type threadIDs struct { - vcpus []int +// ThreadIDs represent a set of threads vCPU IDs. +type ThreadIDs struct { + // VCPUs is a slice of vCPU IDs. + VCPUs []int } -func (conf *HypervisorConfig) checkTemplateConfig() error { +func (conf *Config) checkTemplateConfig() error { if conf.BootToBeTemplate && conf.BootFromTemplate { return fmt.Errorf("Cannot set both 'to be' and 'from' vm tempate") } @@ -307,7 +329,8 @@ func (conf *HypervisorConfig) checkTemplateConfig() error { return nil } -func (conf *HypervisorConfig) valid() error { +// Valid checks if a hypervisor configuration is valid. +func (conf *Config) Valid() error { if conf.KernelPath == "" { return fmt.Errorf("Missing kernel path") } @@ -321,27 +344,27 @@ func (conf *HypervisorConfig) valid() error { } if conf.NumVCPUs == 0 { - conf.NumVCPUs = defaultVCPUs + conf.NumVCPUs = DefaultVCPUs } if conf.MemorySize == 0 { - conf.MemorySize = defaultMemSzMiB + conf.MemorySize = DefaultMemSzMiB } if conf.DefaultBridges == 0 { - conf.DefaultBridges = defaultBridges + conf.DefaultBridges = DefaultBridges } if conf.BlockDeviceDriver == "" { - conf.BlockDeviceDriver = defaultBlockDriver + conf.BlockDeviceDriver = DefaultBlockDriver } if conf.DefaultMaxVCPUs == 0 { - conf.DefaultMaxVCPUs = defaultMaxQemuVCPUs + conf.DefaultMaxVCPUs = DefaultMaxQemuVCPUs } if conf.Msize9p == 0 { - conf.Msize9p = defaultMsize9p + conf.Msize9p = DefaultMsize9p } return nil @@ -349,7 +372,7 @@ func (conf *HypervisorConfig) valid() error { // AddKernelParam allows the addition of new kernel parameters to an existing // hypervisor configuration. -func (conf *HypervisorConfig) AddKernelParam(p Param) error { +func (conf *Config) AddKernelParam(p Param) error { if p.Key == "" { return fmt.Errorf("Empty kernel parameter") } @@ -359,7 +382,8 @@ func (conf *HypervisorConfig) AddKernelParam(p Param) error { return nil } -func (conf *HypervisorConfig) addCustomAsset(a *types.Asset) error { +// AddCustomAsset adds a custom asset to a hypervisor configuration +func (conf *Config) AddCustomAsset(a *types.Asset) error { if a == nil || a.Path() == "" { // We did not get a custom asset, we will use the default one. return nil @@ -369,7 +393,7 @@ func (conf *HypervisorConfig) addCustomAsset(a *types.Asset) error { return fmt.Errorf("Invalid %s at %s", a.Type(), a.Path()) } - virtLog.Debugf("Using custom %v asset %s", a.Type(), a.Path()) + hLog.Debugf("Using custom %v asset %s", a.Type(), a.Path()) if conf.customAssets == nil { conf.customAssets = make(map[types.AssetType]*types.Asset) @@ -380,7 +404,7 @@ func (conf *HypervisorConfig) addCustomAsset(a *types.Asset) error { return nil } -func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) { +func (conf *Config) assetPath(t types.AssetType) (string, error) { // Custom assets take precedence over the configured ones a, ok := conf.customAssets[t] if ok { @@ -405,7 +429,7 @@ func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) { } } -func (conf *HypervisorConfig) isCustomAsset(t types.AssetType) bool { +func (conf *Config) isCustomAsset(t types.AssetType) bool { _, ok := conf.customAssets[t] if ok { return true @@ -415,52 +439,52 @@ func (conf *HypervisorConfig) isCustomAsset(t types.AssetType) bool { } // KernelAssetPath returns the guest kernel path -func (conf *HypervisorConfig) KernelAssetPath() (string, error) { +func (conf *Config) KernelAssetPath() (string, error) { return conf.assetPath(types.KernelAsset) } // CustomKernelAsset returns true if the kernel asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomKernelAsset() bool { +func (conf *Config) CustomKernelAsset() bool { return conf.isCustomAsset(types.KernelAsset) } // ImageAssetPath returns the guest image path -func (conf *HypervisorConfig) ImageAssetPath() (string, error) { +func (conf *Config) ImageAssetPath() (string, error) { return conf.assetPath(types.ImageAsset) } // CustomImageAsset returns true if the image asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomImageAsset() bool { +func (conf *Config) CustomImageAsset() bool { return conf.isCustomAsset(types.ImageAsset) } // InitrdAssetPath returns the guest initrd path -func (conf *HypervisorConfig) InitrdAssetPath() (string, error) { +func (conf *Config) InitrdAssetPath() (string, error) { return conf.assetPath(types.InitrdAsset) } // CustomInitrdAsset returns true if the initrd asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomInitrdAsset() bool { +func (conf *Config) CustomInitrdAsset() bool { return conf.isCustomAsset(types.InitrdAsset) } // HypervisorAssetPath returns the VM hypervisor path -func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) { +func (conf *Config) HypervisorAssetPath() (string, error) { return conf.assetPath(types.HypervisorAsset) } // CustomHypervisorAsset returns true if the hypervisor asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomHypervisorAsset() bool { +func (conf *Config) CustomHypervisorAsset() bool { return conf.isCustomAsset(types.HypervisorAsset) } // FirmwareAssetPath returns the guest firmware path -func (conf *HypervisorConfig) FirmwareAssetPath() (string, error) { +func (conf *Config) FirmwareAssetPath() (string, error) { return conf.assetPath(types.FirmwareAsset) } // CustomFirmwareAsset returns true if the firmware asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomFirmwareAsset() bool { +func (conf *Config) CustomFirmwareAsset() bool { return conf.isCustomAsset(types.FirmwareAsset) } @@ -509,7 +533,8 @@ func DeserializeParams(parameters []string) []Param { return params } -func getHostMemorySizeKb(memInfoPath string) (uint64, error) { +// GetHostMemorySizeKb return the host memory size in KBytes. +func GetHostMemorySizeKb(memInfoPath string) (uint64, error) { f, err := os.Open(memInfoPath) if err != nil { return 0, err @@ -545,7 +570,7 @@ func getHostMemorySizeKb(memInfoPath string) (uint64, error) { // RunningOnVMM checks if the system is running inside a VM. func RunningOnVMM(cpuInfoPath string) (bool, error) { if runtime.GOARCH == "arm64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x" { - virtLog.Info("Unable to know if the system is running inside a VM") + hLog.Info("Unable to know if the system is running inside a VM") return false, nil } @@ -589,24 +614,24 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) { return false, fmt.Errorf("Couldn't find %q from %q output", flagsField, cpuInfoPath) } -// hypervisor is the virtcontainers hypervisor interface. +// Hypervisor is the virtcontainers hypervisor interface. // The default hypervisor implementation is Qemu. -type hypervisor interface { - createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error - startSandbox(timeout int) error - stopSandbox() error - pauseSandbox() error - saveSandbox() error - resumeSandbox() error - addDevice(devInfo interface{}, devType deviceType) error - hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) - hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) - resizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error) - resizeVCPUs(vcpus uint32) (uint32, uint32, error) - getSandboxConsole(sandboxID string) (string, error) - disconnect() - capabilities() types.Capabilities - hypervisorConfig() HypervisorConfig - getThreadIDs() (*threadIDs, error) - cleanup() error +type Hypervisor interface { + CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, store *store.VCStore) error + StartSandbox(timeout int) error + StopSandbox() error + PauseSandbox() error + SaveSandbox() error + ResumeSandbox() error + AddDevice(devInfo interface{}, dev Device) error + HotplugAddDevice(devInfo interface{}, dev Device) (interface{}, error) + HotplugRemoveDevice(devInfo interface{}, dev Device) (interface{}, error) + ResizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error) + ResizeVCPUs(vcpus uint32) (uint32, uint32, error) + GetSandboxConsole(sandboxID string) (string, error) + Disconnect() + Capabilities() types.Capabilities + Config() Config + GetThreadIDs() (*ThreadIDs, error) + Cleanup() error } diff --git a/virtcontainers/hypervisor_test.go b/virtcontainers/hypervisor/hypervisor_test.go similarity index 71% rename from virtcontainers/hypervisor_test.go rename to virtcontainers/hypervisor/hypervisor_test.go index 0a1b2514f4..710418cbd4 100644 --- a/virtcontainers/hypervisor_test.go +++ b/virtcontainers/hypervisor/hypervisor_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -14,8 +14,17 @@ import ( "testing" ) -func testSetHypervisorType(t *testing.T, value string, expected HypervisorType) { - var hypervisorType HypervisorType +const testKernel = "kernel" +const testInitrd = "initrd" +const testImage = "image" +const testHypervisor = "hypervisor" + +var testDir = "" + +const testDisabledAsNonRoot = "Test disabled as requires root privileges" + +func testSetType(t *testing.T, value string, expected Type) { + var hypervisorType Type err := (&hypervisorType).Set(value) if err != nil { @@ -27,52 +36,53 @@ func testSetHypervisorType(t *testing.T, value string, expected HypervisorType) } } -func TestSetQemuHypervisorType(t *testing.T) { - testSetHypervisorType(t, "qemu", QemuHypervisor) +func TestSetQemuType(t *testing.T) { + testSetType(t, "qemu", Qemu) } -func TestSetMockHypervisorType(t *testing.T) { - testSetHypervisorType(t, "mock", MockHypervisor) +func TestSetMockType(t *testing.T) { + testSetType(t, "mock", Mock) } -func TestSetUnknownHypervisorType(t *testing.T) { - var hypervisorType HypervisorType +func TestSetUnknownType(t *testing.T) { + var hypervisorType Type err := (&hypervisorType).Set("unknown") if err == nil { t.Fatal() } - if hypervisorType == QemuHypervisor || - hypervisorType == MockHypervisor { + if hypervisorType == Qemu || + hypervisorType == Mock || + hypervisorType == Firecracker { t.Fatal() } } -func testStringFromHypervisorType(t *testing.T, hypervisorType HypervisorType, expected string) { +func testStringFromType(t *testing.T, hypervisorType Type, expected string) { hypervisorTypeStr := (&hypervisorType).String() if hypervisorTypeStr != expected { t.Fatal() } } -func TestStringFromQemuHypervisorType(t *testing.T) { - hypervisorType := QemuHypervisor - testStringFromHypervisorType(t, hypervisorType, "qemu") +func TestStringFromQemuType(t *testing.T) { + hypervisorType := Qemu + testStringFromType(t, hypervisorType, "qemu") } -func TestStringFromMockHypervisorType(t *testing.T) { - hypervisorType := MockHypervisor - testStringFromHypervisorType(t, hypervisorType, "mock") +func TestStringFromMockType(t *testing.T) { + hypervisorType := Mock + testStringFromType(t, hypervisorType, "mock") } -func TestStringFromUnknownHypervisorType(t *testing.T) { - var hypervisorType HypervisorType - testStringFromHypervisorType(t, hypervisorType, "") +func TestStringFromUnknownType(t *testing.T) { + var hypervisorType Type + testStringFromType(t, hypervisorType, "") } -func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType HypervisorType, expected hypervisor) { - hy, err := newHypervisor(hypervisorType) +func testNewHypervisorFromType(t *testing.T, hypervisorType Type, expected Hypervisor) { + hy, err := New(hypervisorType) if err != nil { t.Fatal(err) } @@ -82,22 +92,22 @@ func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType Hypervisor } } -func TestNewHypervisorFromQemuHypervisorType(t *testing.T) { - hypervisorType := QemuHypervisor - expectedHypervisor := &qemu{} - testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor) -} +// func TestNewHypervisorFromQemuType(t *testing.T) { +// hypervisorType := Qemu +// expectedHypervisor := &qemu{} +// testNewHypervisorFromType(t, hypervisorType, expectedHypervisor) +// } -func TestNewHypervisorFromMockHypervisorType(t *testing.T) { - hypervisorType := MockHypervisor - expectedHypervisor := &mockHypervisor{} - testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor) -} +// func TestNewHypervisorFromMockType(t *testing.T) { +// hypervisorType := Mock +// expectedHypervisor := &mockHypervisor{} +// testNewHypervisorFromType(t, hypervisorType, expectedHypervisor) +// } -func TestNewHypervisorFromUnknownHypervisorType(t *testing.T) { - var hypervisorType HypervisorType +func TestNewHypervisorFromUnknownType(t *testing.T) { + var hypervisorType Type - hy, err := newHypervisor(hypervisorType) + hy, err := New(hypervisorType) if err == nil { t.Fatal() } @@ -107,8 +117,8 @@ func TestNewHypervisorFromUnknownHypervisorType(t *testing.T) { } } -func testHypervisorConfigValid(t *testing.T, hypervisorConfig *HypervisorConfig, success bool) { - err := hypervisorConfig.valid() +func testConfigValid(t *testing.T, hypervisorConfig *Config, success bool) { + err := hypervisorConfig.Valid() if success && err != nil { t.Fatal() } @@ -117,88 +127,88 @@ func testHypervisorConfigValid(t *testing.T, hypervisorConfig *HypervisorConfig, } } -func TestHypervisorConfigNoKernelPath(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigNoKernelPath(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: "", ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) } -func TestHypervisorConfigNoImagePath(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigNoImagePath(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: "", HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) } -func TestHypervisorConfigNoHypervisorPath(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigNoHypervisorPath(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: "", } - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) } -func TestHypervisorConfigIsValid(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigIsValid(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) } -func TestHypervisorConfigValidTemplateConfig(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigValidTemplateConfig(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), BootToBeTemplate: true, BootFromTemplate: true, } - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) hypervisorConfig.BootToBeTemplate = false - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) hypervisorConfig.MemoryPath = "foobar" - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) hypervisorConfig.DevicesStatePath = "foobar" - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) hypervisorConfig.BootFromTemplate = false hypervisorConfig.BootToBeTemplate = true - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) hypervisorConfig.MemoryPath = "" - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) } -func TestHypervisorConfigDefaults(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigDefaults(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: "", } - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) - hypervisorConfigDefaultsExpected := &HypervisorConfig{ + hypervisorConfigDefaultsExpected := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: "", - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: DefaultVCPUs, + MemorySize: DefaultMemSzMiB, + DefaultBridges: DefaultBridges, + BlockDeviceDriver: DefaultBlockDriver, + DefaultMaxVCPUs: DefaultMaxQemuVCPUs, + Msize9p: DefaultMsize9p, } if reflect.DeepEqual(hypervisorConfig, hypervisorConfigDefaultsExpected) == false { @@ -353,7 +363,7 @@ func TestDeserializeParams(t *testing.T) { } func TestAddKernelParamValid(t *testing.T) { - var config HypervisorConfig + var config Config expected := []Param{ {"foo", "bar"}, @@ -366,7 +376,7 @@ func TestAddKernelParamValid(t *testing.T) { } func TestAddKernelParamInvalid(t *testing.T) { - var config HypervisorConfig + var config Config invalid := []Param{ {"", "bar"}, @@ -415,7 +425,7 @@ func TestGetHostMemorySizeKb(t *testing.T) { defer os.RemoveAll(dir) file := filepath.Join(dir, "meminfo") - if _, err := getHostMemorySizeKb(file); err == nil { + if _, err := GetHostMemorySizeKb(file); err == nil { t.Fatalf("expected failure as file %q does not exist", file) } @@ -425,7 +435,7 @@ func TestGetHostMemorySizeKb(t *testing.T) { } defer os.Remove(file) - hostMemKb, err := getHostMemorySizeKb(file) + hostMemKb, err := GetHostMemorySizeKb(file) if (d.expectError && err == nil) || (!d.expectError && err != nil) { t.Fatalf("got %d, input %v", hostMemKb, d) diff --git a/virtcontainers/iostream_test.go b/virtcontainers/iostream_test.go index 152f0a6118..a394e958a9 100644 --- a/virtcontainers/iostream_test.go +++ b/virtcontainers/iostream_test.go @@ -8,12 +8,13 @@ package virtcontainers import ( "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/stretchr/testify/assert" ) func TestIOStream(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{}, nil) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go index b71bbd0fbe..6b8517e7e3 100644 --- a/virtcontainers/ipvlan_endpoint.go +++ b/virtcontainers/ipvlan_endpoint.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) // IPVlanEndpoint represents a ipvlan endpoint that is bridged to the VM @@ -86,13 +87,13 @@ func (endpoint *IPVlanEndpoint) NetworkPair() *NetworkInterfacePair { // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *IPVlanEndpoint) Attach(h hypervisor) error { +func (endpoint *IPVlanEndpoint) Attach(h hypervisor.Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, hypervisor.NetDev) } // Detach for the virtual endpoint tears down the tap and bridge @@ -110,11 +111,11 @@ func (endpoint *IPVlanEndpoint) Detach(netNsCreated bool, netNsPath string) erro } // HotAttach for physical endpoint not supported yet -func (endpoint *IPVlanEndpoint) HotAttach(h hypervisor) error { +func (endpoint *IPVlanEndpoint) HotAttach(h hypervisor.Hypervisor) error { return fmt.Errorf("IPVlanEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *IPVlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *IPVlanEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("IPVlanEndpoint does not support Hot detach") } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index dcf4735d47..1386614532 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -23,6 +23,7 @@ import ( kataclient "github.com/kata-containers/agent/protocols/client" "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" @@ -221,7 +222,7 @@ func (k *kataAgent) capabilities() types.Capabilities { return caps } -func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error { +func (k *kataAgent) configure(h hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error { if config != nil { switch c := config.(type) { case KataAgentConfig: @@ -236,7 +237,7 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, switch s := k.vmSocket.(type) { case types.Socket: - err := h.addDevice(s, serialPortDev) + err := h.AddDevice(s, hypervisor.SerialPortDev) if err != nil { return err } @@ -247,7 +248,7 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, return err } s.port = uint32(vSockPort) - if err := h.addDevice(s, vSockPCIDev); err != nil { + if err := h.AddDevice(s, hypervisor.VSockPCIDev); err != nil { return err } k.vmSocket = s @@ -261,7 +262,7 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, // Neither create shared directory nor add 9p device if hypervisor // doesn't support filesystem sharing. - caps := h.capabilities() + caps := h.Capabilities() if !caps.IsFsSharingSupported() { return nil } @@ -277,7 +278,7 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, return err } - return h.addDevice(sharedVolume, fsDev) + return h.AddDevice(sharedVolume, hypervisor.FsDev) } func (k *kataAgent) createSandbox(sandbox *Sandbox) error { @@ -505,7 +506,7 @@ func (k *kataAgent) startProxy(sandbox *Sandbox) error { return err } - consoleURL, err := sandbox.hypervisor.getSandboxConsole(sandbox.id) + consoleURL, err := sandbox.hypervisor.GetSandboxConsole(sandbox.id) if err != nil { return err } @@ -627,7 +628,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { } storages := []*grpc.Storage{} - caps := sandbox.hypervisor.capabilities() + caps := sandbox.hypervisor.Capabilities() // append 9p shared volume to storages only if filesystem sharing is supported if caps.IsFsSharingSupported() { diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 5a55cbd7e9..68daa1e592 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -29,6 +29,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/mock" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" @@ -412,7 +413,7 @@ func TestAppendDevices(t *testing.T) { } sandboxConfig := &SandboxConfig{ - HypervisorConfig: HypervisorConfig{ + HypervisorConfig: hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, }, } @@ -730,8 +731,8 @@ func TestAgentCreateContainer(t *testing.T) { id: "foobar", config: &SandboxConfig{ ID: "foobar", - HypervisorType: MockHypervisor, - HypervisorConfig: HypervisorConfig{ + HypervisorType: hypervisor.Mock, + HypervisorConfig: hypervisor.Config{ KernelPath: "foo", ImagePath: "bar", }, diff --git a/virtcontainers/macvtap_endpoint.go b/virtcontainers/macvtap_endpoint.go index d35d9c70cb..d21fcf50c4 100644 --- a/virtcontainers/macvtap_endpoint.go +++ b/virtcontainers/macvtap_endpoint.go @@ -8,6 +8,8 @@ package virtcontainers import ( "fmt" "os" + + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) // MacvtapEndpoint represents a macvtap endpoint @@ -54,23 +56,23 @@ func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) { } // Attach for macvtap endpoint passes macvtap device to the hypervisor. -func (endpoint *MacvtapEndpoint) Attach(h hypervisor) error { +func (endpoint *MacvtapEndpoint) Attach(h hypervisor.Hypervisor) error { var err error - endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs)) + endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.Config().NumVCPUs)) if err != nil { return fmt.Errorf("Could not setup macvtap fds %s: %s", endpoint.EndpointProperties.Iface.Name, err) } - if !h.hypervisorConfig().DisableVhostNet { - vhostFds, err := createVhostFds(int(h.hypervisorConfig().NumVCPUs)) + if !h.Config().DisableVhostNet { + vhostFds, err := createVhostFds(int(h.Config().NumVCPUs)) if err != nil { return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.EndpointProperties.Iface.Name, err) } endpoint.VhostFds = vhostFds } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, hypervisor.NetDev) } // Detach for macvtap endpoint does nothing. @@ -79,12 +81,12 @@ func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) err } // HotAttach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor) error { +func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor.Hypervisor) error { return fmt.Errorf("MacvtapEndpoint does not support Hot attach") } // HotDetach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("MacvtapEndpoint does not support Hot detach") } diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index 37dccaf901..fc204473c2 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -9,6 +9,7 @@ import ( "context" "os" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -16,16 +17,16 @@ import ( type mockHypervisor struct { } -func (m *mockHypervisor) capabilities() types.Capabilities { +func (m *mockHypervisor) Capabilities() types.Capabilities { return types.Capabilities{} } -func (m *mockHypervisor) hypervisorConfig() HypervisorConfig { - return HypervisorConfig{} +func (m *mockHypervisor) Config() hypervisor.Config { + return hypervisor.Config{} } -func (m *mockHypervisor) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { - err := hypervisorConfig.valid() +func (m *mockHypervisor) CreateSandbox(ctx context.Context, id string, hypervisorConfig *hypervisor.Config, store *store.VCStore) error { + err := hypervisorConfig.Valid() if err != nil { return err } @@ -33,70 +34,70 @@ func (m *mockHypervisor) createSandbox(ctx context.Context, id string, hyperviso return nil } -func (m *mockHypervisor) startSandbox(timeout int) error { +func (m *mockHypervisor) StartSandbox(timeout int) error { return nil } -func (m *mockHypervisor) stopSandbox() error { +func (m *mockHypervisor) StopSandbox() error { return nil } -func (m *mockHypervisor) pauseSandbox() error { +func (m *mockHypervisor) PauseSandbox() error { return nil } -func (m *mockHypervisor) resumeSandbox() error { +func (m *mockHypervisor) ResumeSandbox() error { return nil } -func (m *mockHypervisor) saveSandbox() error { +func (m *mockHypervisor) SaveSandbox() error { return nil } -func (m *mockHypervisor) addDevice(devInfo interface{}, devType deviceType) error { +func (m *mockHypervisor) AddDevice(devInfo interface{}, devType hypervisor.Device) error { return nil } -func (m *mockHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (m *mockHypervisor) HotplugAddDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { switch devType { - case cpuDev: + case hypervisor.CPUDev: return devInfo.(uint32), nil - case memoryDev: - memdev := devInfo.(*memoryDevice) - return memdev.sizeMB, nil + case hypervisor.MemoryDev: + memdev := devInfo.(*hypervisor.MemoryDevice) + return memdev.SizeMB, nil } return nil, nil } -func (m *mockHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (m *mockHypervisor) HotplugRemoveDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { switch devType { - case cpuDev: + case hypervisor.CPUDev: return devInfo.(uint32), nil - case memoryDev: + case hypervisor.MemoryDev: return 0, nil } return nil, nil } -func (m *mockHypervisor) getSandboxConsole(sandboxID string) (string, error) { +func (m *mockHypervisor) GetSandboxConsole(sandboxID string) (string, error) { return "", nil } -func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { +func (m *mockHypervisor) ResizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { return 0, nil } -func (m *mockHypervisor) resizeVCPUs(cpus uint32) (uint32, uint32, error) { +func (m *mockHypervisor) ResizeVCPUs(cpus uint32) (uint32, uint32, error) { return 0, 0, nil } -func (m *mockHypervisor) disconnect() { +func (m *mockHypervisor) Disconnect() { } -func (m *mockHypervisor) getThreadIDs() (*threadIDs, error) { +func (m *mockHypervisor) GetThreadIDs() (*hypervisor.ThreadIDs, error) { vcpus := []int{os.Getpid()} - return &threadIDs{vcpus}, nil + return &hypervisor.ThreadIDs{vcpus}, nil } -func (m *mockHypervisor) cleanup() error { +func (m *mockHypervisor) Cleanup() error { return nil } diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go index 34885c45f9..d651f018e4 100644 --- a/virtcontainers/mock_hypervisor_test.go +++ b/virtcontainers/mock_hypervisor_test.go @@ -9,6 +9,8 @@ import ( "context" "fmt" "testing" + + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestMockHypervisorCreateSandbox(t *testing.T) { @@ -17,7 +19,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) { sandbox := &Sandbox{ config: &SandboxConfig{ ID: "mock_sandbox", - HypervisorConfig: HypervisorConfig{ + HypervisorConfig: hypervisor.Config{ KernelPath: "", ImagePath: "", HypervisorPath: "", @@ -28,17 +30,17 @@ func TestMockHypervisorCreateSandbox(t *testing.T) { ctx := context.Background() // wrong config - if err := m.createSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err == nil { + if err := m.CreateSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err == nil { t.Fatal() } - sandbox.config.HypervisorConfig = HypervisorConfig{ + sandbox.config.HypervisorConfig = hypervisor.Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - if err := m.createSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err != nil { + if err := m.CreateSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err != nil { t.Fatal(err) } } @@ -46,7 +48,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) { func TestMockHypervisorStartSandbox(t *testing.T) { var m *mockHypervisor - if err := m.startSandbox(vmStartTimeout); err != nil { + if err := m.StartSandbox(vmStartTimeout); err != nil { t.Fatal(err) } } @@ -54,7 +56,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) { func TestMockHypervisorStopSandbox(t *testing.T) { var m *mockHypervisor - if err := m.stopSandbox(); err != nil { + if err := m.StopSandbox(); err != nil { t.Fatal(err) } } @@ -62,7 +64,7 @@ func TestMockHypervisorStopSandbox(t *testing.T) { func TestMockHypervisorAddDevice(t *testing.T) { var m *mockHypervisor - if err := m.addDevice(nil, imgDev); err != nil { + if err := m.AddDevice(nil, hypervisor.ImgDev); err != nil { t.Fatal(err) } } @@ -72,7 +74,7 @@ func TestMockHypervisorGetSandboxConsole(t *testing.T) { expected := "" - result, err := m.getSandboxConsole("testSandboxID") + result, err := m.GetSandboxConsole("testSandboxID") if err != nil { t.Fatal(err) } @@ -85,7 +87,7 @@ func TestMockHypervisorGetSandboxConsole(t *testing.T) { func TestMockHypervisorSaveSandbox(t *testing.T) { var m *mockHypervisor - if err := m.saveSandbox(); err != nil { + if err := m.SaveSandbox(); err != nil { t.Fatal(err) } } @@ -93,5 +95,5 @@ func TestMockHypervisorSaveSandbox(t *testing.T) { func TestMockHypervisorDisconnect(t *testing.T) { var m *mockHypervisor - m.disconnect() + m.Disconnect() } diff --git a/virtcontainers/monitor_test.go b/virtcontainers/monitor_test.go index 2d786acf37..90dd5f2512 100644 --- a/virtcontainers/monitor_test.go +++ b/virtcontainers/monitor_test.go @@ -9,6 +9,7 @@ import ( "errors" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/stretchr/testify/assert" ) @@ -18,7 +19,7 @@ func TestMonitorSuccess(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } @@ -43,7 +44,7 @@ func TestMonitorClosedChannel(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 01a666c8a0..131645ae7d 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -24,6 +24,7 @@ import ( "github.com/vishvananda/netns" "golang.org/x/sys/unix" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/utils" @@ -459,16 +460,16 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. } // The endpoint type should dictate how the connection needs to happen. -func xConnectVMNetwork(endpoint Endpoint, h hypervisor) error { +func xConnectVMNetwork(endpoint Endpoint, h hypervisor.Hypervisor) error { netPair := endpoint.NetworkPair() queues := 0 - caps := h.capabilities() + caps := h.Capabilities() if caps.IsMultiQueueSupported() { - queues = int(h.hypervisorConfig().NumVCPUs) + queues = int(h.Config().NumVCPUs) } - disableVhostNet := h.hypervisorConfig().DisableVhostNet + disableVhostNet := h.Config().DisableVhostNet if netPair.NetInterworkingModel == NetXConnectDefaultModel { netPair.NetInterworkingModel = DefaultNetInterworkingModel @@ -1441,7 +1442,7 @@ func (n *Network) Run(networkNSPath string, cb func() error) error { } // Add adds all needed interfaces inside the network namespace. -func (n *Network) Add(ctx context.Context, config *NetworkConfig, hypervisor hypervisor, hotplug bool) ([]Endpoint, error) { +func (n *Network) Add(ctx context.Context, config *NetworkConfig, h hypervisor.Hypervisor, hotplug bool) ([]Endpoint, error) { span, _ := n.trace(ctx, "add") defer span.Finish() @@ -1454,11 +1455,11 @@ func (n *Network) Add(ctx context.Context, config *NetworkConfig, hypervisor hyp for _, endpoint := range endpoints { networkLogger().WithField("endpoint-type", endpoint.Type()).WithField("hotplug", hotplug).Info("Attaching endpoint") if hotplug { - if err := endpoint.HotAttach(hypervisor); err != nil { + if err := endpoint.HotAttach(h); err != nil { return err } } else { - if err := endpoint.Attach(hypervisor); err != nil { + if err := endpoint.Attach(h); err != nil { return err } } @@ -1477,7 +1478,7 @@ func (n *Network) Add(ctx context.Context, config *NetworkConfig, hypervisor hyp // Remove network endpoints in the network namespace. It also deletes the network // namespace in case the namespace has been created by us. -func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor hypervisor, hotunplug bool) error { +func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, h hypervisor.Hypervisor, hotunplug bool) error { span, _ := n.trace(ctx, "remove") defer span.Finish() @@ -1486,7 +1487,7 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h // if required. networkLogger().WithField("endpoint-type", endpoint.Type()).WithField("hotunplug", hotunplug).Info("Detaching endpoint") if hotunplug { - if err := endpoint.HotDetach(hypervisor, ns.NetNsCreated, ns.NetNsPath); err != nil { + if err := endpoint.HotDetach(h, ns.NetNsCreated, ns.NetNsPath); err != nil { return err } } else { diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go index 6ba15d4b33..c8ca3b1b97 100644 --- a/virtcontainers/noop_agent.go +++ b/virtcontainers/noop_agent.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/types" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -167,7 +168,7 @@ func (n *noopAgent) resumeContainer(sandbox *Sandbox, c Container) error { } // configHypervisor is the Noop agent hypervisor configuration implementation. It does nothing. -func (n *noopAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error { +func (n *noopAgent) configure(h hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error { return nil } diff --git a/virtcontainers/physical_endpoint.go b/virtcontainers/physical_endpoint.go index 989233f158..17b60abf2f 100644 --- a/virtcontainers/physical_endpoint.go +++ b/virtcontainers/physical_endpoint.go @@ -14,6 +14,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/safchain/ethtool" ) @@ -71,7 +72,7 @@ func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair { // Attach for physical endpoint binds the physical network interface to // vfio-pci and adds device to the hypervisor with vfio-passthrough. -func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { +func (endpoint *PhysicalEndpoint) Attach(h hypervisor.Hypervisor) error { // Unbind physical interface from host driver and bind to vfio // so that it can be passed to qemu. if err := bindNICToVFIO(endpoint); err != nil { @@ -83,7 +84,7 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { BDF: endpoint.BDF, } - return h.addDevice(d, vfioDev) + return h.AddDevice(d, hypervisor.VfioDev) } // Detach for physical endpoint unbinds the physical network interface from vfio-pci @@ -99,12 +100,12 @@ func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) er } // HotAttach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error { +func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor.Hypervisor) error { return fmt.Errorf("PhysicalEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("PhysicalEndpoint does not support Hot detach") } diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 4ffb4ec957..991870f22d 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -23,6 +23,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" "github.com/kata-containers/runtime/virtcontainers/types" @@ -100,8 +101,8 @@ type FactoryConfig struct { // RuntimeConfig aggregates all runtime specific settings type RuntimeConfig struct { - HypervisorType vc.HypervisorType - HypervisorConfig vc.HypervisorConfig + HypervisorType hypervisor.Type + HypervisorConfig hypervisor.Config NetmonConfig vc.NetmonConfig @@ -132,7 +133,7 @@ type RuntimeConfig struct { // AddKernelParam allows the addition of new kernel parameters to an existing // hypervisor configuration stored inside the current runtime configuration. -func (config *RuntimeConfig) AddKernelParam(p vc.Param) error { +func (config *RuntimeConfig) AddKernelParam(p hypervisor.Param) error { return config.HypervisorConfig.AddKernelParam(p) } diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index ff1daa9b64..ccd0180d03 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -23,6 +23,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -106,7 +107,7 @@ func TestMinimalSandboxConfig(t *testing.T) { }() runtimeConfig := RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, AgentType: vc.HyperstartAgent, ProxyType: vc.CCProxyType, ShimType: vc.CCShimType, @@ -221,7 +222,7 @@ func TestMinimalSandboxConfig(t *testing.T) { ID: containerID, Hostname: "testHostname", - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, AgentType: vc.HyperstartAgent, ProxyType: vc.CCProxyType, ShimType: vc.CCShimType, @@ -623,7 +624,7 @@ func TestSandboxIDFailure(t *testing.T) { func TestAddKernelParamValid(t *testing.T) { var config RuntimeConfig - expected := []vc.Param{ + expected := []hypervisor.Param{ { Key: "foo", Value: "bar", @@ -639,7 +640,7 @@ func TestAddKernelParamValid(t *testing.T) { func TestAddKernelParamInvalid(t *testing.T) { var config RuntimeConfig - invalid := []vc.Param{ + invalid := []hypervisor.Param{ { Key: "", Value: "bar", diff --git a/virtcontainers/pkg/vcmock/mock_test.go b/virtcontainers/pkg/vcmock/mock_test.go index d369e6574a..84f77e9f2b 100644 --- a/virtcontainers/pkg/vcmock/mock_test.go +++ b/virtcontainers/pkg/vcmock/mock_test.go @@ -13,6 +13,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/sirupsen/logrus" @@ -711,12 +712,12 @@ func TestVCMockSetVMFactory(t *testing.T) { m := &VCMock{} assert.Nil(m.SetFactoryFunc) - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: "foobar", ImagePath: "foobar", } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, HypervisorConfig: hyperConfig, } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 2220108385..38c0a657fa 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -23,6 +23,7 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" @@ -63,7 +64,7 @@ type qemu struct { store *store.VCStore - config HypervisorConfig + config hypervisor.Config qmpMonitorCh qmpChannel @@ -99,15 +100,10 @@ var qemuMajorVersion int var qemuMinorVersion int // agnostic list of kernel parameters -var defaultKernelParameters = []Param{ +var defaultKernelParameters = []hypervisor.Param{ {"panic", "1"}, } -const ( - addDevice operation = iota - removeDevice -) - type qmpLogger struct { logger *logrus.Entry } @@ -151,27 +147,27 @@ func (q *qemu) kernelParameters() string { params = append(params, defaultKernelParameters...) // set the maximum number of vCPUs - params = append(params, Param{"nr_cpus", fmt.Sprintf("%d", q.config.DefaultMaxVCPUs)}) + params = append(params, hypervisor.Param{"nr_cpus", fmt.Sprintf("%d", q.config.DefaultMaxVCPUs)}) // add the params specified by the provided config. As the kernel // honours the last parameter value set and since the config-provided // params are added here, they will take priority over the defaults. params = append(params, q.config.KernelParams...) - paramsStr := SerializeParams(params, "=") + paramsStr := hypervisor.SerializeParams(params, "=") return strings.Join(paramsStr, " ") } -// Adds all capabilities supported by qemu implementation of hypervisor interface -func (q *qemu) capabilities() types.Capabilities { +// Capabilities add all supported capabilities by QEMU +func (q *qemu) Capabilities() types.Capabilities { span, _ := q.trace("capabilities") defer span.Finish() return q.arch.capabilities() } -func (q *qemu) hypervisorConfig() HypervisorConfig { +func (q *qemu) Config() hypervisor.Config { return q.config } @@ -211,12 +207,11 @@ func (q *qemu) trace(name string) (opentracing.Span, context.Context) { } // setup sets the Qemu structure up. -func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (q *qemu) setup(id string, hypervisorConfig *hypervisor.Config, vcStore *store.VCStore) error { span, _ := q.trace("setup") defer span.Finish() - err := hypervisorConfig.valid() - if err != nil { + if err := hypervisorConfig.Valid(); err != nil { return err } @@ -259,7 +254,7 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto } } - nested, err := RunningOnVMM(procCPUInfo) + nested, err := hypervisor.RunningOnVMM(hypervisor.ProcCPUInfo) if err != nil { return err } @@ -286,7 +281,7 @@ func (q *qemu) cpuTopology() govmmQemu.SMP { } func (q *qemu) hostMemMB() (uint64, error) { - hostMemKb, err := getHostMemorySizeKb(procMemInfo) + hostMemKb, err := hypervisor.GetHostMemorySizeKb(hypervisor.ProcMemInfo) if err != nil { return 0, fmt.Errorf("Unable to read memory info: %s", err) } @@ -369,7 +364,7 @@ func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) { func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.IOThread, error) { var devices []govmmQemu.Device - console, err := q.getSandboxConsole(q.id) + console, err := q.GetSandboxConsole(q.id) if err != nil { return nil, nil, err } @@ -416,8 +411,8 @@ func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) g return incoming } -// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. -func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { +// CreateSandbox is the Hypervisor sandbox creation implementation for govmmQemu. +func (q *qemu) CreateSandbox(ctx context.Context, id string, hypervisorConfig *hypervisor.Config, store *store.VCStore) error { // Save the tracing context q.ctx = ctx @@ -535,14 +530,14 @@ func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *H return nil } -// startSandbox will start the Sandbox's VM. -func (q *qemu) startSandbox(timeout int) error { +// StartSandbox will start the Sandbox's VM. +func (q *qemu) StartSandbox(timeout int) error { span, _ := q.trace("startSandbox") defer span.Finish() if q.config.Debug { params := q.arch.kernelParameters(q.config.Debug) - strParams := SerializeParams(params, "=") + strParams := hypervisor.SerializeParams(params, "=") formatted := strings.Join(strParams, " ") // The name of this field matches a similar one generated by @@ -638,7 +633,7 @@ func (q *qemu) waitSandbox(timeout int) error { } // stopSandbox will stop the Sandbox's VM. -func (q *qemu) stopSandbox() error { +func (q *qemu) StopSandbox() error { span, _ := q.trace("stopSandbox") defer span.Finish() @@ -768,7 +763,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error { return err } -func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) error { +func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op hypervisor.Operation, devID string) error { var err error if q.config.BlockDeviceDriver == config.Nvdimm { @@ -831,7 +826,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev return nil } -func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error { +func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op hypervisor.Operation) error { err := q.qmpSetup() if err != nil { return err @@ -839,7 +834,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error devID := "virtio-" + drive.ID - if op == addDevice { + if op == hypervisor.AddDevice { err = q.hotplugAddBlockDevice(drive, op, devID) } else { if q.config.BlockDeviceDriver == config.VirtioBlock { @@ -860,7 +855,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error return err } -func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { +func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op hypervisor.Operation) error { err := q.qmpSetup() if err != nil { return err @@ -868,7 +863,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { devID := device.ID - if op == addDevice { + if op == hypervisor.AddDevice { // In case HotplugVFIOOnRootBus is true, devices are hotplugged on the root bus // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. @@ -933,7 +928,7 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) } -func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error { +func (q *qemu) hotplugNetDevice(endpoint Endpoint, op hypervisor.Operation) error { err := q.qmpSetup() if err != nil { return err @@ -952,7 +947,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error { return fmt.Errorf("this endpoint is not supported") } - if op == addDevice { + if op == hypervisor.AddDevice { if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { return err @@ -990,21 +985,21 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error { return nil } -func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) { +func (q *qemu) hotplugDevice(devInfo interface{}, devType hypervisor.Device, op hypervisor.Operation) (interface{}, error) { switch devType { - case blockDev: + case hypervisor.BlockDev: drive := devInfo.(*config.BlockDrive) return nil, q.hotplugBlockDevice(drive, op) - case cpuDev: + case hypervisor.CPUDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) - case vfioDev: + case hypervisor.VfioDev: device := devInfo.(*config.VFIODev) return nil, q.hotplugVFIODevice(device, op) - case memoryDev: - memdev := devInfo.(*memoryDevice) + case hypervisor.MemoryDev: + memdev := devInfo.(*hypervisor.MemoryDevice) return q.hotplugMemory(memdev, op) - case netDev: + case hypervisor.NetDev: device := devInfo.(Endpoint) return nil, q.hotplugNetDevice(device, op) default: @@ -1012,11 +1007,11 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati } } -func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (q *qemu) HotplugAddDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { span, _ := q.trace("hotplugAddDevice") defer span.Finish() - data, err := q.hotplugDevice(devInfo, devType, addDevice) + data, err := q.hotplugDevice(devInfo, devType, hypervisor.AddDevice) if err != nil { return data, err } @@ -1024,11 +1019,11 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf return data, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (q *qemu) HotplugRemoveDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { span, _ := q.trace("hotplugRemoveDevice") defer span.Finish() - data, err := q.hotplugDevice(devInfo, devType, removeDevice) + data, err := q.hotplugDevice(devInfo, devType, hypervisor.RemoveDevice) if err != nil { return data, err } @@ -1036,7 +1031,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int return data, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) { +func (q *qemu) hotplugCPUs(vcpus uint32, op hypervisor.Operation) (uint32, error) { if vcpus == 0 { q.Logger().Warnf("cannot hotplug 0 vCPUs") return 0, nil @@ -1047,7 +1042,7 @@ func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) { return 0, err } - if op == addDevice { + if op == hypervisor.AddDevice { return q.hotplugAddCPUs(vcpus) } @@ -1149,17 +1144,17 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) { return amount, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) { +func (q *qemu) hotplugMemory(memDev *hypervisor.MemoryDevice, op hypervisor.Operation) (int, error) { if !q.arch.supportGuestMemoryHotplug() { return 0, fmt.Errorf("guest memory hotplug not supported") } - if memDev.sizeMB < 0 { - return 0, fmt.Errorf("cannot hotplug negative size (%d) memory", memDev.sizeMB) + if memDev.SizeMB < 0 { + return 0, fmt.Errorf("cannot hotplug negative size (%d) memory", memDev.SizeMB) } memLog := q.Logger().WithField("hotplug", "memory") - memLog.WithField("hotplug-memory-mb", memDev.sizeMB).Debug("requested memory hotplug") + memLog.WithField("hotplug-memory-mb", memDev.SizeMB).Debug("requested memory hotplug") err := q.qmpSetup() if err != nil { return 0, err @@ -1167,29 +1162,29 @@ func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) { currentMemory := int(q.config.MemorySize) + q.state.HotpluggedMemory - if memDev.sizeMB == 0 { + if memDev.SizeMB == 0 { memLog.Debug("hotplug is not required") return 0, nil } switch op { - case removeDevice: - memLog.WithField("operation", "remove").Debugf("Requested to remove memory: %d MB", memDev.sizeMB) + case hypervisor.RemoveDevice: + memLog.WithField("hypervisor.Operation", "remove").Debugf("Requested to remove memory: %d MB", memDev.SizeMB) // Dont fail but warn that this is not supported. memLog.Warn("hot-remove VM memory not supported") return 0, nil - case addDevice: - memLog.WithField("operation", "add").Debugf("Requested to add memory: %d MB", memDev.sizeMB) + case hypervisor.AddDevice: + memLog.WithField("hypervisor.Operation", "add").Debugf("Requested to add memory: %d MB", memDev.SizeMB) maxMem, err := q.hostMemMB() if err != nil { return 0, err } // Don't exceed the maximum amount of memory - if currentMemory+memDev.sizeMB > int(maxMem) { + if currentMemory+memDev.SizeMB > int(maxMem) { // Fixme: return a typed error return 0, fmt.Errorf("Unable to hotplug %d MiB memory, the SB has %d MiB and the maximum amount is %d MiB", - memDev.sizeMB, currentMemory, maxMem) + memDev.SizeMB, currentMemory, maxMem) } memoryAdded, err := q.hotplugAddMemory(memDev) if err != nil { @@ -1202,33 +1197,33 @@ func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) { } -func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { +func (q *qemu) hotplugAddMemory(memDev *hypervisor.MemoryDevice) (int, error) { memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx) if err != nil { return 0, fmt.Errorf("failed to query memory devices: %v", err) } if len(memoryDevices) != 0 { - memDev.slot = memoryDevices[len(memoryDevices)-1].Data.Slot + 1 + memDev.Slot = memoryDevices[len(memoryDevices)-1].Data.Slot + 1 } - err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, "memory-backend-ram", "mem"+strconv.Itoa(memDev.slot), "", memDev.sizeMB) + err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, "memory-backend-ram", "mem"+strconv.Itoa(memDev.Slot), "", memDev.SizeMB) if err != nil { q.Logger().WithError(err).Error("hotplug memory") return 0, err } - q.state.HotpluggedMemory += memDev.sizeMB - return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state) + q.state.HotpluggedMemory += memDev.SizeMB + return memDev.SizeMB, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) pauseSandbox() error { +func (q *qemu) PauseSandbox() error { span, _ := q.trace("pauseSandbox") defer span.Finish() return q.togglePauseSandbox(true) } -func (q *qemu) resumeSandbox() error { +func (q *qemu) ResumeSandbox() error { span, _ := q.trace("resumeSandbox") defer span.Finish() @@ -1236,7 +1231,7 @@ func (q *qemu) resumeSandbox() error { } // addDevice will add extra devices to Qemu command line. -func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { +func (q *qemu) AddDevice(devInfo interface{}, devType hypervisor.Device) error { var err error span, _ := q.trace("addDevice") defer span.Finish() @@ -1264,16 +1259,16 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { return err } -// getSandboxConsole builds the path of the console where we can read +// GetSandboxConsole builds the path of the console where we can read // logs coming from the sandbox. -func (q *qemu) getSandboxConsole(id string) (string, error) { +func (q *qemu) GetSandboxConsole(id string) (string, error) { span, _ := q.trace("getSandboxConsole") defer span.Finish() return utils.BuildSocketPath(store.RunVMStoragePath, id, consoleSocket) } -func (q *qemu) saveSandbox() error { +func (q *qemu) SaveSandbox() error { q.Logger().Info("save sandbox") err := q.qmpSetup() @@ -1328,7 +1323,7 @@ func (q *qemu) saveSandbox() error { return nil } -func (q *qemu) disconnect() { +func (q *qemu) Disconnect() { span, _ := q.trace("disconnect") defer span.Finish() @@ -1362,10 +1357,10 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, return currentMemory, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotplugMB), + addMemDevice := &hypervisor.MemoryDevice{ + SizeMB: int(memHotplugMB), } - data, err := q.hotplugAddDevice(addMemDevice, memoryDev) + data, err := q.HotplugAddDevice(addMemDevice, hypervisor.MemoryDev) if err != nil { return currentMemory, err } @@ -1382,10 +1377,10 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, return currentMemory, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotunplugMB), + addMemDevice := &hypervisor.MemoryDevice{ + SizeMB: int(memHotunplugMB), } - data, err := q.hotplugRemoveDevice(addMemDevice, memoryDev) + data, err := q.HotplugRemoveDevice(addMemDevice, hypervisor.MemoryDev) if err != nil { return currentMemory, err } @@ -1488,7 +1483,7 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff return memory } -func (q *qemu) getThreadIDs() (*threadIDs, error) { +func (q *qemu) GetThreadIDs() (*hypervisor.ThreadIDs, error) { span, _ := q.trace("getThreadIDs") defer span.Finish() @@ -1503,10 +1498,10 @@ func (q *qemu) getThreadIDs() (*threadIDs, error) { return nil, err } - var tid threadIDs + var tid hypervisor.ThreadIDs for _, i := range cpuInfos { if i.ThreadID > 0 { - tid.vcpus = append(tid.vcpus, i.ThreadID) + tid.VCPUs = append(tid.VCPUs, i.ThreadID) } } return &tid, nil @@ -1521,7 +1516,7 @@ func calcHotplugMemMiBSize(mem uint32, memorySectionSizeMB uint32) (uint32, erro return uint32(math.Ceil(float64(mem)/float64(memorySectionSizeMB))) * memorySectionSizeMB, nil } -func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { +func (q *qemu) ResizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { currentVCPUs = q.config.NumVCPUs + uint32(len(q.state.HotpluggedVCPUs)) newVCPUs = currentVCPUs @@ -1529,7 +1524,7 @@ func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 case currentVCPUs < reqVCPUs: //hotplug addCPUs := reqVCPUs - currentVCPUs - data, err := q.hotplugAddDevice(addCPUs, cpuDev) + data, err := q.HotplugAddDevice(addCPUs, hypervisor.CPUDev) if err != nil { return currentVCPUs, newVCPUs, err } @@ -1541,7 +1536,7 @@ func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 case currentVCPUs > reqVCPUs: //hotunplug removeCPUs := currentVCPUs - reqVCPUs - data, err := q.hotplugRemoveDevice(removeCPUs, cpuDev) + data, err := q.HotplugRemoveDevice(removeCPUs, hypervisor.CPUDev) if err != nil { return currentVCPUs, newVCPUs, err } @@ -1554,7 +1549,7 @@ func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 return currentVCPUs, newVCPUs, nil } -func (q *qemu) cleanup() error { +func (q *qemu) Cleanup() error { span, _ := q.trace("cleanup") defer span.Finish() diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 65d9838e44..dec02b181c 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -8,6 +8,7 @@ package virtcontainers import ( "os" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" govmmQemu "github.com/intel/govmm/qemu" @@ -32,13 +33,13 @@ var qemuPaths = map[string]string{ QemuQ35: defaultQemuPath, } -var kernelRootParams = []Param{ +var kernelRootParams = []hypervisor.Param{ {"root", "/dev/pmem0p1"}, {"rootflags", "dax,data=ordered,errors=remount-ro rw"}, {"rootfstype", "ext4"}, } -var kernelParams = []Param{ +var kernelParams = []hypervisor.Param{ {"tsc", "reliable"}, {"no_timer_check", ""}, {"rcupdate.rcu_expedited", "1"}, @@ -80,7 +81,7 @@ func MaxQemuVCPUs() uint32 { return uint32(240) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config hypervisor.Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_amd64_test.go b/virtcontainers/qemu_amd64_test.go index 31e42341ad..3b62a7184c 100644 --- a/virtcontainers/qemu_amd64_test.go +++ b/virtcontainers/qemu_amd64_test.go @@ -12,12 +12,13 @@ import ( "testing" govmmQemu "github.com/intel/govmm/qemu" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := hypervisor.Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 5e68bfd292..48da96d941 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -14,6 +14,7 @@ import ( govmmQemu "github.com/intel/govmm/qemu" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -42,7 +43,7 @@ type qemuArch interface { // kernelParameters returns the kernel parameters // if debug is true then kernel debug parameters are included - kernelParameters(debug bool) []Param + kernelParameters(debug bool) []hypervisor.Param //capabilities returns the capabilities supported by QEMU capabilities() types.Capabilities @@ -96,7 +97,7 @@ type qemuArch interface { appendRNGDevice(devices []govmmQemu.Device, rngDevice config.RNGDev) []govmmQemu.Device // handleImagePath handles the Hypervisor Config image path - handleImagePath(config HypervisorConfig) + handleImagePath(config hypervisor.Config) // supportGuestMemoryHotplug returns if the guest supports memory hotplug supportGuestMemoryHotplug() bool @@ -110,9 +111,9 @@ type qemuArchBase struct { networkIndex int qemuPaths map[string]string supportedQemuMachines []govmmQemu.Machine - kernelParamsNonDebug []Param - kernelParamsDebug []Param - kernelParams []Param + kernelParamsNonDebug []hypervisor.Param + kernelParamsDebug []hypervisor.Param + kernelParams []hypervisor.Param } const ( @@ -152,27 +153,27 @@ const ( // kernelParamsNonDebug is a list of the default kernel // parameters that will be used in standard (non-debug) mode. -var kernelParamsNonDebug = []Param{ +var kernelParamsNonDebug = []hypervisor.Param{ {"quiet", ""}, } // kernelParamsSystemdNonDebug is a list of the default systemd related // kernel parameters that will be used in standard (non-debug) mode. -var kernelParamsSystemdNonDebug = []Param{ +var kernelParamsSystemdNonDebug = []hypervisor.Param{ {"systemd.show_status", "false"}, } // kernelParamsDebug is a list of the default kernel // parameters that will be used in debug mode (as much boot output as // possible). -var kernelParamsDebug = []Param{ +var kernelParamsDebug = []hypervisor.Param{ {"debug", ""}, } // kernelParamsSystemdDebug is a list of the default systemd related kernel // parameters that will be used in debug mode (as much boot output as // possible). -var kernelParamsSystemdDebug = []Param{ +var kernelParamsSystemdDebug = []hypervisor.Param{ {"systemd.show_status", "true"}, {"systemd.log_level", "debug"}, } @@ -216,7 +217,7 @@ func (q *qemuArchBase) qemuPath() (string, error) { return p, nil } -func (q *qemuArchBase) kernelParameters(debug bool) []Param { +func (q *qemuArchBase) kernelParameters(debug bool) []hypervisor.Param { params := q.kernelParams if debug { @@ -560,7 +561,7 @@ func (q *qemuArchBase) appendRNGDevice(devices []govmmQemu.Device, rngDev config return devices } -func (q *qemuArchBase) handleImagePath(config HypervisorConfig) { +func (q *qemuArchBase) handleImagePath(config hypervisor.Config) { if config.ImagePath != "" { q.kernelParams = append(q.kernelParams, kernelRootParams...) q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...) diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 2a5f551afe..8c284b1df1 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -29,18 +30,18 @@ var qemuArchBaseQemuPaths = map[string]string{ qemuArchBaseMachineType: qemuArchBaseQemuPath, } -var qemuArchBaseKernelParamsNonDebug = []Param{ +var qemuArchBaseKernelParamsNonDebug = []hypervisor.Param{ {"quiet", ""}, {"systemd.show_status", "false"}, } -var qemuArchBaseKernelParamsDebug = []Param{ +var qemuArchBaseKernelParamsDebug = []hypervisor.Param{ {"debug", ""}, {"systemd.show_status", "true"}, {"systemd.log_level", "debug"}, } -var qemuArchBaseKernelParams = []Param{ +var qemuArchBaseKernelParams = []hypervisor.Param{ {"root", "/dev/vda"}, {"rootfstype", "ext4"}, } @@ -120,15 +121,15 @@ func TestQemuArchBaseKernelParameters(t *testing.T) { qemuArchBase := newQemuArchBase() // with debug params - expectedParams := []Param(qemuArchBaseKernelParams) - debugParams := []Param(qemuArchBaseKernelParamsDebug) + expectedParams := []hypervisor.Param(qemuArchBaseKernelParams) + debugParams := []hypervisor.Param(qemuArchBaseKernelParamsDebug) expectedParams = append(expectedParams, debugParams...) p := qemuArchBase.kernelParameters(true) assert.Equal(expectedParams, p) // with non-debug params - expectedParams = []Param(qemuArchBaseKernelParams) - nonDebugParams := []Param(qemuArchBaseKernelParamsNonDebug) + expectedParams = []hypervisor.Param(qemuArchBaseKernelParams) + nonDebugParams := []hypervisor.Param(qemuArchBaseKernelParamsNonDebug) expectedParams = append(expectedParams, nonDebugParams...) p = qemuArchBase.kernelParameters(false) assert.Equal(expectedParams, p) @@ -168,10 +169,10 @@ func TestQemuArchBaseCPUTopology(t *testing.T) { Sockets: vcpus, Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: defaultMaxQemuVCPUs, + MaxCPUs: hypervisor.DefaultMaxQemuVCPUs, } - smp := qemuArchBase.cpuTopology(vcpus, defaultMaxQemuVCPUs) + smp := qemuArchBase.cpuTopology(vcpus, hypervisor.DefaultMaxQemuVCPUs) assert.Equal(expectedSMP, smp) } diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 6c79a9297a..9f23c21ca7 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -15,27 +15,28 @@ import ( "testing" govmmQemu "github.com/intel/govmm/qemu" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) -func newQemuConfig() HypervisorConfig { - return HypervisorConfig{ +func newQemuConfig() hypervisor.Config { + return hypervisor.Config{ KernelPath: testQemuKernelPath, ImagePath: testQemuImagePath, InitrdPath: testQemuInitrdPath, HypervisorPath: testQemuPath, - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: hypervisor.DefaultVCPUs, + MemorySize: hypervisor.DefaultMemSzMiB, + DefaultBridges: hypervisor.DefaultBridges, + BlockDeviceDriver: hypervisor.DefaultBlockDriver, + DefaultMaxVCPUs: hypervisor.DefaultMaxQemuVCPUs, + Msize9p: hypervisor.DefaultMsize9p, } } -func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected string, debug bool) { +func testQemuKernelParameters(t *testing.T, kernelParams []hypervisor.Param, expected string, debug bool) { qemuConfig := newQemuConfig() qemuConfig.KernelParams = kernelParams @@ -56,7 +57,7 @@ func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected strin func TestQemuKernelParameters(t *testing.T) { expectedOut := fmt.Sprintf("panic=1 nr_cpus=%d foo=foo bar=bar", MaxQemuVCPUs()) - params := []Param{ + params := []hypervisor.Param{ { Key: "foo", Value: "foo", @@ -102,7 +103,7 @@ func TestQemuCreateSandbox(t *testing.T) { t.Fatalf("Could not create parent directory %s: %v", parentDir, err) } - if err := q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + if err := q.CreateSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { t.Fatal(err) } @@ -146,7 +147,7 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) { t.Fatal(err) } - if err := q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + if err := q.CreateSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { t.Fatalf("Qemu createSandbox() is not expected to fail because of missing parent directory for storage: %v", err) } } @@ -156,7 +157,7 @@ func TestQemuCPUTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, - config: HypervisorConfig{ + config: hypervisor.Config{ NumVCPUs: uint32(vcpus), DefaultMaxVCPUs: uint32(vcpus), }, @@ -183,13 +184,13 @@ func TestQemuMemoryTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, - config: HypervisorConfig{ + config: hypervisor.Config{ MemorySize: mem, MemSlots: slots, }, } - hostMemKb, err := getHostMemorySizeKb(procMemInfo) + hostMemKb, err := hypervisor.GetHostMemorySizeKb(hypervisor.ProcMemInfo) if err != nil { t.Fatal(err) } @@ -211,13 +212,13 @@ func TestQemuMemoryTopology(t *testing.T) { } } -func testQemuAddDevice(t *testing.T, devInfo interface{}, devType deviceType, expected []govmmQemu.Device) { +func testQemuAddDevice(t *testing.T, devInfo interface{}, devType hypervisor.Device, expected []govmmQemu.Device) { q := &qemu{ ctx: context.Background(), arch: &qemuArchBase{}, } - err := q.addDevice(devInfo, devType) + err := q.AddDevice(devInfo, devType) if err != nil { t.Fatal(err) } @@ -247,7 +248,7 @@ func TestQemuAddDeviceFsDev(t *testing.T) { HostPath: hostPath, } - testQemuAddDevice(t, volume, fsDev, expectedOut) + testQemuAddDevice(t, volume, hypervisor.FsDev, expectedOut) } func TestQemuAddDeviceSerialPortDev(t *testing.T) { @@ -274,7 +275,7 @@ func TestQemuAddDeviceSerialPortDev(t *testing.T) { Name: name, } - testQemuAddDevice(t, socket, serialPortDev, expectedOut) + testQemuAddDevice(t, socket, hypervisor.SerialPortDev, expectedOut) } func TestQemuAddDeviceKataVSOCK(t *testing.T) { @@ -296,7 +297,7 @@ func TestQemuAddDeviceKataVSOCK(t *testing.T) { vhostFd: vHostFD, } - testQemuAddDevice(t, vsock, vSockPCIDev, expectedOut) + testQemuAddDevice(t, vsock, hypervisor.VSockPCIDev, expectedOut) } func TestQemuGetSandboxConsole(t *testing.T) { @@ -306,7 +307,7 @@ func TestQemuGetSandboxConsole(t *testing.T) { sandboxID := "testSandboxID" expected := filepath.Join(store.RunVMStoragePath, sandboxID, consoleSocket) - result, err := q.getSandboxConsole(sandboxID) + result, err := q.GetSandboxConsole(sandboxID) if err != nil { t.Fatal(err) } @@ -322,7 +323,7 @@ func TestQemuCapabilities(t *testing.T) { arch: &qemuArchBase{}, } - caps := q.capabilities() + caps := q.Capabilities() if !caps.IsBlockDeviceHotplugSupported() { t.Fatal("Block device hotplug should be supported") } @@ -392,9 +393,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) { } q.store = vcStore - _, err = q.hotplugAddDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.HotplugAddDevice(&hypervisor.MemoryDevice{0, 128}, hypervisor.FsDev) assert.Error(err) - _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.HotplugRemoveDevice(&hypervisor.MemoryDevice{0, 128}, hypervisor.FsDev) assert.Error(err) } @@ -421,6 +422,6 @@ func TestQemuCleanup(t *testing.T) { config: newQemuConfig(), } - err := q.cleanup() + err := q.Cleanup() assert.Nil(err) } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 1e1e9fc6bf..35440641e4 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -24,6 +24,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -41,8 +42,8 @@ const ( type SandboxStatus struct { ID string State types.State - Hypervisor HypervisorType - HypervisorConfig HypervisorConfig + Hypervisor hypervisor.Type + HypervisorConfig hypervisor.Config Agent AgentType ContainersStatus []ContainerStatus @@ -58,8 +59,8 @@ type SandboxConfig struct { Hostname string - HypervisorType HypervisorType - HypervisorConfig HypervisorConfig + HypervisorType hypervisor.Type + HypervisorConfig hypervisor.Config AgentType AgentType AgentConfig interface{} @@ -130,8 +131,8 @@ func (sandboxConfig *SandboxConfig) valid() bool { return false } - if _, err := newHypervisor(sandboxConfig.HypervisorType); err != nil { - sandboxConfig.HypervisorType = QemuHypervisor + if _, err := hypervisor.New(sandboxConfig.HypervisorType); err != nil { + sandboxConfig.HypervisorType = hypervisor.Qemu } return true @@ -144,7 +145,7 @@ type Sandbox struct { sync.Mutex factory Factory - hypervisor hypervisor + hypervisor hypervisor.Hypervisor agent agent store *store.VCStore network Network @@ -257,7 +258,7 @@ func (s *Sandbox) Release() error { if s.monitor != nil { s.monitor.stop() } - s.hypervisor.disconnect() + s.hypervisor.Disconnect() return s.agent.disconnect() } @@ -391,7 +392,7 @@ func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error { } for _, a := range []*types.Asset{kernel, image, initrd} { - if err := sandboxConfig.HypervisorConfig.addCustomAsset(a); err != nil { + if err := sandboxConfig.HypervisorConfig.AddCustomAsset(a); err != nil { return err } } @@ -484,7 +485,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor agent := newAgent(sandboxConfig.AgentType) - hypervisor, err := newHypervisor(sandboxConfig.HypervisorType) + hypervisor, err := hypervisor.New(sandboxConfig.HypervisorType) if err != nil { return nil, err } @@ -532,7 +533,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor } }() - if err = s.hypervisor.createSandbox(ctx, s.id, &sandboxConfig.HypervisorConfig, s.store); err != nil { + if err = s.hypervisor.CreateSandbox(ctx, s.id, &sandboxConfig.HypervisorConfig, s.store); err != nil { return nil, err } @@ -714,7 +715,7 @@ func (s *Sandbox) Delete() error { s.monitor.stop() } - if err := s.hypervisor.cleanup(); err != nil { + if err := s.hypervisor.Cleanup(); err != nil { s.Logger().WithError(err).Error("failed to cleanup hypervisor") } @@ -927,7 +928,7 @@ func (s *Sandbox) startVM() error { return nil } - return s.hypervisor.startSandbox(vmStartTimeout) + return s.hypervisor.StartSandbox(vmStartTimeout) }); err != nil { return err } @@ -978,7 +979,7 @@ func (s *Sandbox) stopVM() error { } s.Logger().Info("Stopping VM") - return s.hypervisor.stopSandbox() + return s.hypervisor.StopSandbox() } func (s *Sandbox) addContainer(c *Container) error { @@ -1329,7 +1330,7 @@ func (s *Sandbox) Stop() error { // Pause pauses the sandbox func (s *Sandbox) Pause() error { - if err := s.hypervisor.pauseSandbox(); err != nil { + if err := s.hypervisor.PauseSandbox(); err != nil { return err } @@ -1346,7 +1347,7 @@ func (s *Sandbox) Pause() error { // Resume resumes the sandbox func (s *Sandbox) Resume() error { - if err := s.hypervisor.resumeSandbox(); err != nil { + if err := s.hypervisor.ResumeSandbox(); err != nil { return err } @@ -1500,7 +1501,7 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) // adding a group of VFIO devices for _, dev := range vfioDevices { - if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil { + if _, err := s.hypervisor.HotplugAddDevice(dev, hypervisor.VfioDev); err != nil { s.Logger(). WithFields(logrus.Fields{ "sandbox": s.id, @@ -1516,7 +1517,7 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) if !ok { return fmt.Errorf("device type mismatch, expect device type to be %s", devType) } - _, err := s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev) + _, err := s.hypervisor.HotplugAddDevice(blockDevice.BlockDrive, hypervisor.BlockDev) return err case config.DeviceGeneric: // TODO: what? @@ -1537,7 +1538,7 @@ func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceTy // remove a group of VFIO devices for _, dev := range vfioDevices { - if _, err := s.hypervisor.hotplugRemoveDevice(dev, vfioDev); err != nil { + if _, err := s.hypervisor.HotplugRemoveDevice(dev, hypervisor.VfioDev); err != nil { s.Logger().WithError(err). WithFields(logrus.Fields{ "sandbox": s.id, @@ -1553,7 +1554,7 @@ func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceTy if !ok { return fmt.Errorf("device type mismatch, expect device type to be %s", devType) } - _, err := s.hypervisor.hotplugRemoveDevice(blockDrive, blockDev) + _, err := s.hypervisor.HotplugRemoveDevice(blockDrive, hypervisor.BlockDev) return err case config.DeviceGeneric: // TODO: what? @@ -1580,7 +1581,7 @@ func (s *Sandbox) DecrementSandboxBlockIndex() error { func (s *Sandbox) AppendDevice(device api.Device) error { switch device.DeviceType() { case config.VhostUserSCSI, config.VhostUserNet, config.VhostUserBlk: - return s.hypervisor.addDevice(device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), vhostuserDev) + return s.hypervisor.AddDevice(device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), hypervisor.VhostuserDev) } return fmt.Errorf("unsupported device type") } @@ -1634,14 +1635,14 @@ func (s *Sandbox) updateResources() error { } sandboxVCPUs := uint32(utils.ConstraintsToVCPUs(*sumResources.CPU.Quota, *sumResources.CPU.Period)) - sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs + sandboxVCPUs += s.hypervisor.Config().NumVCPUs - sandboxMemoryByte := int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift + sandboxMemoryByte := int64(s.hypervisor.Config().MemorySize) << utils.MibToBytesShift sandboxMemoryByte += *sumResources.Memory.Limit // Update VCPUs s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs") - oldCPUs, newCPUs, err := s.hypervisor.resizeVCPUs(sandboxVCPUs) + oldCPUs, newCPUs, err := s.hypervisor.ResizeVCPUs(sandboxVCPUs) if err != nil { return err } @@ -1656,7 +1657,7 @@ func (s *Sandbox) updateResources() error { // Update Memory s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") - newMemory, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB) + newMemory, err := s.hypervisor.ResizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB) if err != nil { return err } diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index e5b361b035..7bf72161a2 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -23,14 +23,15 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "golang.org/x/sys/unix" ) -func newHypervisorConfig(kernelParams []Param, hParams []Param) HypervisorConfig { - return HypervisorConfig{ +func newHypervisorConfig(kernelParams []hypervisor.Param, hParams []hypervisor.Param) hypervisor.Config { + return hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -41,7 +42,7 @@ func newHypervisorConfig(kernelParams []Param, hParams []Param) HypervisorConfig } func testCreateSandbox(t *testing.T, id string, - htype HypervisorType, hconfig HypervisorConfig, atype AgentType, + htype hypervisor.Type, hconfig hypervisor.Config, atype AgentType, nconfig NetworkConfig, containers []ContainerConfig, volumes []types.Volume) (*Sandbox, error) { @@ -80,7 +81,7 @@ func testCreateSandbox(t *testing.T, id string, } func TestCreateEmptySandbox(t *testing.T) { - _, err := testCreateSandbox(t, testSandboxID, MockHypervisor, HypervisorConfig{}, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hypervisor.Config{}, NoopAgentType, NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("VirtContainers should not allow empty sandboxes") } @@ -88,7 +89,7 @@ func TestCreateEmptySandbox(t *testing.T) { } func TestCreateEmptyHypervisorSandbox(t *testing.T) { - _, err := testCreateSandbox(t, testSandboxID, QemuHypervisor, HypervisorConfig{}, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Qemu, hypervisor.Config{}, NoopAgentType, NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("VirtContainers should not allow sandboxes with empty hypervisors") } @@ -98,7 +99,7 @@ func TestCreateEmptyHypervisorSandbox(t *testing.T) { func TestCreateMockSandbox(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - _, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) if err != nil { t.Fatal(err) } @@ -108,7 +109,7 @@ func TestCreateMockSandbox(t *testing.T) { func TestCreateSandboxEmptyID(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, "", MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, "", hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("Expected sandbox with empty ID to fail, but got sandbox %v", p) } @@ -118,7 +119,7 @@ func TestCreateSandboxEmptyID(t *testing.T) { func testSandboxStateTransition(t *testing.T, state types.StateString, newState types.StateString) error { hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) if err != nil { return err } @@ -450,7 +451,7 @@ func TestSandboxSetSandboxAndContainerState(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - p, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } @@ -648,7 +649,7 @@ func TestSandboxGetContainer(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) if err != nil { t.Fatal(err) } @@ -694,7 +695,7 @@ func TestContainerSetStateBlockIndex(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - sandbox, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) + sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) if err != nil { t.Fatal(err) } @@ -789,7 +790,7 @@ func TestContainerStateSetFstype(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - sandbox, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) + sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) if err != nil { t.Fatal(err) } @@ -967,7 +968,7 @@ func TestSandboxCreateAssets(t *testing.T) { originalKernelPath := filepath.Join(testDir, testKernel) - hc := HypervisorConfig{ + hc := hypervisor.Config{ KernelPath: originalKernelPath, ImagePath: filepath.Join(testDir, testImage), } @@ -984,9 +985,9 @@ func TestSandboxCreateAssets(t *testing.T) { err = createAssets(context.Background(), p) assert.Nil(err) - a, ok := p.HypervisorConfig.customAssets[types.KernelAsset] - assert.True(ok) - assert.Equal(a.Path(), tmpfile.Name()) + path, err := p.HypervisorConfig.KernelAssetPath() + assert.Nil(err) + assert.Equal(path, tmpfile.Name()) p = &SandboxConfig{ Annotations: map[string]string{ @@ -1061,7 +1062,7 @@ func TestRemoveContainerSuccess(t *testing.T) { } func TestCreateContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1072,7 +1073,7 @@ func TestCreateContainer(t *testing.T) { } func TestDeleteContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1089,7 +1090,7 @@ func TestDeleteContainer(t *testing.T) { } func TestStartContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1109,7 +1110,7 @@ func TestStartContainer(t *testing.T) { } func TestStatusContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1129,7 +1130,7 @@ func TestStatusContainer(t *testing.T) { } func TestStatusSandbox(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1137,7 +1138,7 @@ func TestStatusSandbox(t *testing.T) { } func TestEnterContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1161,7 +1162,7 @@ func TestEnterContainer(t *testing.T) { } func TestMonitor(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1181,7 +1182,7 @@ func TestMonitor(t *testing.T) { } func TestWaitProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1211,7 +1212,7 @@ func TestWaitProcess(t *testing.T) { } func TestSignalProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1241,7 +1242,7 @@ func TestSignalProcess(t *testing.T) { } func TestWinsizeProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1271,7 +1272,7 @@ func TestWinsizeProcess(t *testing.T) { } func TestContainerProcessIOStream(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1301,9 +1302,9 @@ func TestContainerProcessIOStream(t *testing.T) { } func TestAttachBlockDevice(t *testing.T) { - hypervisor := &mockHypervisor{} + h := &mockHypervisor{} - hConfig := HypervisorConfig{ + hConfig := hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, } @@ -1313,7 +1314,7 @@ func TestAttachBlockDevice(t *testing.T) { sandbox := &Sandbox{ id: testSandboxID, - hypervisor: hypervisor, + hypervisor: h, config: sconfig, ctx: context.Background(), } @@ -1389,9 +1390,9 @@ func TestAttachBlockDevice(t *testing.T) { } func TestPreAddDevice(t *testing.T) { - hypervisor := &mockHypervisor{} + h := &mockHypervisor{} - hConfig := HypervisorConfig{ + hConfig := hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, } @@ -1403,7 +1404,7 @@ func TestPreAddDevice(t *testing.T) { // create a sandbox first sandbox := &Sandbox{ id: testSandboxID, - hypervisor: hypervisor, + hypervisor: h, config: sconfig, devManager: dm, ctx: context.Background(), diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go index c3cd3c34cc..9535a2b59d 100644 --- a/virtcontainers/tap_endpoint.go +++ b/virtcontainers/tap_endpoint.go @@ -11,6 +11,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" ) @@ -63,7 +64,7 @@ func (endpoint *TapEndpoint) SetProperties(properties NetworkInfo) { } // Attach for tap endpoint adds the tap interface to the hypervisor. -func (endpoint *TapEndpoint) Attach(h hypervisor) error { +func (endpoint *TapEndpoint) Attach(h hypervisor.Hypervisor) error { return fmt.Errorf("TapEndpoint does not support Attach, if you're using docker please use --net none") } @@ -80,14 +81,14 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error { } // HotAttach for the tap endpoint uses hot plug device -func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { +func (endpoint *TapEndpoint) HotAttach(h hypervisor.Hypervisor) error { networkLogger().Info("Hot attaching tap endpoint") - if err := tapNetwork(endpoint, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := tapNetwork(endpoint, h.Config().NumVCPUs, h.Config().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging tap ep") return err } - if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugAddDevice(endpoint, hypervisor.NetDev); err != nil { networkLogger().WithError(err).Error("Error attach tap ep") return err } @@ -95,7 +96,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { } // HotDetach for the tap endpoint uses hot pull device -func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *TapEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { networkLogger().Info("Hot detaching tap endpoint") if err := doNetNS(netNsPath, func(_ ns.NetNS) error { return unTapNetwork(endpoint.TapInterface.TAPIface.Name) @@ -103,7 +104,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat networkLogger().WithError(err).Warn("Error un-bridging tap ep") } - if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugRemoveDevice(endpoint, hypervisor.NetDev); err != nil { networkLogger().WithError(err).Error("Error detach tap ep") return err } diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go index 66994a1497..aaad8a3132 100644 --- a/virtcontainers/veth_endpoint.go +++ b/virtcontainers/veth_endpoint.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) // VethEndpoint gathers a network pair and its properties. @@ -87,13 +88,13 @@ func (endpoint *VethEndpoint) SetProperties(properties NetworkInfo) { // Attach for veth endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *VethEndpoint) Attach(h hypervisor) error { +func (endpoint *VethEndpoint) Attach(h hypervisor.Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual endpoint") return err } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, hypervisor.NetDev) } // Detach for the veth endpoint tears down the tap and bridge @@ -111,13 +112,13 @@ func (endpoint *VethEndpoint) Detach(netNsCreated bool, netNsPath string) error } // HotAttach for the veth endpoint uses hot plug device -func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { +func (endpoint *VethEndpoint) HotAttach(h hypervisor.Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugAddDevice(endpoint, hypervisor.NetDev); err != nil { networkLogger().WithError(err).Error("Error attach virtual ep") return err } @@ -125,7 +126,7 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { } // HotDetach for the veth endpoint uses hot pull device -func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VethEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { if !netNsCreated { return nil } @@ -136,7 +137,7 @@ func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPa networkLogger().WithError(err).Warn("Error un-bridging virtual ep") } - if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugRemoveDevice(endpoint, hypervisor.NetDev); err != nil { networkLogger().WithError(err).Error("Error detach virtual ep") return err } diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go index 4960dba1be..4c1f9a43c5 100644 --- a/virtcontainers/vhostuser_endpoint.go +++ b/virtcontainers/vhostuser_endpoint.go @@ -11,6 +11,7 @@ import ( "os" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -73,7 +74,7 @@ func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair { } // Attach for vhostuser endpoint -func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { +func (endpoint *VhostUserEndpoint) Attach(h hypervisor.Hypervisor) error { // Generate a unique ID to be used for hypervisor commandline fields randBytes, err := utils.GenerateRandomBytes(8) if err != nil { @@ -88,7 +89,7 @@ func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { Type: config.VhostUserNet, } - return h.addDevice(d, vhostuserDev) + return h.AddDevice(d, hypervisor.VhostuserDev) } // Detach for vhostuser endpoint @@ -97,12 +98,12 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e } // HotAttach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error { +func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor.Hypervisor) error { return fmt.Errorf("VhostUserEndpoint does not support Hot attach") } // HotDetach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("VhostUserEndpoint does not support Hot detach") } diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index e04f521727..a023c6f3d5 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -11,6 +11,7 @@ import ( "path/filepath" "time" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/sirupsen/logrus" @@ -20,7 +21,7 @@ import ( type VM struct { id string - hypervisor hypervisor + hypervisor hypervisor.Hypervisor agent agent proxy proxy @@ -37,8 +38,8 @@ type VM struct { // VMConfig is a collection of all info that a new blackbox VM needs. type VMConfig struct { - HypervisorType HypervisorType - HypervisorConfig HypervisorConfig + HypervisorType hypervisor.Type + HypervisorConfig hypervisor.Config AgentType AgentType AgentConfig interface{} @@ -49,11 +50,11 @@ type VMConfig struct { // Valid check VMConfig validity. func (c *VMConfig) Valid() error { - return c.HypervisorConfig.valid() + return c.HypervisorConfig.Valid() } -func setupProxy(h hypervisor, agent agent, config VMConfig, id string) (int, string, proxy, error) { - consoleURL, err := h.getSandboxConsole(id) +func setupProxy(h hypervisor.Hypervisor, agent agent, config VMConfig, id string) (int, string, proxy, error) { + consoleURL, err := h.GetSandboxConsole(id) if err != nil { return -1, "", nil, err } @@ -102,7 +103,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { ) // 1. setup hypervisor - hypervisor, err := newHypervisor(config.HypervisorType) + hypervisor, err := hypervisor.New(config.HypervisorType) if err != nil { return nil, err } @@ -130,7 +131,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { } }() - if err = hypervisor.createSandbox(ctx, id, &config.HypervisorConfig, vcStore); err != nil { + if err = hypervisor.CreateSandbox(ctx, id, &config.HypervisorConfig, vcStore); err != nil { return nil, err } @@ -143,14 +144,14 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { } // 3. boot up guest vm - if err = hypervisor.startSandbox(vmStartTimeout); err != nil { + if err = hypervisor.StartSandbox(vmStartTimeout); err != nil { return nil, err } defer func() { if err != nil { virtLog.WithField("vm", id).WithError(err).Info("clean up vm") - hypervisor.stopSandbox() + hypervisor.StopSandbox() } }() @@ -203,25 +204,25 @@ func (v *VM) logger() logrus.FieldLogger { // Pause pauses a VM. func (v *VM) Pause() error { v.logger().Info("pause vm") - return v.hypervisor.pauseSandbox() + return v.hypervisor.PauseSandbox() } // Save saves a VM to persistent disk. func (v *VM) Save() error { v.logger().Info("save vm") - return v.hypervisor.saveSandbox() + return v.hypervisor.SaveSandbox() } // Resume resumes a paused VM. func (v *VM) Resume() error { v.logger().Info("resume vm") - return v.hypervisor.resumeSandbox() + return v.hypervisor.ResumeSandbox() } // Start kicks off a configured VM. func (v *VM) Start() error { v.logger().Info("start vm") - return v.hypervisor.startSandbox(vmStartTimeout) + return v.hypervisor.StartSandbox(vmStartTimeout) } // Disconnect agent and proxy connections to a VM @@ -242,7 +243,7 @@ func (v *VM) Disconnect() error { func (v *VM) Stop() error { v.logger().Info("stop vm") - if err := v.hypervisor.stopSandbox(); err != nil { + if err := v.hypervisor.StopSandbox(); err != nil { return err } @@ -253,7 +254,7 @@ func (v *VM) Stop() error { func (v *VM) AddCPUs(num uint32) error { if num > 0 { v.logger().Infof("hot adding %d vCPUs", num) - if _, err := v.hypervisor.hotplugAddDevice(num, cpuDev); err != nil { + if _, err := v.hypervisor.HotplugAddDevice(num, hypervisor.CPUDev); err != nil { return err } v.cpuDelta += num @@ -267,8 +268,11 @@ func (v *VM) AddCPUs(num uint32) error { func (v *VM) AddMemory(numMB uint32) error { if numMB > 0 { v.logger().Infof("hot adding %d MB memory", numMB) - dev := &memoryDevice{1, int(numMB)} - if _, err := v.hypervisor.hotplugAddDevice(dev, memoryDev); err != nil { + dev := &hypervisor.MemoryDevice{ + Slot: 1, + SizeMB: int(numMB), + } + if _, err := v.hypervisor.HotplugAddDevice(dev, hypervisor.MemoryDev); err != nil { return err } } diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go index 40a9729d6b..53cf47897a 100644 --- a/virtcontainers/vm_test.go +++ b/virtcontainers/vm_test.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/stretchr/testify/assert" ) @@ -18,11 +19,11 @@ func TestNewVM(t *testing.T) { testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") config := VMConfig{ - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: NoopAgentType, ProxyType: NoopProxyType, } - hyperConfig := HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } @@ -82,7 +83,7 @@ func TestVMConfigValid(t *testing.T) { assert.Error(err) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config.HypervisorConfig = HypervisorConfig{ + config.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, InitrdPath: testDir, } @@ -94,7 +95,7 @@ func TestSetupProxy(t *testing.T) { assert := assert.New(t) config := VMConfig{ - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: NoopAgentType, } From 1eb9eff21311e62e95123251bdc3007ec0b3bce5 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Thu, 31 Jan 2019 14:20:27 +0100 Subject: [PATCH 2/7] virtcontainers: hypervisor: Move endpoints under hypervisor There is currently a cyclic logical dependency between network, endpoint and hypervisor structures, types and APIs. Hypervisor interface implementations call into the Endpoint interface, which itself calls into the Network* APIs. Since we want to move all hypervisor implementations under the hypervisor package, we need to: - Split the network implementation into 3 pieces: Types definitions, core namespace API and Endpoints. - Move the Endpoint handling under the hypervisor package. This could be splitted further if we manage to decouple the Hypervisor interface from the Endpoint one, and vice-versa. Fixes: #1229 Signed-off-by: Samuel Ortiz --- pkg/katautils/config.go | 3 +- pkg/katautils/config_test.go | 17 +- pkg/katautils/network.go | 4 +- pkg/katautils/network_test.go | 12 +- virtcontainers/api_test.go | 6 +- virtcontainers/endpoint.go | 106 -- virtcontainers/fc.go | 4 +- virtcontainers/hack/virtc/main.go | 2 +- virtcontainers/hyperstart_agent.go | 4 +- virtcontainers/hyperstart_agent_test.go | 4 +- .../bridgedmacvlan_endpoint.go | 26 +- .../bridgedmacvlan_endpoint_test.go | 16 +- virtcontainers/hypervisor/endpoint.go | 794 ++++++++++ .../{ => hypervisor}/endpoint_test.go | 2 +- .../{ => hypervisor}/ipvlan_endpoint.go | 26 +- .../{ => hypervisor}/ipvlan_endpoint_test.go | 14 +- .../{ => hypervisor}/macvtap_endpoint.go | 24 +- .../{ => hypervisor}/macvtap_endpoint_test.go | 8 +- virtcontainers/hypervisor/network.go | 387 +++++ virtcontainers/hypervisor/network_test.go | 166 ++ .../{ => hypervisor}/physical_endpoint.go | 22 +- .../physical_endpoint_test.go | 4 +- .../{ => hypervisor}/tap_endpoint.go | 32 +- .../{ => hypervisor}/veth_endpoint.go | 32 +- .../{ => hypervisor}/veth_endpoint_test.go | 30 +- .../{ => hypervisor}/vhostuser_endpoint.go | 26 +- .../vhostuser_endpoint_test.go | 12 +- virtcontainers/iostream_test.go | 3 +- virtcontainers/kata_agent.go | 2 +- virtcontainers/monitor_test.go | 5 +- virtcontainers/network.go | 1335 +---------------- virtcontainers/network_test.go | 208 +-- virtcontainers/pkg/oci/utils.go | 10 +- virtcontainers/pkg/oci/utils_test.go | 2 +- virtcontainers/qemu.go | 16 +- virtcontainers/qemu_arch_base.go | 14 +- virtcontainers/qemu_arch_base_test.go | 22 +- virtcontainers/sandbox.go | 18 +- virtcontainers/sandbox_test.go | 46 +- virtcontainers/types/network.go | 175 +++ virtcontainers/types/network_test.go | 57 + 41 files changed, 1893 insertions(+), 1803 deletions(-) delete mode 100644 virtcontainers/endpoint.go rename virtcontainers/{ => hypervisor}/bridgedmacvlan_endpoint.go (78%) rename virtcontainers/{ => hypervisor}/bridgedmacvlan_endpoint_test.go (69%) create mode 100644 virtcontainers/hypervisor/endpoint.go rename virtcontainers/{ => hypervisor}/endpoint_test.go (98%) rename virtcontainers/{ => hypervisor}/ipvlan_endpoint.go (78%) rename virtcontainers/{ => hypervisor}/ipvlan_endpoint_test.go (76%) rename virtcontainers/{ => hypervisor}/macvtap_endpoint.go (77%) rename virtcontainers/{ => hypervisor}/macvtap_endpoint_test.go (78%) create mode 100644 virtcontainers/hypervisor/network.go create mode 100644 virtcontainers/hypervisor/network_test.go rename virtcontainers/{ => hypervisor}/physical_endpoint.go (87%) rename virtcontainers/{ => hypervisor}/physical_endpoint_test.go (96%) rename virtcontainers/{ => hypervisor}/tap_endpoint.go (84%) rename virtcontainers/{ => hypervisor}/veth_endpoint.go (77%) rename virtcontainers/{ => hypervisor}/veth_endpoint_test.go (72%) rename virtcontainers/{ => hypervisor}/vhostuser_endpoint.go (81%) rename virtcontainers/{ => hypervisor}/vhostuser_endpoint_test.go (93%) create mode 100644 virtcontainers/types/network.go create mode 100644 virtcontainers/types/network_test.go diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index f4ab1206cf..0a37065253 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -18,6 +18,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" ) @@ -894,7 +895,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error { if config.NetmonConfig.Enable { return fmt.Errorf("config disable_new_netns conflicts with enable_netmon") } - if config.InterNetworkModel != vc.NetXConnectNoneModel { + if config.InterNetworkModel != types.NetXConnectNoneModel { return fmt.Errorf("config disable_new_netns only works with 'none' internetworking_model") } } else if config.ShimConfig.(vc.ShimConfig).Trace { diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index ac82b950eb..7a57fe2862 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -23,6 +23,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/stretchr/testify/assert" ) @@ -1532,7 +1533,7 @@ func TestCheckNetNsConfig(t *testing.T) { config = oci.RuntimeConfig{ DisableNewNetNs: true, - InterNetworkModel: vc.NetXConnectDefaultModel, + InterNetworkModel: types.NetXConnectDefaultModel, } err = checkNetNsConfig(config) assert.Error(err) @@ -1584,18 +1585,18 @@ func TestCheckNetNsConfigShimTrace(t *testing.T) { type testData struct { disableNetNs bool - networkModel vc.NetInterworkingModel + networkModel types.NetInterworkingModel shimTrace bool expectError bool } data := []testData{ - {false, vc.NetXConnectMacVtapModel, false, false}, - {false, vc.NetXConnectMacVtapModel, true, true}, - {true, vc.NetXConnectMacVtapModel, true, true}, - {true, vc.NetXConnectMacVtapModel, false, true}, - {true, vc.NetXConnectNoneModel, false, false}, - {true, vc.NetXConnectNoneModel, true, false}, + {false, types.NetXConnectMacVtapModel, false, false}, + {false, types.NetXConnectMacVtapModel, true, true}, + {true, types.NetXConnectMacVtapModel, true, true}, + {true, types.NetXConnectMacVtapModel, false, true}, + {true, types.NetXConnectNoneModel, false, false}, + {true, types.NetXConnectNoneModel, true, false}, } for i, d := range data { diff --git a/pkg/katautils/network.go b/pkg/katautils/network.go index 036b611dbb..089e7436be 100644 --- a/pkg/katautils/network.go +++ b/pkg/katautils/network.go @@ -15,7 +15,7 @@ import ( "strings" "github.com/containernetworking/plugins/pkg/ns" - vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/types" "golang.org/x/sys/unix" ) @@ -52,7 +52,7 @@ func EnterNetNS(netNSPath string, cb func() error) error { } // SetupNetworkNamespace create a network namespace -func SetupNetworkNamespace(config *vc.NetworkConfig) error { +func SetupNetworkNamespace(config *types.NetworkConfig) error { if config.DisableNewNetNs { kataUtilsLogger.Info("DisableNewNetNs is on, shim and hypervisor are running in the host netns") return nil diff --git a/pkg/katautils/network_test.go b/pkg/katautils/network_test.go index 857568ce50..4a1d30d77d 100644 --- a/pkg/katautils/network_test.go +++ b/pkg/katautils/network_test.go @@ -14,7 +14,7 @@ import ( "testing" "github.com/containernetworking/plugins/pkg/ns" - vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" "golang.org/x/sys/unix" ) @@ -108,14 +108,14 @@ func TestSetupNetworkNamespace(t *testing.T) { assert := assert.New(t) // Network namespace same as the host - config := &vc.NetworkConfig{ + config := &types.NetworkConfig{ NetNSPath: "/proc/self/ns/net", } err := SetupNetworkNamespace(config) assert.Error(err) // Non-existent netns path - config = &vc.NetworkConfig{ + config = &types.NetworkConfig{ NetNSPath: "/proc/123456789/ns/net", } err = SetupNetworkNamespace(config) @@ -124,7 +124,7 @@ func TestSetupNetworkNamespace(t *testing.T) { // Existent netns path n, err := ns.NewNS() assert.NoError(err) - config = &vc.NetworkConfig{ + config = &types.NetworkConfig{ NetNSPath: n.Path(), } err = SetupNetworkNamespace(config) @@ -132,7 +132,7 @@ func TestSetupNetworkNamespace(t *testing.T) { n.Close() // Empty netns path - config = &vc.NetworkConfig{} + config = &types.NetworkConfig{} err = SetupNetworkNamespace(config) assert.NoError(err) n, err = ns.GetNS(config.NetNSPath) @@ -144,7 +144,7 @@ func TestSetupNetworkNamespace(t *testing.T) { os.RemoveAll(config.NetNSPath) // Config with DisableNewNetNs - config = &vc.NetworkConfig{DisableNewNetNs: true} + config = &types.NetworkConfig{DisableNewNetNs: true} err = SetupNetworkNamespace(config) assert.NoError(err) } diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go index 8b869a5803..c9bb40adb3 100644 --- a/virtcontainers/api_test.go +++ b/virtcontainers/api_test.go @@ -146,7 +146,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { SockTtyName: testHyperstartTtySocket, } - netConfig := NetworkConfig{} + netConfig := types.NetworkConfig{} sandboxConfig := SandboxConfig{ ID: testSandboxID, @@ -2001,7 +2001,7 @@ func createNewSandboxConfig(hType hypervisor.Type, aType AgentType, aConfig inte HypervisorPath: "/usr/bin/qemu-system-x86_64", } - netConfig := NetworkConfig{} + netConfig := types.NetworkConfig{} return SandboxConfig{ ID: testSandboxID, @@ -2421,7 +2421,7 @@ func TestNetworkOperation(t *testing.T) { defer deleteNetNS(netNSPath) config := newTestSandboxConfigNoop() - config.NetworkConfig = NetworkConfig{ + config.NetworkConfig = types.NetworkConfig{ NetNSPath: netNSPath, } diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go deleted file mode 100644 index 2819832cf6..0000000000 --- a/virtcontainers/endpoint.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "fmt" - - "github.com/kata-containers/runtime/virtcontainers/hypervisor" -) - -// Endpoint represents a physical or virtual network interface. -type Endpoint interface { - Properties() NetworkInfo - Name() string - HardwareAddr() string - Type() EndpointType - PciAddr() string - NetworkPair() *NetworkInterfacePair - - SetProperties(NetworkInfo) - SetPciAddr(string) - Attach(hypervisor.Hypervisor) error - Detach(netNsCreated bool, netNsPath string) error - HotAttach(hypervisor.Hypervisor) error - HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error -} - -// EndpointType identifies the type of the network endpoint. -type EndpointType string - -const ( - // PhysicalEndpointType is the physical network interface. - PhysicalEndpointType EndpointType = "physical" - - // VethEndpointType is the virtual network interface. - VethEndpointType EndpointType = "virtual" - - // VhostUserEndpointType is the vhostuser network interface. - VhostUserEndpointType EndpointType = "vhost-user" - - // BridgedMacvlanEndpointType is macvlan network interface. - BridgedMacvlanEndpointType EndpointType = "macvlan" - - // MacvtapEndpointType is macvtap network interface. - MacvtapEndpointType EndpointType = "macvtap" - - // TapEndpointType is tap network interface. - TapEndpointType EndpointType = "tap" - - // IPVlanEndpointType is ipvlan network interface. - IPVlanEndpointType EndpointType = "ipvlan" -) - -// Set sets an endpoint type based on the input string. -func (endpointType *EndpointType) Set(value string) error { - switch value { - case "physical": - *endpointType = PhysicalEndpointType - return nil - case "virtual": - *endpointType = VethEndpointType - return nil - case "vhost-user": - *endpointType = VhostUserEndpointType - return nil - case "macvlan": - *endpointType = BridgedMacvlanEndpointType - return nil - case "macvtap": - *endpointType = MacvtapEndpointType - return nil - case "tap": - *endpointType = TapEndpointType - return nil - case "ipvlan": - *endpointType = IPVlanEndpointType - return nil - default: - return fmt.Errorf("Unknown endpoint type %s", value) - } -} - -// String converts an endpoint type to a string. -func (endpointType *EndpointType) String() string { - switch *endpointType { - case PhysicalEndpointType: - return string(PhysicalEndpointType) - case VethEndpointType: - return string(VethEndpointType) - case VhostUserEndpointType: - return string(VhostUserEndpointType) - case BridgedMacvlanEndpointType: - return string(BridgedMacvlanEndpointType) - case MacvtapEndpointType: - return string(MacvtapEndpointType) - case TapEndpointType: - return string(TapEndpointType) - case IPVlanEndpointType: - return string(IPVlanEndpointType) - default: - return "" - } -} diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 12058c422e..291c01741a 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -509,7 +509,7 @@ func (fc *firecracker) fcAddVsock(vs kataVSOCK) error { return nil } -func (fc *firecracker) fcAddNetDevice(endpoint Endpoint) error { +func (fc *firecracker) fcAddNetDevice(endpoint hypervisor.Endpoint) error { span, _ := fc.trace("fcAddNetDevice") defer span.Finish() @@ -613,7 +613,7 @@ func (fc *firecracker) addDevice(devInfo interface{}, devType hypervisor.Device) } switch v := devInfo.(type) { - case Endpoint: + case hypervisor.Endpoint: fc.Logger().WithField("device-type-endpoint", devInfo).Info("Adding device") return fc.fcAddNetDevice(v) case config.BlockDrive: diff --git a/virtcontainers/hack/virtc/main.go b/virtcontainers/hack/virtc/main.go index 846c922855..cb73b51c09 100644 --- a/virtcontainers/hack/virtc/main.go +++ b/virtcontainers/hack/virtc/main.go @@ -171,7 +171,7 @@ func buildSandboxConfig(context *cli.Context) (vc.SandboxConfig, error) { return vc.SandboxConfig{}, err } - netConfig := vc.NetworkConfig{} + netConfig := types.NetworkConfig{} switch *agentType { case vc.HyperstartAgent: diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index 14c00a24c7..93b665d415 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -153,8 +153,8 @@ func (h *hyper) processHyperRoute(route netlink.Route, deviceName string) *hyper destination = "" } else { destination = route.Dst.String() - if destination == defaultRouteDest { - destination = defaultRouteLabel + if destination == types.DefaultRouteDest { + destination = types.DefaultRouteLabel } // Skip IPv6 because not supported by hyperstart diff --git a/virtcontainers/hyperstart_agent_test.go b/virtcontainers/hyperstart_agent_test.go index 69d75e3b73..cf914550f7 100644 --- a/virtcontainers/hyperstart_agent_test.go +++ b/virtcontainers/hyperstart_agent_test.go @@ -130,12 +130,12 @@ func TestProcessHyperRouteEmptyGWSuccessful(t *testing.T) { func TestProcessHyperRouteEmptyDestSuccessful(t *testing.T) { expected := &hyperstart.Route{ - Dest: defaultRouteLabel, + Dest: types.DefaultRouteLabel, Gateway: testRouteGateway, Device: testRouteDeviceName, } - _, dest, err := net.ParseCIDR(defaultRouteDest) + _, dest, err := net.ParseCIDR(types.DefaultRouteDest) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/hypervisor/bridgedmacvlan_endpoint.go similarity index 78% rename from virtcontainers/bridgedmacvlan_endpoint.go rename to virtcontainers/hypervisor/bridgedmacvlan_endpoint.go index aa7e862ef3..8a741cfb37 100644 --- a/virtcontainers/bridgedmacvlan_endpoint.go +++ b/virtcontainers/hypervisor/bridgedmacvlan_endpoint.go @@ -3,24 +3,24 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" ) // BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM type BridgedMacvlanEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo + NetPair types.NetworkInterfacePair + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } -func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { +func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel types.NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { if idx < 0 { return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) } @@ -42,7 +42,7 @@ func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingMod } // Properties returns properties of the interface. -func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo { +func (endpoint *BridgedMacvlanEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -63,7 +63,7 @@ func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType { } // SetProperties sets the properties for the endpoint. -func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } @@ -78,19 +78,19 @@ func (endpoint *BridgedMacvlanEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *types.NetworkInterfacePair { return &endpoint.NetPair } // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor.Hypervisor) error { +func (endpoint *BridgedMacvlanEndpoint) Attach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - return h.AddDevice(endpoint, hypervisor.NetDev) + return h.AddDevice(endpoint, NetDev) } // Detach for the virtual endpoint tears down the tap and bridge @@ -102,17 +102,17 @@ func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath stri return nil } - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }) } // HotAttach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor.Hypervisor) error { +func (endpoint *BridgedMacvlanEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *BridgedMacvlanEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") } diff --git a/virtcontainers/bridgedmacvlan_endpoint_test.go b/virtcontainers/hypervisor/bridgedmacvlan_endpoint_test.go similarity index 69% rename from virtcontainers/bridgedmacvlan_endpoint_test.go rename to virtcontainers/hypervisor/bridgedmacvlan_endpoint_test.go index 0cc4a94012..13003feee6 100644 --- a/virtcontainers/bridgedmacvlan_endpoint_test.go +++ b/virtcontainers/hypervisor/bridgedmacvlan_endpoint_test.go @@ -3,36 +3,38 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateBridgedMacvlanEndpoint(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &BridgedMacvlanEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth4", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, EndpointType: BridgedMacvlanEndpointType, } - result, err := createBridgedMacvlanNetworkEndpoint(4, "", DefaultNetInterworkingModel) + result, err := createBridgedMacvlanNetworkEndpoint(4, "", types.DefaultNetInterworkingModel) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/hypervisor/endpoint.go b/virtcontainers/hypervisor/endpoint.go new file mode 100644 index 0000000000..d24ec35073 --- /dev/null +++ b/virtcontainers/hypervisor/endpoint.go @@ -0,0 +1,794 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + "encoding/json" + "fmt" + "net" + "sort" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + + "github.com/kata-containers/runtime/virtcontainers/types" +) + +// Endpoint represents a physical or virtual network interface. +type Endpoint interface { + Properties() types.NetworkInfo + Name() string + HardwareAddr() string + Type() EndpointType + PciAddr() string + NetworkPair() *types.NetworkInterfacePair + + SetProperties(types.NetworkInfo) + SetPciAddr(string) + Attach(Hypervisor) error + Detach(netNsCreated bool, netNsPath string) error + HotAttach(Hypervisor) error + HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error +} + +// EndpointType identifies the type of the network endpoint. +type EndpointType string + +const ( + // PhysicalEndpointType is the physical network interface. + PhysicalEndpointType EndpointType = "physical" + + // VethEndpointType is the virtual network interface. + VethEndpointType EndpointType = "virtual" + + // VhostUserEndpointType is the vhostuser network interface. + VhostUserEndpointType EndpointType = "vhost-user" + + // BridgedMacvlanEndpointType is macvlan network interface. + BridgedMacvlanEndpointType EndpointType = "macvlan" + + // MacvtapEndpointType is macvtap network interface. + MacvtapEndpointType EndpointType = "macvtap" + + // TapEndpointType is tap network interface. + TapEndpointType EndpointType = "tap" + + // IPVlanEndpointType is ipvlan network interface. + IPVlanEndpointType EndpointType = "ipvlan" +) + +// Set sets an endpoint type based on the input string. +func (endpointType *EndpointType) Set(value string) error { + switch value { + case "physical": + *endpointType = PhysicalEndpointType + return nil + case "virtual": + *endpointType = VethEndpointType + return nil + case "vhost-user": + *endpointType = VhostUserEndpointType + return nil + case "macvlan": + *endpointType = BridgedMacvlanEndpointType + return nil + case "macvtap": + *endpointType = MacvtapEndpointType + return nil + case "tap": + *endpointType = TapEndpointType + return nil + case "ipvlan": + *endpointType = IPVlanEndpointType + return nil + default: + return fmt.Errorf("Unknown endpoint type %s", value) + } +} + +// String converts an endpoint type to a string. +func (endpointType *EndpointType) String() string { + switch *endpointType { + case PhysicalEndpointType: + return string(PhysicalEndpointType) + case VethEndpointType: + return string(VethEndpointType) + case VhostUserEndpointType: + return string(VhostUserEndpointType) + case BridgedMacvlanEndpointType: + return string(BridgedMacvlanEndpointType) + case MacvtapEndpointType: + return string(MacvtapEndpointType) + case TapEndpointType: + return string(TapEndpointType) + case IPVlanEndpointType: + return string(IPVlanEndpointType) + default: + return "" + } +} + +func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.Link, error) { + var link netlink.Link + + switch ep := endpoint.(type) { + case *VethEndpoint: + link = &netlink.Veth{} + case *BridgedMacvlanEndpoint: + link = &netlink.Macvlan{} + case *IPVlanEndpoint: + link = &netlink.IPVlan{} + default: + return nil, fmt.Errorf("Unexpected endpointType %s", ep.Type()) + } + + return getLinkByName(netHandle, endpoint.NetworkPair().VirtIface.Name, link) +} + +func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + attrs := link.Attrs() + + // Attach the macvtap interface to the underlying container + // interface. Also picks relevant attributes from the parent + tapLink, err := createMacVtap(netHandle, netPair.TAPIface.Name, + &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + LinkAttrs: netlink.LinkAttrs{ + TxQLen: attrs.TxQLen, + ParentIndex: attrs.Index, + }, + }, + }, queues) + + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } + + // Save the veth MAC address to the TAP so that it can later be used + // to build the hypervisor command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + tapHardAddr := attrs.HardwareAddr + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() + + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) + } + + hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetHardwareAddr(tapLink, tapHardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) + } + + // Clear the IP addresses from the veth interface to prevent ARP conflict + netPair.VirtIface.Addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return fmt.Errorf("Unable to obtain veth IP addresses: %s", err) + } + + if err := clearIPs(link, netPair.VirtIface.Addrs); err != nil { + return fmt.Errorf("Unable to clear veth IP addresses: %s", err) + } + + if err := netHandle.LinkSetUp(link); err != nil { + return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) + } + + // Note: The underlying interfaces need to be up prior to fd creation. + + netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, queues) + if err != nil { + return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err) + } + + if !disableVhostNet { + vhostFds, err := createVhostFds(queues) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds + } + + return nil +} + +func bridgeNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } + netPair.VMFds = fds + + if !disableVhostNet { + vhostFds, err := createVhostFds(queues) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds + } + + var attrs *netlink.LinkAttrs + var link netlink.Link + + link, err = getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + attrs = link.Attrs() + + // Save the veth MAC address to the TAP so that it can later be used + // to build the hypervisor command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() + + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) + } + + hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + mcastSnoop := false + bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, queues) + if err != nil { + return fmt.Errorf("Could not create bridge: %s", err) + } + + if err := netHandle.LinkSetMaster(tapLink, bridgeLink.(*netlink.Bridge)); err != nil { + return fmt.Errorf("Could not attach TAP %s to the bridge %s: %s", + netPair.TAPIface.Name, netPair.Name, err) + } + + if err := netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkSetMaster(link, bridgeLink.(*netlink.Bridge)); err != nil { + return fmt.Errorf("Could not attach veth %s to the bridge %s: %s", + netPair.VirtIface.Name, netPair.Name, err) + } + + if err := netHandle.LinkSetUp(link); err != nil { + return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetUp(bridgeLink); err != nil { + return fmt.Errorf("Could not enable bridge %s: %s", netPair.Name, err) + } + + return nil +} + +func setupTCFiltering(endpoint Endpoint, queues int, disableVhostNet bool) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } + netPair.VMFds = fds + + if !disableVhostNet { + vhostFds, err := createVhostFds(queues) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds + } + + var attrs *netlink.LinkAttrs + var link netlink.Link + + link, err = getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + attrs = link.Attrs() + + // Save the veth MAC address to the TAP so that it can later be used + // to build the hypervisor command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() + + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) + } + + if err := netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) + } + + tapAttrs := tapLink.Attrs() + + if err := addQdiscIngress(tapAttrs.Index); err != nil { + return err + } + + if err := addQdiscIngress(attrs.Index); err != nil { + return err + } + + if err := addRedirectTCFilter(attrs.Index, tapAttrs.Index); err != nil { + return err + } + + if err := addRedirectTCFilter(tapAttrs.Index, attrs.Index); err != nil { + return err + } + + return nil +} + +func untapNetworkPair(endpoint Endpoint) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Macvtap{}) + if err != nil { + return fmt.Errorf("Could not get TAP interface %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkDel(tapLink); err != nil { + return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) + } + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + + // Restore the IPs that were cleared + err = setIPs(link, netPair.VirtIface.Addrs) + return err +} + +func unBridgeNetworkPair(endpoint Endpoint) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) + if err != nil { + return fmt.Errorf("Could not get TAP interface: %s", err) + } + + bridgeLink, err := getLinkByName(netHandle, netPair.Name, &netlink.Bridge{}) + if err != nil { + return fmt.Errorf("Could not get bridge interface: %s", err) + } + + if err := netHandle.LinkSetDown(bridgeLink); err != nil { + return fmt.Errorf("Could not disable bridge %s: %s", netPair.Name, err) + } + + if err := netHandle.LinkSetDown(tapLink); err != nil { + return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkSetNoMaster(tapLink); err != nil { + return fmt.Errorf("Could not detach TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkDel(bridgeLink); err != nil { + return fmt.Errorf("Could not remove bridge %s: %s", netPair.Name, err) + } + + if err := netHandle.LinkDel(tapLink); err != nil { + return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) + } + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetNoMaster(link); err != nil { + return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) + } + + return nil +} + +func removeTCFiltering(endpoint Endpoint) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) + if err != nil { + return fmt.Errorf("Could not get TAP interface: %s", err) + } + + if err := netHandle.LinkSetDown(tapLink); err != nil { + return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkDel(tapLink); err != nil { + return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) + } + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + if err := removeRedirectTCFilter(link); err != nil { + return err + } + + if err := removeQdiscIngress(link); err != nil { + return err + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + + return nil +} + +// The endpoint type should dictate how the connection needs to happen. +func xConnectVMNetwork(endpoint Endpoint, h Hypervisor) error { + netPair := endpoint.NetworkPair() + + queues := 0 + caps := h.Capabilities() + if caps.IsMultiQueueSupported() { + queues = int(h.Config().NumVCPUs) + } + + disableVhostNet := h.Config().DisableVhostNet + + if netPair.NetInterworkingModel == types.NetXConnectDefaultModel { + netPair.NetInterworkingModel = types.DefaultNetInterworkingModel + } + + switch netPair.NetInterworkingModel { + case types.NetXConnectBridgedModel: + return bridgeNetworkPair(endpoint, queues, disableVhostNet) + case types.NetXConnectMacVtapModel: + return tapNetworkPair(endpoint, queues, disableVhostNet) + case types.NetXConnectTCFilterModel: + return setupTCFiltering(endpoint, queues, disableVhostNet) + case types.NetXConnectEnlightenedModel: + return fmt.Errorf("Unsupported networking model") + default: + return fmt.Errorf("Invalid internetworking model") + } +} + +// The endpoint type should dictate how the disconnection needs to happen. +func xDisconnectVMNetwork(endpoint Endpoint) error { + netPair := endpoint.NetworkPair() + + if netPair.NetInterworkingModel == types.NetXConnectDefaultModel { + netPair.NetInterworkingModel = types.DefaultNetInterworkingModel + } + + switch netPair.NetInterworkingModel { + case types.NetXConnectBridgedModel: + return unBridgeNetworkPair(endpoint) + case types.NetXConnectMacVtapModel: + return untapNetworkPair(endpoint) + case types.NetXConnectTCFilterModel: + return removeTCFiltering(endpoint) + case types.NetXConnectEnlightenedModel: + return fmt.Errorf("Unsupported networking model") + default: + return fmt.Errorf("Invalid internetworking model") + } +} + +// CreateEndpoint creates an Endpoint from a network info and interworking model. +func CreateEndpoint(netInfo types.NetworkInfo, idx int, model types.NetInterworkingModel) (Endpoint, error) { + var endpoint Endpoint + // TODO: This is the incoming interface + // based on the incoming interface we should create + // an appropriate EndPoint based on interface type + // This should be a switch + + // Check if interface is a physical interface. Do not create + // tap interface/bridge if it is. + isPhysical, err := isPhysicalIface(netInfo.Iface.Name) + if err != nil { + return nil, err + } + + if isPhysical { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") + endpoint, err = createPhysicalEndpoint(netInfo) + } else { + var socketPath string + + // Check if this is a dummy interface which has a vhost-user socket associated with it + socketPath, err = vhostUserSocketPath(netInfo) + if err != nil { + return nil, err + } + + if socketPath != "" { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") + endpoint, err = createVhostUserEndpoint(netInfo, socketPath) + } else if netInfo.Iface.Type == "macvlan" { + networkLogger().Infof("macvlan interface found") + endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) + } else if netInfo.Iface.Type == "macvtap" { + networkLogger().Infof("macvtap interface found") + endpoint, err = createMacvtapNetworkEndpoint(netInfo) + } else if netInfo.Iface.Type == "tap" { + networkLogger().Info("tap interface found") + endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) + } else if netInfo.Iface.Type == "veth" { + endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) + } else if netInfo.Iface.Type == "ipvlan" { + endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) + } else { + return nil, fmt.Errorf("Unsupported network interface") + } + } + + return endpoint, err +} + +// CreateEndpointsFromScan creates a slice of Endpoints from a network namespace scan. +func CreateEndpointsFromScan(networkNSPath string, config *types.NetworkConfig) ([]Endpoint, error) { + var endpoints []Endpoint + + netnsHandle, err := netns.GetFromPath(networkNSPath) + if err != nil { + return []Endpoint{}, err + } + defer netnsHandle.Close() + + netlinkHandle, err := netlink.NewHandleAt(netnsHandle) + if err != nil { + return []Endpoint{}, err + } + defer netlinkHandle.Delete() + + linkList, err := netlinkHandle.LinkList() + if err != nil { + return []Endpoint{}, err + } + + idx := 0 + for _, link := range linkList { + var ( + endpoint Endpoint + errCreate error + ) + + netInfo, err := networkInfoFromLink(netlinkHandle, link) + if err != nil { + return []Endpoint{}, err + } + + // Ignore unconfigured network interfaces. These are + // either base tunnel devices that are not namespaced + // like gre0, gretap0, sit0, ipip0, tunl0 or incorrectly + // setup interfaces. + if len(netInfo.Addrs) == 0 { + continue + } + + // Skip any loopback interfaces: + if (netInfo.Iface.Flags & net.FlagLoopback) != 0 { + continue + } + + if err := DoNetNS(networkNSPath, func(_ ns.NetNS) error { + endpoint, errCreate = CreateEndpoint(netInfo, idx, config.InterworkingModel) + return errCreate + }); err != nil { + return []Endpoint{}, err + } + + endpoint.SetProperties(netInfo) + endpoints = append(endpoints, endpoint) + + idx++ + } + + sort.Slice(endpoints, func(i, j int) bool { + return endpoints[i].Name() < endpoints[j].Name() + }) + + networkLogger().WithField("endpoints", endpoints).Info("Endpoints found after scan") + + return endpoints, nil +} + +// TypedJSONEndpoint is used as an intermediate representation for +// marshalling and unmarshalling Endpoint objects. +type TypedJSONEndpoint struct { + Type EndpointType + Data json.RawMessage +} + +// UnmarshalEndpoints unmarshalls a slice of typed Endpoints into a slice of regular ones. +func UnmarshalEndpoints(typedEndpoints []TypedJSONEndpoint) ([]Endpoint, error) { + var endpoints []Endpoint + + for _, e := range typedEndpoints { + switch e.Type { + case PhysicalEndpointType: + var endpoint PhysicalEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "physical", + }).Info("endpoint unmarshalled") + + case VethEndpointType: + var endpoint VethEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "virtual", + }).Info("endpoint unmarshalled") + + case VhostUserEndpointType: + var endpoint VhostUserEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "vhostuser", + }).Info("endpoint unmarshalled") + + case BridgedMacvlanEndpointType: + var endpoint BridgedMacvlanEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "macvlan", + }).Info("endpoint unmarshalled") + + case MacvtapEndpointType: + var endpoint MacvtapEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "macvtap", + }).Info("endpoint unmarshalled") + + case TapEndpointType: + var endpoint TapEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "tap", + }).Info("endpoint unmarshalled") + + default: + networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type") + } + } + return endpoints, nil +} diff --git a/virtcontainers/endpoint_test.go b/virtcontainers/hypervisor/endpoint_test.go similarity index 98% rename from virtcontainers/endpoint_test.go rename to virtcontainers/hypervisor/endpoint_test.go index e5473a96c7..63a919fba3 100644 --- a/virtcontainers/endpoint_test.go +++ b/virtcontainers/hypervisor/endpoint_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import "testing" diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/hypervisor/ipvlan_endpoint.go similarity index 78% rename from virtcontainers/ipvlan_endpoint.go rename to virtcontainers/hypervisor/ipvlan_endpoint.go index 6b8517e7e3..6d3c642678 100644 --- a/virtcontainers/ipvlan_endpoint.go +++ b/virtcontainers/hypervisor/ipvlan_endpoint.go @@ -3,19 +3,19 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" ) // IPVlanEndpoint represents a ipvlan endpoint that is bridged to the VM type IPVlanEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo + NetPair types.NetworkInterfacePair + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } @@ -27,7 +27,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error // Use tc filtering for ipvlan, since the other inter networking models will // not work for ipvlan. - interworkingModel := NetXConnectTCFilterModel + interworkingModel := types.NetXConnectTCFilterModel netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) if err != nil { return nil, err @@ -45,7 +45,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error } // Properties returns properties of the interface. -func (endpoint *IPVlanEndpoint) Properties() NetworkInfo { +func (endpoint *IPVlanEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -66,7 +66,7 @@ func (endpoint *IPVlanEndpoint) Type() EndpointType { } // SetProperties sets the properties for the endpoint. -func (endpoint *IPVlanEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *IPVlanEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } @@ -81,19 +81,19 @@ func (endpoint *IPVlanEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *IPVlanEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *IPVlanEndpoint) NetworkPair() *types.NetworkInterfacePair { return &endpoint.NetPair } // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *IPVlanEndpoint) Attach(h hypervisor.Hypervisor) error { +func (endpoint *IPVlanEndpoint) Attach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - return h.AddDevice(endpoint, hypervisor.NetDev) + return h.AddDevice(endpoint, NetDev) } // Detach for the virtual endpoint tears down the tap and bridge @@ -105,17 +105,17 @@ func (endpoint *IPVlanEndpoint) Detach(netNsCreated bool, netNsPath string) erro return nil } - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }) } // HotAttach for physical endpoint not supported yet -func (endpoint *IPVlanEndpoint) HotAttach(h hypervisor.Hypervisor) error { +func (endpoint *IPVlanEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("IPVlanEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *IPVlanEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *IPVlanEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("IPVlanEndpoint does not support Hot detach") } diff --git a/virtcontainers/ipvlan_endpoint_test.go b/virtcontainers/hypervisor/ipvlan_endpoint_test.go similarity index 76% rename from virtcontainers/ipvlan_endpoint_test.go rename to virtcontainers/hypervisor/ipvlan_endpoint_test.go index 945427c80b..ed5bb7b64f 100644 --- a/virtcontainers/ipvlan_endpoint_test.go +++ b/virtcontainers/hypervisor/ipvlan_endpoint_test.go @@ -3,32 +3,34 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateIPVlanEndpoint(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &IPVlanEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-5", Name: "br5_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap5_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth5", HardAddr: macAddr.String(), }, - NetInterworkingModel: NetXConnectTCFilterModel, + NetInterworkingModel: types.NetXConnectTCFilterModel, }, EndpointType: IPVlanEndpointType, } diff --git a/virtcontainers/macvtap_endpoint.go b/virtcontainers/hypervisor/macvtap_endpoint.go similarity index 77% rename from virtcontainers/macvtap_endpoint.go rename to virtcontainers/hypervisor/macvtap_endpoint.go index d21fcf50c4..9f34d00b6f 100644 --- a/virtcontainers/macvtap_endpoint.go +++ b/virtcontainers/hypervisor/macvtap_endpoint.go @@ -3,25 +3,25 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "os" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" ) // MacvtapEndpoint represents a macvtap endpoint type MacvtapEndpoint struct { - EndpointProperties NetworkInfo + EndpointProperties types.NetworkInfo EndpointType EndpointType VMFds []*os.File VhostFds []*os.File PCIAddr string } -func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) { +func createMacvtapNetworkEndpoint(netInfo types.NetworkInfo) (*MacvtapEndpoint, error) { endpoint := &MacvtapEndpoint{ EndpointType: MacvtapEndpointType, EndpointProperties: netInfo, @@ -31,7 +31,7 @@ func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) } // Properties returns the properties of the macvtap interface. -func (endpoint *MacvtapEndpoint) Properties() NetworkInfo { +func (endpoint *MacvtapEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -51,12 +51,12 @@ func (endpoint *MacvtapEndpoint) Type() EndpointType { } // SetProperties sets the properties of the macvtap endpoint. -func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *MacvtapEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } -// Attach for macvtap endpoint passes macvtap device to the hypervisor. -func (endpoint *MacvtapEndpoint) Attach(h hypervisor.Hypervisor) error { +// Attach for macvtap endpoint passes macvtap device to the hypervisir. +func (endpoint *MacvtapEndpoint) Attach(h Hypervisor) error { var err error endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.Config().NumVCPUs)) @@ -72,7 +72,7 @@ func (endpoint *MacvtapEndpoint) Attach(h hypervisor.Hypervisor) error { endpoint.VhostFds = vhostFds } - return h.AddDevice(endpoint, hypervisor.NetDev) + return h.AddDevice(endpoint, NetDev) } // Detach for macvtap endpoint does nothing. @@ -81,12 +81,12 @@ func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) err } // HotAttach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor.Hypervisor) error { +func (endpoint *MacvtapEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("MacvtapEndpoint does not support Hot attach") } // HotDetach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *MacvtapEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("MacvtapEndpoint does not support Hot detach") } @@ -101,6 +101,6 @@ func (endpoint *MacvtapEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *MacvtapEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } diff --git a/virtcontainers/macvtap_endpoint_test.go b/virtcontainers/hypervisor/macvtap_endpoint_test.go similarity index 78% rename from virtcontainers/macvtap_endpoint_test.go rename to virtcontainers/hypervisor/macvtap_endpoint_test.go index fbef06f30e..e0fb2df3f0 100644 --- a/virtcontainers/macvtap_endpoint_test.go +++ b/virtcontainers/hypervisor/macvtap_endpoint_test.go @@ -3,16 +3,18 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateMacvtapEndpoint(t *testing.T) { - netInfo := NetworkInfo{ - Iface: NetlinkIface{ + netInfo := types.NetworkInfo{ + Iface: types.NetlinkIface{ Type: "macvtap", }, } diff --git a/virtcontainers/hypervisor/network.go b/virtcontainers/hypervisor/network.go new file mode 100644 index 0000000000..a60495615f --- /dev/null +++ b/virtcontainers/hypervisor/network.go @@ -0,0 +1,387 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + cryptoRand "crypto/rand" + "fmt" + "math/rand" + "net" + "os" + "runtime" + "time" + + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +func networkLogger() *logrus.Entry { + return logrus.WithField("source", "virtcontainers/hypervisor").WithField("subsystem", "network") +} + +// DoNetNS is free from any call to a go routine, and it calls +// into runtime.LockOSThread(), meaning it won't be executed in a +// different thread than the one expected by the caller. +func DoNetNS(netNSPath string, cb func(ns.NetNS) error) error { + // if netNSPath is empty, the callback function will be run in the current network namespace. + // So skip the whole function, just call cb(). cb() needs a NetNS as arg but ignored, give it a fake one. + if netNSPath == "" { + var netNs ns.NetNS + return cb(netNs) + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + currentNS, err := ns.GetCurrentNS() + if err != nil { + return err + } + defer currentNS.Close() + + targetNS, err := ns.GetNS(netNSPath) + if err != nil { + return err + } + + if err := targetNS.Set(); err != nil { + return err + } + defer currentNS.Set() + + return cb(targetNS) +} + +func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, queues int) (netlink.Link, []*os.File, error) { + var newLink netlink.Link + var fds []*os.File + + switch expectedLink.Type() { + case (&netlink.Bridge{}).Type(): + newLink = &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{Name: name}, + MulticastSnooping: expectedLink.(*netlink.Bridge).MulticastSnooping, + } + case (&netlink.Tuntap{}).Type(): + flags := netlink.TUNTAP_VNET_HDR + if queues > 0 { + flags |= netlink.TUNTAP_MULTI_QUEUE_DEFAULTS + } + newLink = &netlink.Tuntap{ + LinkAttrs: netlink.LinkAttrs{Name: name}, + Mode: netlink.TUNTAP_MODE_TAP, + Queues: queues, + Flags: flags, + } + case (&netlink.Macvtap{}).Type(): + qlen := expectedLink.Attrs().TxQLen + if qlen <= 0 { + qlen = types.DefaultQlen + } + newLink = &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + Mode: netlink.MACVLAN_MODE_BRIDGE, + LinkAttrs: netlink.LinkAttrs{ + Index: expectedLink.Attrs().Index, + Name: name, + TxQLen: qlen, + ParentIndex: expectedLink.Attrs().ParentIndex, + }, + }, + } + default: + return nil, fds, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) + } + + if err := netHandle.LinkAdd(newLink); err != nil { + return nil, fds, fmt.Errorf("LinkAdd() failed for %s name %s: %s", expectedLink.Type(), name, err) + } + + tuntapLink, ok := newLink.(*netlink.Tuntap) + if ok { + fds = tuntapLink.Fds + } + + newLink, err := getLinkByName(netHandle, name, expectedLink) + return newLink, fds, err +} + +const hostLinkOffset = 8192 // Host should not have more than 8k interfaces +const linkRange = 0xFFFF // This will allow upto 2^16 containers +const linkRetries = 128 // The numbers of time we try to find a non conflicting index +const macvtapWorkaround = true + +func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, queues int) (taplink netlink.Link, err error) { + if !macvtapWorkaround { + taplink, _, err = createLink(netHandle, name, link, queues) + return + } + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for i := 0; i < linkRetries; i++ { + index := hostLinkOffset + (r.Int() & linkRange) + link.Attrs().Index = index + taplink, _, err = createLink(netHandle, name, link, queues) + if err == nil { + break + } + } + + return +} + +func clearIPs(link netlink.Link, addrs []netlink.Addr) error { + for _, addr := range addrs { + if err := netlink.AddrDel(link, &addr); err != nil { + return err + } + } + return nil +} + +func setIPs(link netlink.Link, addrs []netlink.Addr) error { + for _, addr := range addrs { + if err := netlink.AddrAdd(link, &addr); err != nil { + return err + } + } + return nil +} + +func createFds(device string, numFds int) ([]*os.File, error) { + fds := make([]*os.File, numFds) + + for i := 0; i < numFds; i++ { + f, err := os.OpenFile(device, os.O_RDWR, types.DefaultFilePerms) + if err != nil { + utils.CleanupFds(fds, i) + return nil, err + } + fds[i] = f + } + return fds, nil +} + +func createMacvtapFds(linkIndex int, queues int) ([]*os.File, error) { + tapDev := fmt.Sprintf("/dev/tap%d", linkIndex) + return createFds(tapDev, queues) +} + +func createVhostFds(numFds int) ([]*os.File, error) { + vhostDev := "/dev/vhost-net" + return createFds(vhostDev, numFds) +} + +func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.Link) (netlink.Link, error) { + link, err := netHandle.LinkByName(name) + if err != nil { + return nil, fmt.Errorf("LinkByName() failed for %s name %s: %s", expectedLink.Type(), name, err) + } + + switch expectedLink.Type() { + case (&netlink.Bridge{}).Type(): + if l, ok := link.(*netlink.Bridge); ok { + return l, nil + } + case (&netlink.Tuntap{}).Type(): + if l, ok := link.(*netlink.GenericLink); ok { + return l, nil + } + case (&netlink.Veth{}).Type(): + if l, ok := link.(*netlink.Veth); ok { + return l, nil + } + case (&netlink.Macvtap{}).Type(): + if l, ok := link.(*netlink.Macvtap); ok { + return l, nil + } + case (&netlink.Macvlan{}).Type(): + if l, ok := link.(*netlink.Macvlan); ok { + return l, nil + } + case (&netlink.IPVlan{}).Type(): + if l, ok := link.(*netlink.IPVlan); ok { + return l, nil + } + default: + return nil, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) + } + + return nil, fmt.Errorf("Incorrect link type %s, expecting %s", link.Type(), expectedLink.Type()) +} + +// addQdiscIngress creates a new qdisc for nwtwork interface with the specified network index +// on "ingress". qdiscs normally don't work on ingress so this is really a special qdisc +// that you can consider an "alternate root" for inbound packets. +// Handle for ingress qdisc defaults to "ffff:" +// +// This is equivalent to calling `tc qdisc add dev eth0 ingress` +func addQdiscIngress(index int) error { + qdisc := &netlink.Ingress{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: index, + Parent: netlink.HANDLE_INGRESS, + }, + } + + err := netlink.QdiscAdd(qdisc) + if err != nil { + return fmt.Errorf("Failed to add qdisc for network index %d : %s", index, err) + } + + return nil +} + +// addRedirectTCFilter adds a tc filter for device with index "sourceIndex". +// All traffic for interface with index "sourceIndex" is redirected to interface with +// index "destIndex" +// +// This is equivalent to calling: +// `tc filter add dev source parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev dest` +func addRedirectTCFilter(sourceIndex, destIndex int) error { + filter := &netlink.U32{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: sourceIndex, + Parent: netlink.MakeHandle(0xffff, 0), + Protocol: unix.ETH_P_ALL, + }, + Actions: []netlink.Action{ + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: destIndex, + }, + }, + } + + if err := netlink.FilterAdd(filter); err != nil { + return fmt.Errorf("Failed to add filter for index %d : %s", sourceIndex, err) + } + + return nil +} + +// removeRedirectTCFilter removes all tc u32 filters created on ingress qdisc for "link". +func removeRedirectTCFilter(link netlink.Link) error { + if link == nil { + return nil + } + + // Handle 0xffff is used for ingress + filters, err := netlink.FilterList(link, netlink.MakeHandle(0xffff, 0)) + if err != nil { + return err + } + + for _, f := range filters { + u32, ok := f.(*netlink.U32) + + if !ok { + continue + } + + if err := netlink.FilterDel(u32); err != nil { + return err + } + } + return nil +} + +// removeQdiscIngress removes the ingress qdisc previously created on "link". +func removeQdiscIngress(link netlink.Link) error { + if link == nil { + return nil + } + + qdiscs, err := netlink.QdiscList(link) + if err != nil { + return err + } + + for _, qdisc := range qdiscs { + ingress, ok := qdisc.(*netlink.Ingress) + if !ok { + continue + } + + if err := netlink.QdiscDel(ingress); err != nil { + return err + } + } + return nil +} + +func generateRandomPrivateMacAddr() (string, error) { + buf := make([]byte, 6) + _, err := cryptoRand.Read(buf) + if err != nil { + return "", err + } + + // Set the local bit for local addresses + // Addresses in this range are local mac addresses: + // x2-xx-xx-xx-xx-xx , x6-xx-xx-xx-xx-xx , xA-xx-xx-xx-xx-xx , xE-xx-xx-xx-xx-xx + buf[0] = (buf[0] | 2) & 0xfe + + hardAddr := net.HardwareAddr(buf) + return hardAddr.String(), nil +} + +func createNetworkInterfacePair(idx int, ifName string, interworkingModel types.NetInterworkingModel) (types.NetworkInterfacePair, error) { + uniqueID := uuid.Generate().String() + + randomMacAddr, err := generateRandomPrivateMacAddr() + if err != nil { + return types.NetworkInterfacePair{}, fmt.Errorf("Could not generate random mac address: %s", err) + } + + netPair := types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ + ID: uniqueID, + Name: fmt.Sprintf("br%d_kata", idx), + TAPIface: types.NetworkInterface{ + Name: fmt.Sprintf("tap%d_kata", idx), + }, + }, + VirtIface: types.NetworkInterface{ + Name: fmt.Sprintf("eth%d", idx), + HardAddr: randomMacAddr, + }, + NetInterworkingModel: interworkingModel, + } + + return netPair, nil +} + +func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (types.NetworkInfo, error) { + addrs, err := handle.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return types.NetworkInfo{}, err + } + + routes, err := handle.RouteList(link, netlink.FAMILY_ALL) + if err != nil { + return types.NetworkInfo{}, err + } + + return types.NetworkInfo{ + Iface: types.NetlinkIface{ + LinkAttrs: *(link.Attrs()), + Type: link.Type(), + }, + Addrs: addrs, + Routes: routes, + }, nil +} diff --git a/virtcontainers/hypervisor/network_test.go b/virtcontainers/hypervisor/network_test.go new file mode 100644 index 0000000000..a1bc57af3b --- /dev/null +++ b/virtcontainers/hypervisor/network_test.go @@ -0,0 +1,166 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + "net" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + + "github.com/kata-containers/runtime/virtcontainers/types" +) + +func TestGenerateRandomPrivateMacAdd(t *testing.T) { + assert := assert.New(t) + + addr1, err := generateRandomPrivateMacAddr() + assert.NoError(err) + + _, err = net.ParseMAC(addr1) + assert.NoError(err) + + addr2, err := generateRandomPrivateMacAddr() + assert.NoError(err) + + _, err = net.ParseMAC(addr2) + assert.NoError(err) + + assert.NotEqual(addr1, addr2) +} + +func TestCreateGetBridgeLink(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + brName := "testbr0" + brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) + assert.NoError(err) + assert.NotNil(brLink) + + brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) + assert.NoError(err) + + err = netHandle.LinkDel(brLink) + assert.NoError(err) +} + +func TestCreateGetTunTapLink(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + tapName := "testtap0" + tapLink, fds, err := createLink(netHandle, tapName, &netlink.Tuntap{}, 1) + assert.NoError(err) + assert.NotNil(tapLink) + assert.NotZero(len(fds)) + + tapLink, err = getLinkByName(netHandle, tapName, &netlink.Tuntap{}) + assert.NoError(err) + + err = netHandle.LinkDel(tapLink) + assert.NoError(err) +} + +func TestCreateMacVtap(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + brName := "testbr0" + brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) + assert.NoError(err) + + attrs := brLink.Attrs() + + mcLink := &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + LinkAttrs: netlink.LinkAttrs{ + TxQLen: attrs.TxQLen, + ParentIndex: attrs.Index, + }, + }, + } + + macvtapName := "testmc0" + _, err = createMacVtap(netHandle, macvtapName, mcLink, 1) + assert.NoError(err) + + macvtapLink, err := getLinkByName(netHandle, macvtapName, &netlink.Macvtap{}) + assert.NoError(err) + + err = netHandle.LinkDel(macvtapLink) + assert.NoError(err) + + brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) + assert.NoError(err) + + err = netHandle.LinkDel(brLink) + assert.NoError(err) +} + +func TestTcRedirectNetwork(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + assert.NoError(err) + defer netHandle.Delete() + + // Create a test veth interface. + vethName := "foo" + veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: vethName, TxQLen: 200, MTU: 1400}, PeerName: "bar"} + + err = netlink.LinkAdd(veth) + assert.NoError(err) + + endpoint, err := createVethNetworkEndpoint(1, vethName, types.NetXConnectTCFilterModel) + assert.NoError(err) + + link, err := netlink.LinkByName(vethName) + assert.NoError(err) + + err = netHandle.LinkSetUp(link) + assert.NoError(err) + + err = setupTCFiltering(endpoint, 1, true) + assert.NoError(err) + + err = removeTCFiltering(endpoint) + assert.NoError(err) + + // Remove the veth created for testing. + err = netHandle.LinkDel(link) + assert.NoError(err) +} diff --git a/virtcontainers/physical_endpoint.go b/virtcontainers/hypervisor/physical_endpoint.go similarity index 87% rename from virtcontainers/physical_endpoint.go rename to virtcontainers/hypervisor/physical_endpoint.go index 17b60abf2f..478a5f0ae4 100644 --- a/virtcontainers/physical_endpoint.go +++ b/virtcontainers/hypervisor/physical_endpoint.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -14,7 +14,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/safchain/ethtool" ) @@ -22,7 +22,7 @@ import ( type PhysicalEndpoint struct { IfaceName string HardAddr string - EndpointProperties NetworkInfo + EndpointProperties types.NetworkInfo EndpointType EndpointType BDF string Driver string @@ -31,7 +31,7 @@ type PhysicalEndpoint struct { } // Properties returns the properties of the physical interface. -func (endpoint *PhysicalEndpoint) Properties() NetworkInfo { +func (endpoint *PhysicalEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -61,18 +61,18 @@ func (endpoint *PhysicalEndpoint) SetPciAddr(pciAddr string) { } // SetProperties sets the properties of the physical endpoint. -func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *PhysicalEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } // NetworkPair returns the network pair of the endpoint. -func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *PhysicalEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } // Attach for physical endpoint binds the physical network interface to // vfio-pci and adds device to the hypervisor with vfio-passthrough. -func (endpoint *PhysicalEndpoint) Attach(h hypervisor.Hypervisor) error { +func (endpoint *PhysicalEndpoint) Attach(h Hypervisor) error { // Unbind physical interface from host driver and bind to vfio // so that it can be passed to qemu. if err := bindNICToVFIO(endpoint); err != nil { @@ -84,7 +84,7 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor.Hypervisor) error { BDF: endpoint.BDF, } - return h.AddDevice(d, hypervisor.VfioDev) + return h.AddDevice(d, VfioDev) } // Detach for physical endpoint unbinds the physical network interface from vfio-pci @@ -100,12 +100,12 @@ func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) er } // HotAttach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor.Hypervisor) error { +func (endpoint *PhysicalEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("PhysicalEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *PhysicalEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("PhysicalEndpoint does not support Hot detach") } @@ -138,7 +138,7 @@ func isPhysicalIface(ifaceName string) (bool, error) { var sysPCIDevicesPath = "/sys/bus/pci/devices" -func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { +func createPhysicalEndpoint(netInfo types.NetworkInfo) (*PhysicalEndpoint, error) { // Get ethtool handle to derive driver and bus ethHandle, err := ethtool.NewEthtool() if err != nil { diff --git a/virtcontainers/physical_endpoint_test.go b/virtcontainers/hypervisor/physical_endpoint_test.go similarity index 96% rename from virtcontainers/physical_endpoint_test.go rename to virtcontainers/hypervisor/physical_endpoint_test.go index f22a8f1520..d00cd0ad87 100644 --- a/virtcontainers/physical_endpoint_test.go +++ b/virtcontainers/hypervisor/physical_endpoint_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" @@ -88,7 +88,7 @@ func TestIsPhysicalIface(t *testing.T) { } var isPhysical bool - err = doNetNS(n.Path(), func(_ ns.NetNS) error { + err = DoNetNS(n.Path(), func(_ ns.NetNS) error { var err error isPhysical, err = isPhysicalIface(testNetIface) return err diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/hypervisor/tap_endpoint.go similarity index 84% rename from virtcontainers/tap_endpoint.go rename to virtcontainers/hypervisor/tap_endpoint.go index 9535a2b59d..61be0f2d6c 100644 --- a/virtcontainers/tap_endpoint.go +++ b/virtcontainers/hypervisor/tap_endpoint.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -11,20 +11,20 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/types" ) // TapEndpoint represents just a tap endpoint type TapEndpoint struct { - TapInterface TapInterface - EndpointProperties NetworkInfo + TapInterface types.TapInterface + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } // Properties returns the properties of the tap interface. -func (endpoint *TapEndpoint) Properties() NetworkInfo { +func (endpoint *TapEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -54,17 +54,17 @@ func (endpoint *TapEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *TapEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *TapEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } // SetProperties sets the properties for the endpoint. -func (endpoint *TapEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *TapEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } // Attach for tap endpoint adds the tap interface to the hypervisor. -func (endpoint *TapEndpoint) Attach(h hypervisor.Hypervisor) error { +func (endpoint *TapEndpoint) Attach(h Hypervisor) error { return fmt.Errorf("TapEndpoint does not support Attach, if you're using docker please use --net none") } @@ -75,20 +75,20 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error { } networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint") - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return unTapNetwork(endpoint.TapInterface.TAPIface.Name) }) } // HotAttach for the tap endpoint uses hot plug device -func (endpoint *TapEndpoint) HotAttach(h hypervisor.Hypervisor) error { +func (endpoint *TapEndpoint) HotAttach(h Hypervisor) error { networkLogger().Info("Hot attaching tap endpoint") if err := tapNetwork(endpoint, h.Config().NumVCPUs, h.Config().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging tap ep") return err } - if _, err := h.HotplugAddDevice(endpoint, hypervisor.NetDev); err != nil { + if _, err := h.HotplugAddDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error attach tap ep") return err } @@ -96,15 +96,15 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor.Hypervisor) error { } // HotDetach for the tap endpoint uses hot pull device -func (endpoint *TapEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *TapEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { networkLogger().Info("Hot detaching tap endpoint") - if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + if err := DoNetNS(netNsPath, func(_ ns.NetNS) error { return unTapNetwork(endpoint.TapInterface.TAPIface.Name) }); err != nil { networkLogger().WithError(err).Warn("Error un-bridging tap ep") } - if _, err := h.HotplugRemoveDevice(endpoint, hypervisor.NetDev); err != nil { + if _, err := h.HotplugRemoveDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach tap ep") return err } @@ -118,10 +118,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { uniqueID := uuid.Generate().String() endpoint := &TapEndpoint{ - TapInterface: TapInterface{ + TapInterface: types.TapInterface{ ID: uniqueID, Name: fmt.Sprintf("eth%d", idx), - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: fmt.Sprintf("tap%d_kata", idx), }, }, diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/hypervisor/veth_endpoint.go similarity index 77% rename from virtcontainers/veth_endpoint.go rename to virtcontainers/hypervisor/veth_endpoint.go index aaad8a3132..ac9e4841a3 100644 --- a/virtcontainers/veth_endpoint.go +++ b/virtcontainers/hypervisor/veth_endpoint.go @@ -3,25 +3,25 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" ) // VethEndpoint gathers a network pair and its properties. type VethEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo + NetPair types.NetworkInterfacePair + EndpointProperties types.NetworkInfo Physical bool EndpointType EndpointType PCIAddr string } -func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) { +func createVethNetworkEndpoint(idx int, ifName string, interworkingModel types.NetInterworkingModel) (*VethEndpoint, error) { if idx < 0 { return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) } @@ -46,7 +46,7 @@ func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInte } // Properties returns properties for the veth interface in the network pair. -func (endpoint *VethEndpoint) Properties() NetworkInfo { +func (endpoint *VethEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -77,24 +77,24 @@ func (endpoint *VethEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *VethEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *VethEndpoint) NetworkPair() *types.NetworkInterfacePair { return &endpoint.NetPair } // SetProperties sets the properties for the endpoint. -func (endpoint *VethEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *VethEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } // Attach for veth endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *VethEndpoint) Attach(h hypervisor.Hypervisor) error { +func (endpoint *VethEndpoint) Attach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual endpoint") return err } - return h.AddDevice(endpoint, hypervisor.NetDev) + return h.AddDevice(endpoint, NetDev) } // Detach for the veth endpoint tears down the tap and bridge @@ -106,19 +106,19 @@ func (endpoint *VethEndpoint) Detach(netNsCreated bool, netNsPath string) error return nil } - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }) } // HotAttach for the veth endpoint uses hot plug device -func (endpoint *VethEndpoint) HotAttach(h hypervisor.Hypervisor) error { +func (endpoint *VethEndpoint) HotAttach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - if _, err := h.HotplugAddDevice(endpoint, hypervisor.NetDev); err != nil { + if _, err := h.HotplugAddDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error attach virtual ep") return err } @@ -126,18 +126,18 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor.Hypervisor) error { } // HotDetach for the veth endpoint uses hot pull device -func (endpoint *VethEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VethEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { if !netNsCreated { return nil } - if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + if err := DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }); err != nil { networkLogger().WithError(err).Warn("Error un-bridging virtual ep") } - if _, err := h.HotplugRemoveDevice(endpoint, hypervisor.NetDev); err != nil { + if _, err := h.HotplugRemoveDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach virtual ep") return err } diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/hypervisor/veth_endpoint_test.go similarity index 72% rename from virtcontainers/veth_endpoint_test.go rename to virtcontainers/hypervisor/veth_endpoint_test.go index ae62ddd525..0c16447aea 100644 --- a/virtcontainers/veth_endpoint_test.go +++ b/virtcontainers/hypervisor/veth_endpoint_test.go @@ -3,36 +3,38 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateVethNetworkEndpoint(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &VethEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth4", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, EndpointType: VethEndpointType, } - result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel) + result, err := createVethNetworkEndpoint(4, "", types.DefaultNetInterworkingModel) if err != nil { t.Fatal(err) } @@ -52,24 +54,24 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &VethEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth1", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, EndpointType: VethEndpointType, } - result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel) + result, err := createVethNetworkEndpoint(4, "eth1", types.DefaultNetInterworkingModel) if err != nil { t.Fatal(err) } @@ -98,7 +100,7 @@ func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) { } for _, d := range failingValues { - result, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel) + result, err := createVethNetworkEndpoint(d.idx, d.ifName, types.DefaultNetInterworkingModel) if err == nil { t.Fatalf("expected invalid endpoint for %v, got %v", d, result) } diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/hypervisor/vhostuser_endpoint.go similarity index 81% rename from virtcontainers/vhostuser_endpoint.go rename to virtcontainers/hypervisor/vhostuser_endpoint.go index 4c1f9a43c5..fae735c3b3 100644 --- a/virtcontainers/vhostuser_endpoint.go +++ b/virtcontainers/hypervisor/vhostuser_endpoint.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "encoding/hex" @@ -11,7 +11,7 @@ import ( "os" "github.com/kata-containers/runtime/virtcontainers/device/config" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -28,13 +28,13 @@ type VhostUserEndpoint struct { // MAC address of the interface HardAddr string IfaceName string - EndpointProperties NetworkInfo + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } // Properties returns the properties of the interface. -func (endpoint *VhostUserEndpoint) Properties() NetworkInfo { +func (endpoint *VhostUserEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -54,7 +54,7 @@ func (endpoint *VhostUserEndpoint) Type() EndpointType { } // SetProperties sets the properties of the endpoint. -func (endpoint *VhostUserEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *VhostUserEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } @@ -69,12 +69,12 @@ func (endpoint *VhostUserEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *VhostUserEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } // Attach for vhostuser endpoint -func (endpoint *VhostUserEndpoint) Attach(h hypervisor.Hypervisor) error { +func (endpoint *VhostUserEndpoint) Attach(h Hypervisor) error { // Generate a unique ID to be used for hypervisor commandline fields randBytes, err := utils.GenerateRandomBytes(8) if err != nil { @@ -89,7 +89,7 @@ func (endpoint *VhostUserEndpoint) Attach(h hypervisor.Hypervisor) error { Type: config.VhostUserNet, } - return h.AddDevice(d, hypervisor.VhostuserDev) + return h.AddDevice(d, VhostuserDev) } // Detach for vhostuser endpoint @@ -98,17 +98,17 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e } // HotAttach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor.Hypervisor) error { +func (endpoint *VhostUserEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("VhostUserEndpoint does not support Hot attach") } // HotDetach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor.Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VhostUserEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("VhostUserEndpoint does not support Hot detach") } // Create a vhostuser endpoint -func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { +func createVhostUserEndpoint(netInfo types.NetworkInfo, socket string) (*VhostUserEndpoint, error) { vhostUserEndpoint := &VhostUserEndpoint{ SocketPath: socket, @@ -121,7 +121,7 @@ func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndp // findVhostUserNetSocketPath checks if an interface is a dummy placeholder // for a vhost-user socket, and if it is it returns the path to the socket -func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { +func findVhostUserNetSocketPath(netInfo types.NetworkInfo) (string, error) { if netInfo.Iface.Name == "lo" { return "", nil } @@ -143,7 +143,7 @@ func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { func vhostUserSocketPath(info interface{}) (string, error) { switch v := info.(type) { - case NetworkInfo: + case types.NetworkInfo: return findVhostUserNetSocketPath(v) default: return "", nil diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/hypervisor/vhostuser_endpoint_test.go similarity index 93% rename from virtcontainers/vhostuser_endpoint_test.go rename to virtcontainers/hypervisor/vhostuser_endpoint_test.go index 6d1ad91e86..e13eddb013 100644 --- a/virtcontainers/vhostuser_endpoint_test.go +++ b/virtcontainers/hypervisor/vhostuser_endpoint_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestVhostUserSocketPath(t *testing.T) { @@ -47,7 +49,7 @@ func TestVhostUserSocketPath(t *testing.T) { if err != nil { t.Fatal(err) } - netinfo := NetworkInfo{ + netinfo := types.NetworkInfo{ Addrs: addresses, } @@ -66,7 +68,7 @@ func TestVhostUserSocketPath(t *testing.T) { }, }, } - netinfoFail := NetworkInfo{ + netinfoFail := types.NetworkInfo{ Addrs: addressesFalse, } @@ -135,8 +137,8 @@ func TestCreateVhostUserEndpoint(t *testing.T) { ifcName := "vhost-deadbeef" socket := "/tmp/vhu_192.168.0.1" - netinfo := NetworkInfo{ - Iface: NetlinkIface{ + netinfo := types.NetworkInfo{ + Iface: types.NetlinkIface{ LinkAttrs: netlink.LinkAttrs{ HardwareAddr: macAddr, Name: ifcName, diff --git a/virtcontainers/iostream_test.go b/virtcontainers/iostream_test.go index a394e958a9..c080db5acb 100644 --- a/virtcontainers/iostream_test.go +++ b/virtcontainers/iostream_test.go @@ -9,12 +9,13 @@ import ( "testing" "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) func TestIOStream(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{}, nil) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 1386614532..6e1a3505c9 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -616,7 +616,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { // // Setup network interfaces and routes // - interfaces, routes, err := generateInterfacesAndRoutes(sandbox.networkNS) + interfaces, routes, err := sandbox.networkNS.interfacesAndRoutes(sandbox.networkNS) if err != nil { return err } diff --git a/virtcontainers/monitor_test.go b/virtcontainers/monitor_test.go index 90dd5f2512..f7c9853520 100644 --- a/virtcontainers/monitor_test.go +++ b/virtcontainers/monitor_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) @@ -19,7 +20,7 @@ func TestMonitorSuccess(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } @@ -44,7 +45,7 @@ func TestMonitorClosedChannel(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 131645ae7d..d50e2476ae 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -7,1159 +7,136 @@ package virtcontainers import ( "context" - cryptoRand "crypto/rand" "encoding/json" "fmt" - "math/rand" - "net" "os" - "runtime" - "sort" - "time" "github.com/containernetworking/plugins/pkg/ns" opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" "golang.org/x/sys/unix" "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" - "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" - "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/kata-containers/runtime/virtcontainers/types" ) -// NetInterworkingModel defines the network model connecting -// the network interface to the virtual machine. -type NetInterworkingModel int - -const ( - // NetXConnectDefaultModel Ask to use DefaultNetInterworkingModel - NetXConnectDefaultModel NetInterworkingModel = iota - - // NetXConnectBridgedModel uses a linux bridge to interconnect - // the container interface to the VM. This is the - // safe default that works for most cases except - // macvlan and ipvlan - NetXConnectBridgedModel - - // NetXConnectMacVtapModel can be used when the Container network - // interface can be bridged using macvtap - NetXConnectMacVtapModel - - // NetXConnectEnlightenedModel can be used when the Network plugins - // are enlightened to create VM native interfaces - // when requested by the runtime - // This will be used for vethtap, macvtap, ipvtap - NetXConnectEnlightenedModel - - // NetXConnectTCFilterModel redirects traffic from the network interface - // provided by the network plugin to a tap interface. - // This works for ipvlan and macvlan as well. - NetXConnectTCFilterModel - - // NetXConnectNoneModel can be used when the VM is in the host network namespace - NetXConnectNoneModel - - // NetXConnectInvalidModel is the last item to check valid values by IsValid() - NetXConnectInvalidModel -) - -//IsValid checks if a model is valid -func (n NetInterworkingModel) IsValid() bool { - return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel) -} - -const ( - defaultNetModelStr = "default" - - bridgedNetModelStr = "bridged" - - macvtapNetModelStr = "macvtap" - - enlightenedNetModelStr = "enlightened" - - tcFilterNetModelStr = "tcfilter" - - noneNetModelStr = "none" -) - -//SetModel change the model string value -func (n *NetInterworkingModel) SetModel(modelName string) error { - switch modelName { - case defaultNetModelStr: - *n = DefaultNetInterworkingModel - return nil - case bridgedNetModelStr: - *n = NetXConnectBridgedModel - return nil - case macvtapNetModelStr: - *n = NetXConnectMacVtapModel - return nil - case enlightenedNetModelStr: - *n = NetXConnectEnlightenedModel - return nil - case tcFilterNetModelStr: - *n = NetXConnectTCFilterModel - return nil - case noneNetModelStr: - *n = NetXConnectNoneModel - return nil - } - return fmt.Errorf("Unknown type %s", modelName) -} - -// DefaultNetInterworkingModel is a package level default -// that determines how the VM should be connected to the -// the container network interface -var DefaultNetInterworkingModel = NetXConnectMacVtapModel - -// Introduces constants related to networking -const ( - defaultRouteDest = "0.0.0.0/0" - defaultRouteLabel = "default" - defaultFilePerms = 0600 - defaultQlen = 1500 -) - -// DNSInfo describes the DNS setup related to a network interface. -type DNSInfo struct { - Servers []string - Domain string - Searches []string - Options []string -} - -// NetlinkIface describes fully a network interface. -type NetlinkIface struct { - netlink.LinkAttrs - Type string -} - -// NetworkInfo gathers all information related to a network interface. -// It can be used to store the description of the underlying network. -type NetworkInfo struct { - Iface NetlinkIface - Addrs []netlink.Addr - Routes []netlink.Route - DNS DNSInfo -} - -// NetworkInterface defines a network interface. -type NetworkInterface struct { - Name string - HardAddr string - Addrs []netlink.Addr -} - -// TapInterface defines a tap interface -type TapInterface struct { - ID string - Name string - TAPIface NetworkInterface - VMFds []*os.File - VhostFds []*os.File -} - -// NetworkInterfacePair defines a pair between VM and virtual network interfaces. -type NetworkInterfacePair struct { - TapInterface - VirtIface NetworkInterface - NetInterworkingModel -} - -// NetworkConfig is the network configuration related to a network. -type NetworkConfig struct { - NetNSPath string - NetNsCreated bool - DisableNewNetNs bool - NetmonConfig NetmonConfig - InterworkingModel NetInterworkingModel -} - -func networkLogger() *logrus.Entry { - return virtLog.WithField("subsystem", "network") -} - -// NetworkNamespace contains all data related to its network namespace. -type NetworkNamespace struct { - NetNsPath string - NetNsCreated bool - Endpoints []Endpoint - NetmonPID int -} - -// TypedJSONEndpoint is used as an intermediate representation for -// marshalling and unmarshalling Endpoint objects. -type TypedJSONEndpoint struct { - Type EndpointType - Data json.RawMessage -} - -// MarshalJSON is the custom NetworkNamespace JSON marshalling routine. -// This is needed to properly marshall Endpoints array. -func (n NetworkNamespace) MarshalJSON() ([]byte, error) { - // We need a shadow structure in order to prevent json from - // entering a recursive loop when only calling json.Marshal(). - type shadow struct { - NetNsPath string - NetNsCreated bool - Endpoints []TypedJSONEndpoint - } - - s := &shadow{ - NetNsPath: n.NetNsPath, - NetNsCreated: n.NetNsCreated, - } - - var typedEndpoints []TypedJSONEndpoint - for _, endpoint := range n.Endpoints { - tempJSON, _ := json.Marshal(endpoint) - - t := TypedJSONEndpoint{ - Type: endpoint.Type(), - Data: tempJSON, - } - - typedEndpoints = append(typedEndpoints, t) - } - - s.Endpoints = typedEndpoints - - b, err := json.Marshal(s) - return b, err -} - -func generateEndpoints(typedEndpoints []TypedJSONEndpoint) ([]Endpoint, error) { - var endpoints []Endpoint - - for _, e := range typedEndpoints { - switch e.Type { - case PhysicalEndpointType: - var endpoint PhysicalEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "physical", - }).Info("endpoint unmarshalled") - - case VethEndpointType: - var endpoint VethEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "virtual", - }).Info("endpoint unmarshalled") - - case VhostUserEndpointType: - var endpoint VhostUserEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "vhostuser", - }).Info("endpoint unmarshalled") - - case BridgedMacvlanEndpointType: - var endpoint BridgedMacvlanEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "macvlan", - }).Info("endpoint unmarshalled") - - case MacvtapEndpointType: - var endpoint MacvtapEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "macvtap", - }).Info("endpoint unmarshalled") - - case TapEndpointType: - var endpoint TapEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "tap", - }).Info("endpoint unmarshalled") - - default: - networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type") - } - } - return endpoints, nil -} - -// UnmarshalJSON is the custom NetworkNamespace unmarshalling routine. -// This is needed for unmarshalling the Endpoints interfaces array. -func (n *NetworkNamespace) UnmarshalJSON(b []byte) error { - var s struct { - NetNsPath string - NetNsCreated bool - Endpoints json.RawMessage - } - - if err := json.Unmarshal(b, &s); err != nil { - return err - } - - (*n).NetNsPath = s.NetNsPath - (*n).NetNsCreated = s.NetNsCreated - - var typedEndpoints []TypedJSONEndpoint - if err := json.Unmarshal([]byte(string(s.Endpoints)), &typedEndpoints); err != nil { - return err - } - endpoints, err := generateEndpoints(typedEndpoints) - if err != nil { - return err - } - - (*n).Endpoints = endpoints - return nil -} - -func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, queues int) (netlink.Link, []*os.File, error) { - var newLink netlink.Link - var fds []*os.File - - switch expectedLink.Type() { - case (&netlink.Bridge{}).Type(): - newLink = &netlink.Bridge{ - LinkAttrs: netlink.LinkAttrs{Name: name}, - MulticastSnooping: expectedLink.(*netlink.Bridge).MulticastSnooping, - } - case (&netlink.Tuntap{}).Type(): - flags := netlink.TUNTAP_VNET_HDR - if queues > 0 { - flags |= netlink.TUNTAP_MULTI_QUEUE_DEFAULTS - } - newLink = &netlink.Tuntap{ - LinkAttrs: netlink.LinkAttrs{Name: name}, - Mode: netlink.TUNTAP_MODE_TAP, - Queues: queues, - Flags: flags, - } - case (&netlink.Macvtap{}).Type(): - qlen := expectedLink.Attrs().TxQLen - if qlen <= 0 { - qlen = defaultQlen - } - newLink = &netlink.Macvtap{ - Macvlan: netlink.Macvlan{ - Mode: netlink.MACVLAN_MODE_BRIDGE, - LinkAttrs: netlink.LinkAttrs{ - Index: expectedLink.Attrs().Index, - Name: name, - TxQLen: qlen, - ParentIndex: expectedLink.Attrs().ParentIndex, - }, - }, - } - default: - return nil, fds, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) - } - - if err := netHandle.LinkAdd(newLink); err != nil { - return nil, fds, fmt.Errorf("LinkAdd() failed for %s name %s: %s", expectedLink.Type(), name, err) - } - - tuntapLink, ok := newLink.(*netlink.Tuntap) - if ok { - fds = tuntapLink.Fds - } - - newLink, err := getLinkByName(netHandle, name, expectedLink) - return newLink, fds, err -} - -func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.Link, error) { - var link netlink.Link - - switch ep := endpoint.(type) { - case *VethEndpoint: - link = &netlink.Veth{} - case *BridgedMacvlanEndpoint: - link = &netlink.Macvlan{} - case *IPVlanEndpoint: - link = &netlink.IPVlan{} - default: - return nil, fmt.Errorf("Unexpected endpointType %s", ep.Type()) - } - - return getLinkByName(netHandle, endpoint.NetworkPair().VirtIface.Name, link) -} - -func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.Link) (netlink.Link, error) { - link, err := netHandle.LinkByName(name) - if err != nil { - return nil, fmt.Errorf("LinkByName() failed for %s name %s: %s", expectedLink.Type(), name, err) - } - - switch expectedLink.Type() { - case (&netlink.Bridge{}).Type(): - if l, ok := link.(*netlink.Bridge); ok { - return l, nil - } - case (&netlink.Tuntap{}).Type(): - if l, ok := link.(*netlink.GenericLink); ok { - return l, nil - } - case (&netlink.Veth{}).Type(): - if l, ok := link.(*netlink.Veth); ok { - return l, nil - } - case (&netlink.Macvtap{}).Type(): - if l, ok := link.(*netlink.Macvtap); ok { - return l, nil - } - case (&netlink.Macvlan{}).Type(): - if l, ok := link.(*netlink.Macvlan); ok { - return l, nil - } - case (&netlink.IPVlan{}).Type(): - if l, ok := link.(*netlink.IPVlan); ok { - return l, nil - } - default: - return nil, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) - } - - return nil, fmt.Errorf("Incorrect link type %s, expecting %s", link.Type(), expectedLink.Type()) -} - -// The endpoint type should dictate how the connection needs to happen. -func xConnectVMNetwork(endpoint Endpoint, h hypervisor.Hypervisor) error { - netPair := endpoint.NetworkPair() - - queues := 0 - caps := h.Capabilities() - if caps.IsMultiQueueSupported() { - queues = int(h.Config().NumVCPUs) - } - - disableVhostNet := h.Config().DisableVhostNet - - if netPair.NetInterworkingModel == NetXConnectDefaultModel { - netPair.NetInterworkingModel = DefaultNetInterworkingModel - } - - switch netPair.NetInterworkingModel { - case NetXConnectBridgedModel: - return bridgeNetworkPair(endpoint, queues, disableVhostNet) - case NetXConnectMacVtapModel: - return tapNetworkPair(endpoint, queues, disableVhostNet) - case NetXConnectTCFilterModel: - return setupTCFiltering(endpoint, queues, disableVhostNet) - case NetXConnectEnlightenedModel: - return fmt.Errorf("Unsupported networking model") - default: - return fmt.Errorf("Invalid internetworking model") - } -} - -// The endpoint type should dictate how the disconnection needs to happen. -func xDisconnectVMNetwork(endpoint Endpoint) error { - netPair := endpoint.NetworkPair() - - if netPair.NetInterworkingModel == NetXConnectDefaultModel { - netPair.NetInterworkingModel = DefaultNetInterworkingModel - } - - switch netPair.NetInterworkingModel { - case NetXConnectBridgedModel: - return unBridgeNetworkPair(endpoint) - case NetXConnectMacVtapModel: - return untapNetworkPair(endpoint) - case NetXConnectTCFilterModel: - return removeTCFiltering(endpoint) - case NetXConnectEnlightenedModel: - return fmt.Errorf("Unsupported networking model") - default: - return fmt.Errorf("Invalid internetworking model") - } -} - -func createMacvtapFds(linkIndex int, queues int) ([]*os.File, error) { - tapDev := fmt.Sprintf("/dev/tap%d", linkIndex) - return createFds(tapDev, queues) -} - -func createVhostFds(numFds int) ([]*os.File, error) { - vhostDev := "/dev/vhost-net" - return createFds(vhostDev, numFds) -} - -func createFds(device string, numFds int) ([]*os.File, error) { - fds := make([]*os.File, numFds) - - for i := 0; i < numFds; i++ { - f, err := os.OpenFile(device, os.O_RDWR, defaultFilePerms) - if err != nil { - utils.CleanupFds(fds, i) - return nil, err - } - fds[i] = f - } - return fds, nil -} - -// There is a limitation in the linux kernel that prevents a macvtap/macvlan link -// from getting the correct link index when created in a network namespace -// https://github.com/clearcontainers/runtime/issues/708 -// -// Till that bug is fixed we need to pick a random non conflicting index and try to -// create a link. If that fails, we need to try with another. -// All the kernel does not check if the link id conflicts with a link id on the host -// hence we need to offset the link id to prevent any overlaps with the host index -// -// Here the kernel will ensure that there is no race condition - -const hostLinkOffset = 8192 // Host should not have more than 8k interfaces -const linkRange = 0xFFFF // This will allow upto 2^16 containers -const linkRetries = 128 // The numbers of time we try to find a non conflicting index -const macvtapWorkaround = true - -func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, queues int) (taplink netlink.Link, err error) { - - if !macvtapWorkaround { - taplink, _, err = createLink(netHandle, name, link, queues) - return - } - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - - for i := 0; i < linkRetries; i++ { - index := hostLinkOffset + (r.Int() & linkRange) - link.Attrs().Index = index - taplink, _, err = createLink(netHandle, name, link, queues) - if err == nil { - break - } - } - - return -} - -func clearIPs(link netlink.Link, addrs []netlink.Addr) error { - for _, addr := range addrs { - if err := netlink.AddrDel(link, &addr); err != nil { - return err - } - } - return nil -} - -func setIPs(link netlink.Link, addrs []netlink.Addr) error { - for _, addr := range addrs { - if err := netlink.AddrAdd(link, &addr); err != nil { - return err - } - } - return nil -} - -func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - link, err := getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - attrs := link.Attrs() - - // Attach the macvtap interface to the underlying container - // interface. Also picks relevant attributes from the parent - tapLink, err := createMacVtap(netHandle, netPair.TAPIface.Name, - &netlink.Macvtap{ - Macvlan: netlink.Macvlan{ - LinkAttrs: netlink.LinkAttrs{ - TxQLen: attrs.TxQLen, - ParentIndex: attrs.Index, - }, - }, - }, queues) - - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } - - // Save the veth MAC address to the TAP so that it can later be used - // to build the hypervisor command line. This MAC address has to be - // the one inside the VM in order to avoid any firewall issues. The - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - tapHardAddr := attrs.HardwareAddr - netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - - if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) - } - - hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) - if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetHardwareAddr(tapLink, tapHardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) - } - - // Clear the IP addresses from the veth interface to prevent ARP conflict - netPair.VirtIface.Addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - return fmt.Errorf("Unable to obtain veth IP addresses: %s", err) - } - - if err := clearIPs(link, netPair.VirtIface.Addrs); err != nil { - return fmt.Errorf("Unable to clear veth IP addresses: %s", err) - } - - if err := netHandle.LinkSetUp(link); err != nil { - return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) - } - - // Note: The underlying interfaces need to be up prior to fd creation. - - netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, queues) - if err != nil { - return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err) - } - - if !disableVhostNet { - vhostFds, err := createVhostFds(queues) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) - } - netPair.VhostFds = vhostFds - } - - return nil -} - -func bridgeNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } - netPair.VMFds = fds - - if !disableVhostNet { - vhostFds, err := createVhostFds(queues) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) - } - netPair.VhostFds = vhostFds - } - - var attrs *netlink.LinkAttrs - var link netlink.Link - - link, err = getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - attrs = link.Attrs() - - // Save the veth MAC address to the TAP so that it can later be used - // to build the hypervisor command line. This MAC address has to be - // the one inside the VM in order to avoid any firewall issues. The - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - - if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) - } - - hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) - if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - mcastSnoop := false - bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, queues) - if err != nil { - return fmt.Errorf("Could not create bridge: %s", err) - } - - if err := netHandle.LinkSetMaster(tapLink, bridgeLink.(*netlink.Bridge)); err != nil { - return fmt.Errorf("Could not attach TAP %s to the bridge %s: %s", - netPair.TAPIface.Name, netPair.Name, err) - } - - if err := netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkSetMaster(link, bridgeLink.(*netlink.Bridge)); err != nil { - return fmt.Errorf("Could not attach veth %s to the bridge %s: %s", - netPair.VirtIface.Name, netPair.Name, err) - } - - if err := netHandle.LinkSetUp(link); err != nil { - return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetUp(bridgeLink); err != nil { - return fmt.Errorf("Could not enable bridge %s: %s", netPair.Name, err) - } - - return nil -} - -func setupTCFiltering(endpoint Endpoint, queues int, disableVhostNet bool) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } - netPair.VMFds = fds - - if !disableVhostNet { - vhostFds, err := createVhostFds(queues) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) - } - netPair.VhostFds = vhostFds - } - - var attrs *netlink.LinkAttrs - var link netlink.Link - - link, err = getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - attrs = link.Attrs() - - // Save the veth MAC address to the TAP so that it can later be used - // to build the hypervisor command line. This MAC address has to be - // the one inside the VM in order to avoid any firewall issues. The - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - - if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) - } - - if err := netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) - } - - tapAttrs := tapLink.Attrs() - - if err := addQdiscIngress(tapAttrs.Index); err != nil { - return err - } - - if err := addQdiscIngress(attrs.Index); err != nil { - return err - } - - if err := addRedirectTCFilter(attrs.Index, tapAttrs.Index); err != nil { - return err - } - - if err := addRedirectTCFilter(tapAttrs.Index, attrs.Index); err != nil { - return err - } - - return nil -} - -// addQdiscIngress creates a new qdisc for nwtwork interface with the specified network index -// on "ingress". qdiscs normally don't work on ingress so this is really a special qdisc -// that you can consider an "alternate root" for inbound packets. -// Handle for ingress qdisc defaults to "ffff:" -// -// This is equivalent to calling `tc qdisc add dev eth0 ingress` -func addQdiscIngress(index int) error { - qdisc := &netlink.Ingress{ - QdiscAttrs: netlink.QdiscAttrs{ - LinkIndex: index, - Parent: netlink.HANDLE_INGRESS, - }, - } - - err := netlink.QdiscAdd(qdisc) - if err != nil { - return fmt.Errorf("Failed to add qdisc for network index %d : %s", index, err) - } - - return nil -} - -// addRedirectTCFilter adds a tc filter for device with index "sourceIndex". -// All traffic for interface with index "sourceIndex" is redirected to interface with -// index "destIndex" -// -// This is equivalent to calling: -// `tc filter add dev source parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev dest` -func addRedirectTCFilter(sourceIndex, destIndex int) error { - filter := &netlink.U32{ - FilterAttrs: netlink.FilterAttrs{ - LinkIndex: sourceIndex, - Parent: netlink.MakeHandle(0xffff, 0), - Protocol: unix.ETH_P_ALL, - }, - Actions: []netlink.Action{ - &netlink.MirredAction{ - ActionAttrs: netlink.ActionAttrs{ - Action: netlink.TC_ACT_STOLEN, - }, - MirredAction: netlink.TCA_EGRESS_REDIR, - Ifindex: destIndex, - }, - }, - } - - if err := netlink.FilterAdd(filter); err != nil { - return fmt.Errorf("Failed to add filter for index %d : %s", sourceIndex, err) - } - - return nil -} - -// removeRedirectTCFilter removes all tc u32 filters created on ingress qdisc for "link". -func removeRedirectTCFilter(link netlink.Link) error { - if link == nil { - return nil - } - - // Handle 0xffff is used for ingress - filters, err := netlink.FilterList(link, netlink.MakeHandle(0xffff, 0)) - if err != nil { - return err - } - - for _, f := range filters { - u32, ok := f.(*netlink.U32) - - if !ok { - continue - } - - if err := netlink.FilterDel(u32); err != nil { - return err - } - } - return nil -} - -// removeQdiscIngress removes the ingress qdisc previously created on "link". -func removeQdiscIngress(link netlink.Link) error { - if link == nil { - return nil - } - - qdiscs, err := netlink.QdiscList(link) - if err != nil { - return err - } - - for _, qdisc := range qdiscs { - ingress, ok := qdisc.(*netlink.Ingress) - if !ok { - continue - } - - if err := netlink.QdiscDel(ingress); err != nil { - return err - } - } - return nil +func networkLogger() *logrus.Entry { + return virtLog.WithField("subsystem", "network") } -func untapNetworkPair(endpoint Endpoint) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Macvtap{}) - if err != nil { - return fmt.Errorf("Could not get TAP interface %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkDel(tapLink); err != nil { - return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) - } - - link, err := getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) +func createNetNS() (string, error) { + n, err := ns.NewNS() if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetDown(link); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + return "", err } - // Restore the IPs that were cleared - err = setIPs(link, netPair.VirtIface.Addrs) - return err + return n.Path(), nil } -func unBridgeNetworkPair(endpoint Endpoint) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) - if err != nil { - return fmt.Errorf("Could not get TAP interface: %s", err) - } - - bridgeLink, err := getLinkByName(netHandle, netPair.Name, &netlink.Bridge{}) - if err != nil { - return fmt.Errorf("Could not get bridge interface: %s", err) - } - - if err := netHandle.LinkSetDown(bridgeLink); err != nil { - return fmt.Errorf("Could not disable bridge %s: %s", netPair.Name, err) - } - - if err := netHandle.LinkSetDown(tapLink); err != nil { - return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkSetNoMaster(tapLink); err != nil { - return fmt.Errorf("Could not detach TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkDel(bridgeLink); err != nil { - return fmt.Errorf("Could not remove bridge %s: %s", netPair.Name, err) - } - - if err := netHandle.LinkDel(tapLink); err != nil { - return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) - } - - link, err := getLinkForEndpoint(endpoint, netHandle) +func deleteNetNS(netNSPath string) error { + n, err := ns.GetNS(netNSPath) if err != nil { return err } - hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) + err = n.Close() if err != nil { return err } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - if err := netHandle.LinkSetDown(link); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + if err = unix.Unmount(netNSPath, unix.MNT_DETACH); err != nil { + return fmt.Errorf("Failed to unmount namespace %s: %v", netNSPath, err) } - - if err := netHandle.LinkSetNoMaster(link); err != nil { - return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) + if err := os.RemoveAll(netNSPath); err != nil { + return fmt.Errorf("Failed to clean up namespace %s: %v", netNSPath, err) } return nil } -func removeTCFiltering(endpoint Endpoint) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) - if err != nil { - return fmt.Errorf("Could not get TAP interface: %s", err) - } - - if err := netHandle.LinkSetDown(tapLink); err != nil { - return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) - } +// NetworkNamespace contains all data related to its network namespace. +type NetworkNamespace struct { + NetNsPath string + NetNsCreated bool + Endpoints []hypervisor.Endpoint + NetmonPID int +} - if err := netHandle.LinkDel(tapLink); err != nil { - return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) +// MarshalJSON is the custom NetworkNamespace JSON marshalling routine. +// This is needed to properly marshall Endpoints array. +func (n NetworkNamespace) MarshalJSON() ([]byte, error) { + // We need a shadow structure in order to prevent json from + // entering a recursive loop when only calling json.Marshal(). + type shadow struct { + NetNsPath string + NetNsCreated bool + Endpoints []hypervisor.TypedJSONEndpoint } - link, err := getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err + s := &shadow{ + NetNsPath: n.NetNsPath, + NetNsCreated: n.NetNsCreated, } - if err := removeRedirectTCFilter(link); err != nil { - return err - } + var typedEndpoints []hypervisor.TypedJSONEndpoint + for _, ep := range n.Endpoints { + tempJSON, _ := json.Marshal(ep) - if err := removeQdiscIngress(link); err != nil { - return err - } + t := hypervisor.TypedJSONEndpoint{ + Type: ep.Type(), + Data: tempJSON, + } - if err := netHandle.LinkSetDown(link); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + typedEndpoints = append(typedEndpoints, t) } - return nil -} - -func createNetNS() (string, error) { - n, err := ns.NewNS() - if err != nil { - return "", err - } + s.Endpoints = typedEndpoints - return n.Path(), nil + b, err := json.Marshal(s) + return b, err } -// doNetNS is free from any call to a go routine, and it calls -// into runtime.LockOSThread(), meaning it won't be executed in a -// different thread than the one expected by the caller. -func doNetNS(netNSPath string, cb func(ns.NetNS) error) error { - // if netNSPath is empty, the callback function will be run in the current network namespace. - // So skip the whole function, just call cb(). cb() needs a NetNS as arg but ignored, give it a fake one. - if netNSPath == "" { - var netNs ns.NetNS - return cb(netNs) - } - - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - currentNS, err := ns.GetCurrentNS() - if err != nil { - return err - } - defer currentNS.Close() - - targetNS, err := ns.GetNS(netNSPath) - if err != nil { - return err +// UnmarshalJSON is the custom NetworkNamespace unmarshalling routine. +// This is needed for unmarshalling the Endpoints interfaces array. +func (n *NetworkNamespace) UnmarshalJSON(b []byte) error { + var s struct { + NetNsPath string + NetNsCreated bool + Endpoints json.RawMessage } - if err := targetNS.Set(); err != nil { + if err := json.Unmarshal(b, &s); err != nil { return err } - defer currentNS.Set() - return cb(targetNS) -} + (*n).NetNsPath = s.NetNsPath + (*n).NetNsCreated = s.NetNsCreated -func deleteNetNS(netNSPath string) error { - n, err := ns.GetNS(netNSPath) - if err != nil { + var typedEndpoints []hypervisor.TypedJSONEndpoint + if err := json.Unmarshal([]byte(string(s.Endpoints)), &typedEndpoints); err != nil { return err } - - err = n.Close() + endpoints, err := hypervisor.UnmarshalEndpoints(typedEndpoints) if err != nil { return err } - if err = unix.Unmount(netNSPath, unix.MNT_DETACH); err != nil { - return fmt.Errorf("Failed to unmount namespace %s: %v", netNSPath, err) - } - if err := os.RemoveAll(netNSPath); err != nil { - return fmt.Errorf("Failed to clean up namespace %s: %v", netNSPath, err) - } - + (*n).Endpoints = endpoints return nil } -func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) { +func (n NetworkNamespace) interfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) { - if networkNS.NetNsPath == "" { + if n.NetNsPath == "" { return nil, nil, nil } var routes []*vcTypes.Route var ifaces []*vcTypes.Interface - for _, endpoint := range networkNS.Endpoints { + for _, endpoint := range n.Endpoints { var ipAddresses []*vcTypes.IPAddress for _, addr := range endpoint.Properties().Addrs { @@ -1238,186 +215,6 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa return ifaces, routes, nil } -func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { - uniqueID := uuid.Generate().String() - - randomMacAddr, err := generateRandomPrivateMacAddr() - if err != nil { - return NetworkInterfacePair{}, fmt.Errorf("Could not generate random mac address: %s", err) - } - - netPair := NetworkInterfacePair{ - TapInterface: TapInterface{ - ID: uniqueID, - Name: fmt.Sprintf("br%d_kata", idx), - TAPIface: NetworkInterface{ - Name: fmt.Sprintf("tap%d_kata", idx), - }, - }, - VirtIface: NetworkInterface{ - Name: fmt.Sprintf("eth%d", idx), - HardAddr: randomMacAddr, - }, - NetInterworkingModel: interworkingModel, - } - - return netPair, nil -} - -func generateRandomPrivateMacAddr() (string, error) { - buf := make([]byte, 6) - _, err := cryptoRand.Read(buf) - if err != nil { - return "", err - } - - // Set the local bit for local addresses - // Addresses in this range are local mac addresses: - // x2-xx-xx-xx-xx-xx , x6-xx-xx-xx-xx-xx , xA-xx-xx-xx-xx-xx , xE-xx-xx-xx-xx-xx - buf[0] = (buf[0] | 2) & 0xfe - - hardAddr := net.HardwareAddr(buf) - return hardAddr.String(), nil -} - -func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo, error) { - addrs, err := handle.AddrList(link, netlink.FAMILY_ALL) - if err != nil { - return NetworkInfo{}, err - } - - routes, err := handle.RouteList(link, netlink.FAMILY_ALL) - if err != nil { - return NetworkInfo{}, err - } - - return NetworkInfo{ - Iface: NetlinkIface{ - LinkAttrs: *(link.Attrs()), - Type: link.Type(), - }, - Addrs: addrs, - Routes: routes, - }, nil -} - -func createEndpointsFromScan(networkNSPath string, config *NetworkConfig) ([]Endpoint, error) { - var endpoints []Endpoint - - netnsHandle, err := netns.GetFromPath(networkNSPath) - if err != nil { - return []Endpoint{}, err - } - defer netnsHandle.Close() - - netlinkHandle, err := netlink.NewHandleAt(netnsHandle) - if err != nil { - return []Endpoint{}, err - } - defer netlinkHandle.Delete() - - linkList, err := netlinkHandle.LinkList() - if err != nil { - return []Endpoint{}, err - } - - idx := 0 - for _, link := range linkList { - var ( - endpoint Endpoint - errCreate error - ) - - netInfo, err := networkInfoFromLink(netlinkHandle, link) - if err != nil { - return []Endpoint{}, err - } - - // Ignore unconfigured network interfaces. These are - // either base tunnel devices that are not namespaced - // like gre0, gretap0, sit0, ipip0, tunl0 or incorrectly - // setup interfaces. - if len(netInfo.Addrs) == 0 { - continue - } - - // Skip any loopback interfaces: - if (netInfo.Iface.Flags & net.FlagLoopback) != 0 { - continue - } - - if err := doNetNS(networkNSPath, func(_ ns.NetNS) error { - endpoint, errCreate = createEndpoint(netInfo, idx, config.InterworkingModel) - return errCreate - }); err != nil { - return []Endpoint{}, err - } - - endpoint.SetProperties(netInfo) - endpoints = append(endpoints, endpoint) - - idx++ - } - - sort.Slice(endpoints, func(i, j int) bool { - return endpoints[i].Name() < endpoints[j].Name() - }) - - networkLogger().WithField("endpoints", endpoints).Info("Endpoints found after scan") - - return endpoints, nil -} - -func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (Endpoint, error) { - var endpoint Endpoint - // TODO: This is the incoming interface - // based on the incoming interface we should create - // an appropriate EndPoint based on interface type - // This should be a switch - - // Check if interface is a physical interface. Do not create - // tap interface/bridge if it is. - isPhysical, err := isPhysicalIface(netInfo.Iface.Name) - if err != nil { - return nil, err - } - - if isPhysical { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") - endpoint, err = createPhysicalEndpoint(netInfo) - } else { - var socketPath string - - // Check if this is a dummy interface which has a vhost-user socket associated with it - socketPath, err = vhostUserSocketPath(netInfo) - if err != nil { - return nil, err - } - - if socketPath != "" { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") - endpoint, err = createVhostUserEndpoint(netInfo, socketPath) - } else if netInfo.Iface.Type == "macvlan" { - networkLogger().Infof("macvlan interface found") - endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) - } else if netInfo.Iface.Type == "macvtap" { - networkLogger().Infof("macvtap interface found") - endpoint, err = createMacvtapNetworkEndpoint(netInfo) - } else if netInfo.Iface.Type == "tap" { - networkLogger().Info("tap interface found") - endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) - } else if netInfo.Iface.Type == "veth" { - endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) - } else if netInfo.Iface.Type == "ipvlan" { - endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) - } else { - return nil, fmt.Errorf("Unsupported network interface") - } - } - - return endpoint, err -} - // Network is the virtcontainer network structure type Network struct { } @@ -1436,22 +233,22 @@ func (n *Network) Run(networkNSPath string, cb func() error) error { span, _ := n.trace(context.Background(), "run") defer span.Finish() - return doNetNS(networkNSPath, func(_ ns.NetNS) error { + return hypervisor.DoNetNS(networkNSPath, func(_ ns.NetNS) error { return cb() }) } // Add adds all needed interfaces inside the network namespace. -func (n *Network) Add(ctx context.Context, config *NetworkConfig, h hypervisor.Hypervisor, hotplug bool) ([]Endpoint, error) { +func (n *Network) Add(ctx context.Context, config *types.NetworkConfig, h hypervisor.Hypervisor, hotplug bool) ([]hypervisor.Endpoint, error) { span, _ := n.trace(ctx, "add") defer span.Finish() - endpoints, err := createEndpointsFromScan(config.NetNSPath, config) + endpoints, err := hypervisor.CreateEndpointsFromScan(config.NetNSPath, config) if err != nil { return endpoints, err } - err = doNetNS(config.NetNSPath, func(_ ns.NetNS) error { + err = hypervisor.DoNetNS(config.NetNSPath, func(_ ns.NetNS) error { for _, endpoint := range endpoints { networkLogger().WithField("endpoint-type", endpoint.Type()).WithField("hotplug", hotplug).Info("Attaching endpoint") if hotplug { @@ -1468,7 +265,7 @@ func (n *Network) Add(ctx context.Context, config *NetworkConfig, h hypervisor.H return nil }) if err != nil { - return []Endpoint{}, err + return []hypervisor.Endpoint{}, err } networkLogger().Debug("Network added") diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go index c1d4804f45..3f6bb5f2d7 100644 --- a/virtcontainers/network_test.go +++ b/virtcontainers/network_test.go @@ -11,7 +11,9 @@ import ( "reflect" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" ) @@ -63,8 +65,8 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {LinkIndex: 329, Dst: dst2, Src: src2, Gw: gw2}, } - networkInfo := NetworkInfo{ - Iface: NetlinkIface{ + networkInfo := types.NetworkInfo{ + Iface: types.NetlinkIface{ LinkAttrs: netlink.LinkAttrs{MTU: 1500}, Type: "", }, @@ -72,17 +74,17 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { Routes: routes, } - ep0 := &PhysicalEndpoint{ + ep0 := &hypervisor.PhysicalEndpoint{ IfaceName: "eth0", HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), EndpointProperties: networkInfo, } - endpoints := []Endpoint{ep0} + endpoints := []hypervisor.Endpoint{ep0} nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints} - resInterfaces, resRoutes, err := generateInterfacesAndRoutes(nns) + resInterfaces, resRoutes, err := nns.interfacesAndRoutes(nns) // // Build expected results: @@ -108,199 +110,3 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { "Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes) } - -func TestNetInterworkingModelIsValid(t *testing.T) { - tests := []struct { - name string - n NetInterworkingModel - want bool - }{ - {"Invalid Model", NetXConnectInvalidModel, false}, - {"Default Model", NetXConnectDefaultModel, true}, - {"Bridged Model", NetXConnectBridgedModel, true}, - {"TC Filter Model", NetXConnectTCFilterModel, true}, - {"Macvtap Model", NetXConnectMacVtapModel, true}, - {"Enlightened Model", NetXConnectEnlightenedModel, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.n.IsValid(); got != tt.want { - t.Errorf("NetInterworkingModel.IsValid() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestNetInterworkingModelSetModel(t *testing.T) { - var n NetInterworkingModel - tests := []struct { - name string - modelName string - wantErr bool - }{ - {"Invalid Model", "Invalid", true}, - {"default Model", defaultNetModelStr, false}, - {"bridged Model", bridgedNetModelStr, false}, - {"macvtap Model", macvtapNetModelStr, false}, - {"enlightened Model", enlightenedNetModelStr, false}, - {"tcfilter Model", tcFilterNetModelStr, false}, - {"none Model", noneNetModelStr, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := n.SetModel(tt.modelName); (err != nil) != tt.wantErr { - t.Errorf("NetInterworkingModel.SetModel() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestGenerateRandomPrivateMacAdd(t *testing.T) { - assert := assert.New(t) - - addr1, err := generateRandomPrivateMacAddr() - assert.NoError(err) - - _, err = net.ParseMAC(addr1) - assert.NoError(err) - - addr2, err := generateRandomPrivateMacAddr() - assert.NoError(err) - - _, err = net.ParseMAC(addr2) - assert.NoError(err) - - assert.NotEqual(addr1, addr2) -} - -func TestCreateGetBridgeLink(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - defer netHandle.Delete() - - assert.NoError(err) - - brName := "testbr0" - brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) - assert.NoError(err) - assert.NotNil(brLink) - - brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) - assert.NoError(err) - - err = netHandle.LinkDel(brLink) - assert.NoError(err) -} - -func TestCreateGetTunTapLink(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - defer netHandle.Delete() - - assert.NoError(err) - - tapName := "testtap0" - tapLink, fds, err := createLink(netHandle, tapName, &netlink.Tuntap{}, 1) - assert.NoError(err) - assert.NotNil(tapLink) - assert.NotZero(len(fds)) - - tapLink, err = getLinkByName(netHandle, tapName, &netlink.Tuntap{}) - assert.NoError(err) - - err = netHandle.LinkDel(tapLink) - assert.NoError(err) -} - -func TestCreateMacVtap(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - defer netHandle.Delete() - - assert.NoError(err) - - brName := "testbr0" - brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) - assert.NoError(err) - - attrs := brLink.Attrs() - - mcLink := &netlink.Macvtap{ - Macvlan: netlink.Macvlan{ - LinkAttrs: netlink.LinkAttrs{ - TxQLen: attrs.TxQLen, - ParentIndex: attrs.Index, - }, - }, - } - - macvtapName := "testmc0" - _, err = createMacVtap(netHandle, macvtapName, mcLink, 1) - assert.NoError(err) - - macvtapLink, err := getLinkByName(netHandle, macvtapName, &netlink.Macvtap{}) - assert.NoError(err) - - err = netHandle.LinkDel(macvtapLink) - assert.NoError(err) - - brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) - assert.NoError(err) - - err = netHandle.LinkDel(brLink) - assert.NoError(err) -} - -func TestTcRedirectNetwork(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - assert.NoError(err) - defer netHandle.Delete() - - // Create a test veth interface. - vethName := "foo" - veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: vethName, TxQLen: 200, MTU: 1400}, PeerName: "bar"} - - err = netlink.LinkAdd(veth) - assert.NoError(err) - - endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel) - assert.NoError(err) - - link, err := netlink.LinkByName(vethName) - assert.NoError(err) - - err = netHandle.LinkSetUp(link) - assert.NoError(err) - - err = setupTCFiltering(endpoint, 1, true) - assert.NoError(err) - - err = removeTCFiltering(endpoint) - assert.NoError(err) - - // Remove the veth created for testing. - err = netHandle.LinkDel(link) - assert.NoError(err) -} diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 991870f22d..06557ded1d 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -119,7 +119,7 @@ type RuntimeConfig struct { //Determines how the VM should be connected to the //the container network interface - InterNetworkModel vc.NetInterworkingModel + InterNetworkModel types.NetInterworkingModel FactoryConfig FactoryConfig Debug bool Trace bool @@ -316,13 +316,13 @@ func ContainerCapabilities(s CompatOCISpec) (types.LinuxCapabilities, error) { return containerCapabilities(s) } -func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.NetworkConfig, error) { +func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (types.NetworkConfig, error) { linux := ocispec.Linux if linux == nil { - return vc.NetworkConfig{}, ErrNoLinux + return types.NetworkConfig{}, ErrNoLinux } - var netConf vc.NetworkConfig + var netConf types.NetworkConfig for _, n := range linux.Namespaces { if n.Type != spec.NetworkNamespace { @@ -336,7 +336,7 @@ func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.NetworkConfi netConf.InterworkingModel = config.InterNetworkModel netConf.DisableNewNetNs = config.DisableNewNetNs - netConf.NetmonConfig = vc.NetmonConfig{ + netConf.NetmonConfig = types.NetmonConfig{ Path: config.NetmonConfig.Path, Debug: config.NetmonConfig.Debug, Enable: config.NetmonConfig.Enable, diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index ccd0180d03..3f367d0816 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -216,7 +216,7 @@ func TestMinimalSandboxConfig(t *testing.T) { }}, } - expectedNetworkConfig := vc.NetworkConfig{} + expectedNetworkConfig := types.NetworkConfig{} expectedSandboxConfig := vc.SandboxConfig{ ID: containerID, diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 38c0a657fa..f3d230c445 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -928,20 +928,20 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) } -func (q *qemu) hotplugNetDevice(endpoint Endpoint, op hypervisor.Operation) error { +func (q *qemu) hotplugNetDevice(endpoint hypervisor.Endpoint, op hypervisor.Operation) error { err := q.qmpSetup() if err != nil { return err } - var tap TapInterface + var tap types.TapInterface devID := "virtio-" + tap.ID switch endpoint.Type() { - case VethEndpointType: - drive := endpoint.(*VethEndpoint) + case hypervisor.VethEndpointType: + drive := endpoint.(*hypervisor.VethEndpoint) tap = drive.NetPair.TapInterface - case TapEndpointType: - drive := endpoint.(*TapEndpoint) + case hypervisor.TapEndpointType: + drive := endpoint.(*hypervisor.TapEndpoint) tap = drive.TapInterface default: return fmt.Errorf("this endpoint is not supported") @@ -1000,7 +1000,7 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType hypervisor.Device, op memdev := devInfo.(*hypervisor.MemoryDevice) return q.hotplugMemory(memdev, op) case hypervisor.NetDev: - device := devInfo.(Endpoint) + device := devInfo.(hypervisor.Endpoint) return nil, q.hotplugNetDevice(device, op) default: return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) @@ -1244,7 +1244,7 @@ func (q *qemu) AddDevice(devInfo interface{}, devType hypervisor.Device) error { case kataVSOCK: q.fds = append(q.fds, v.vhostFd) q.qemuConfig.Devices = q.arch.appendVSockPCI(q.qemuConfig.Devices, v) - case Endpoint: + case hypervisor.Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) case config.BlockDrive: q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v) diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 48da96d941..d98c707ecc 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -82,7 +82,7 @@ type qemuArch interface { appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOCK) []govmmQemu.Device // appendNetwork appends a endpoint device to devices - appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device + appendNetwork(devices []govmmQemu.Device, endpoint hypervisor.Endpoint) []govmmQemu.Device // appendBlockDevice appends a block drive to devices appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device @@ -431,11 +431,11 @@ func (q *qemuArchBase) appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOC } -func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType { +func networkModelToQemuType(model types.NetInterworkingModel) govmmQemu.NetDeviceType { switch model { - case NetXConnectBridgedModel: + case types.NetXConnectBridgedModel: return govmmQemu.MACVTAP //TODO: We should rename MACVTAP to .NET_FD - case NetXConnectMacVtapModel: + case types.NetXConnectMacVtapModel: return govmmQemu.MACVTAP //case ModelEnlightened: // Here the Network plugin will create a VM native interface @@ -448,9 +448,9 @@ func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType } } -func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device { +func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint hypervisor.Endpoint) []govmmQemu.Device { switch ep := endpoint.(type) { - case *VethEndpoint, *BridgedMacvlanEndpoint, *IPVlanEndpoint: + case *hypervisor.VethEndpoint, *hypervisor.BridgedMacvlanEndpoint, *hypervisor.IPVlanEndpoint: netPair := ep.NetworkPair() devices = append(devices, govmmQemu.NetDevice{ @@ -468,7 +468,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi }, ) q.networkIndex++ - case *MacvtapEndpoint: + case *hypervisor.MacvtapEndpoint: devices = append(devices, govmmQemu.NetDevice{ Type: govmmQemu.MACVTAP, diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 8c284b1df1..665c17c3a3 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -446,28 +446,28 @@ func TestQemuArchBaseAppendNetwork(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} - macvlanEp := &BridgedMacvlanEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + macvlanEp := &hypervisor.BridgedMacvlanEndpoint{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth4", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, - EndpointType: BridgedMacvlanEndpointType, + EndpointType: hypervisor.BridgedMacvlanEndpointType, } - macvtapEp := &MacvtapEndpoint{ - EndpointType: MacvtapEndpointType, - EndpointProperties: NetworkInfo{ - Iface: NetlinkIface{ + macvtapEp := &hypervisor.MacvtapEndpoint{ + EndpointType: hypervisor.MacvtapEndpointType, + EndpointProperties: types.NetworkInfo{ + Iface: types.NetlinkIface{ Type: "macvtap", }, }, diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 35440641e4..c3a3a11187 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -71,7 +71,7 @@ type SandboxConfig struct { ShimType ShimType ShimConfig interface{} - NetworkConfig NetworkConfig + NetworkConfig types.NetworkConfig // Volumes is a list of shared volumes between the host and the Sandbox. Volumes []types.Volume @@ -807,10 +807,10 @@ func (s *Sandbox) removeNetwork() error { return s.network.Remove(s.ctx, &s.networkNS, s.hypervisor, s.factory != nil) } -func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { +func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (types.NetworkInfo, error) { hw, err := net.ParseMAC(inf.HwAddr) if err != nil { - return NetworkInfo{}, err + return types.NetworkInfo{}, err } var addrs []netlink.Addr @@ -818,14 +818,14 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask) netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr) if err != nil { - return NetworkInfo{}, fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err) + return types.NetworkInfo{}, fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err) } addrs = append(addrs, *netlinkAddr) } - return NetworkInfo{ - Iface: NetlinkIface{ + return types.NetworkInfo{ + Iface: types.NetlinkIface{ LinkAttrs: netlink.LinkAttrs{ Name: inf.Name, HardwareAddr: hw, @@ -844,13 +844,13 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro return nil, err } - endpoint, err := createEndpoint(netInfo, len(s.networkNS.Endpoints), s.config.NetworkConfig.InterworkingModel) + endpoint, err := hypervisor.CreateEndpoint(netInfo, len(s.networkNS.Endpoints), s.config.NetworkConfig.InterworkingModel) if err != nil { return nil, err } endpoint.SetProperties(netInfo) - if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { + if err := hypervisor.DoNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint") return endpoint.HotAttach(s.hypervisor) }); err != nil { @@ -1324,7 +1324,7 @@ func (s *Sandbox) Stop() error { return err } - // Remove the network. + // Remove the hypervisor. return s.removeNetwork() } diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index 7bf72161a2..b831aecab6 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -43,7 +43,7 @@ func newHypervisorConfig(kernelParams []hypervisor.Param, hParams []hypervisor.P func testCreateSandbox(t *testing.T, id string, htype hypervisor.Type, hconfig hypervisor.Config, atype AgentType, - nconfig NetworkConfig, containers []ContainerConfig, + nconfig types.NetworkConfig, containers []ContainerConfig, volumes []types.Volume) (*Sandbox, error) { sconfig := SandboxConfig{ @@ -81,7 +81,7 @@ func testCreateSandbox(t *testing.T, id string, } func TestCreateEmptySandbox(t *testing.T) { - _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hypervisor.Config{}, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hypervisor.Config{}, NoopAgentType, types.NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("VirtContainers should not allow empty sandboxes") } @@ -89,7 +89,7 @@ func TestCreateEmptySandbox(t *testing.T) { } func TestCreateEmptyHypervisorSandbox(t *testing.T) { - _, err := testCreateSandbox(t, testSandboxID, hypervisor.Qemu, hypervisor.Config{}, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Qemu, hypervisor.Config{}, NoopAgentType, types.NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("VirtContainers should not allow sandboxes with empty hypervisors") } @@ -99,7 +99,7 @@ func TestCreateEmptyHypervisorSandbox(t *testing.T) { func TestCreateMockSandbox(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err != nil { t.Fatal(err) } @@ -109,7 +109,7 @@ func TestCreateMockSandbox(t *testing.T) { func TestCreateSandboxEmptyID(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, "", hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, "", hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("Expected sandbox with empty ID to fail, but got sandbox %v", p) } @@ -119,7 +119,7 @@ func TestCreateSandboxEmptyID(t *testing.T) { func testSandboxStateTransition(t *testing.T, state types.StateString, newState types.StateString) error { hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err != nil { return err } @@ -451,7 +451,7 @@ func TestSandboxSetSandboxAndContainerState(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } @@ -649,7 +649,7 @@ func TestSandboxGetContainer(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err != nil { t.Fatal(err) } @@ -695,7 +695,7 @@ func TestContainerSetStateBlockIndex(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) + sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, containers, nil) if err != nil { t.Fatal(err) } @@ -790,7 +790,7 @@ func TestContainerStateSetFstype(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) + sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, containers, nil) if err != nil { t.Fatal(err) } @@ -1062,7 +1062,7 @@ func TestRemoveContainerSuccess(t *testing.T) { } func TestCreateContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1073,7 +1073,7 @@ func TestCreateContainer(t *testing.T) { } func TestDeleteContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1090,7 +1090,7 @@ func TestDeleteContainer(t *testing.T) { } func TestStartContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1110,7 +1110,7 @@ func TestStartContainer(t *testing.T) { } func TestStatusContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1130,7 +1130,7 @@ func TestStatusContainer(t *testing.T) { } func TestStatusSandbox(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1138,7 +1138,7 @@ func TestStatusSandbox(t *testing.T) { } func TestEnterContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1162,7 +1162,7 @@ func TestEnterContainer(t *testing.T) { } func TestMonitor(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1182,7 +1182,7 @@ func TestMonitor(t *testing.T) { } func TestWaitProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1212,7 +1212,7 @@ func TestWaitProcess(t *testing.T) { } func TestSignalProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1242,7 +1242,7 @@ func TestSignalProcess(t *testing.T) { } func TestWinsizeProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1272,7 +1272,7 @@ func TestWinsizeProcess(t *testing.T) { } func TestContainerProcessIOStream(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1498,8 +1498,8 @@ func TestStartNetworkMonitor(t *testing.T) { s := &Sandbox{ id: testSandboxID, config: &SandboxConfig{ - NetworkConfig: NetworkConfig{ - NetmonConfig: NetmonConfig{ + NetworkConfig: types.NetworkConfig{ + NetmonConfig: types.NetmonConfig{ Path: trueBinPath, }, }, diff --git a/virtcontainers/types/network.go b/virtcontainers/types/network.go new file mode 100644 index 0000000000..a784b3d88c --- /dev/null +++ b/virtcontainers/types/network.go @@ -0,0 +1,175 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package types + +import ( + "fmt" + "os" + + "github.com/vishvananda/netlink" +) + +// NetInterworkingModel defines the network model connecting +// the network interface to the virtual machine. +type NetInterworkingModel int + +const ( + // NetXConnectDefaultModel Ask to use DefaultNetInterworkingModel + NetXConnectDefaultModel NetInterworkingModel = iota + + // NetXConnectBridgedModel uses a linux bridge to interconnect + // the container interface to the VM. This is the + // safe default that works for most cases except + // macvlan and ipvlan + NetXConnectBridgedModel + + // NetXConnectMacVtapModel can be used when the Container network + // interface can be bridged using macvtap + NetXConnectMacVtapModel + + // NetXConnectEnlightenedModel can be used when the Network plugins + // are enlightened to create VM native interfaces + // when requested by the runtime + // This will be used for vethtap, macvtap, ipvtap + NetXConnectEnlightenedModel + + // NetXConnectTCFilterModel redirects traffic from the network interface + // provided by the network plugin to a tap interface. + // This works for ipvlan and macvlan as well. + NetXConnectTCFilterModel + + // NetXConnectNoneModel can be used when the VM is in the host network namespace + NetXConnectNoneModel + + // NetXConnectInvalidModel is the last item to check valid values by IsValid() + NetXConnectInvalidModel +) + +// DefaultNetInterworkingModel is a package level default +// that determines how the VM should be connected to the +// the container network interface +var DefaultNetInterworkingModel = NetXConnectMacVtapModel + +//IsValid checks if a model is valid +func (n NetInterworkingModel) IsValid() bool { + return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel) +} + +const ( + // DefaultNetModelStr is the default networking model. + DefaultNetModelStr = "default" + + // BridgedNetModelStr is the bridged networking model. + BridgedNetModelStr = "bridged" + + // MacvtapNetModelStr is the MacVTap networking model. + MacvtapNetModelStr = "macvtap" + + // EnlightenedNetModelStr is the VM enlightened networking model. + EnlightenedNetModelStr = "enlightened" + + // TcFilterNetModelStr is the Traffic Control filtering networking model. + TcFilterNetModelStr = "tcfilter" + + // NoneNetModelStr is the no network networking model. + NoneNetModelStr = "none" +) + +//SetModel change the model string value +func (n *NetInterworkingModel) SetModel(modelName string) error { + switch modelName { + case DefaultNetModelStr: + *n = DefaultNetInterworkingModel + return nil + case BridgedNetModelStr: + *n = NetXConnectBridgedModel + return nil + case MacvtapNetModelStr: + *n = NetXConnectMacVtapModel + return nil + case EnlightenedNetModelStr: + *n = NetXConnectEnlightenedModel + return nil + case TcFilterNetModelStr: + *n = NetXConnectTCFilterModel + return nil + case NoneNetModelStr: + *n = NetXConnectNoneModel + return nil + } + return fmt.Errorf("Unknown type %s", modelName) +} + +// Introduces constants related to networking +const ( + DefaultRouteDest = "0.0.0.0/0" + DefaultRouteLabel = "default" + DefaultFilePerms = 0600 + DefaultQlen = 1500 +) + +// DNSInfo describes the DNS setup related to a network interface. +type DNSInfo struct { + Servers []string + Domain string + Searches []string + Options []string +} + +// NetlinkIface describes fully a network interface. +type NetlinkIface struct { + netlink.LinkAttrs + Type string +} + +// NetworkInfo gathers all information related to a network interface. +// It can be used to store the description of the underlying network. +type NetworkInfo struct { + Iface NetlinkIface + Addrs []netlink.Addr + Routes []netlink.Route + DNS DNSInfo +} + +// NetworkInterface defines a network interface. +type NetworkInterface struct { + Name string + HardAddr string + Addrs []netlink.Addr +} + +// TapInterface defines a tap interface +type TapInterface struct { + ID string + Name string + TAPIface NetworkInterface + VMFds []*os.File + VhostFds []*os.File +} + +// NetworkInterfacePair defines a pair between VM and virtual network interfaces. +type NetworkInterfacePair struct { + TapInterface + VirtIface NetworkInterface + NetInterworkingModel +} + +// NetworkConfig is the network configuration related to a network. +type NetworkConfig struct { + NetNSPath string + NetNsCreated bool + DisableNewNetNs bool + NetmonConfig NetmonConfig + InterworkingModel NetInterworkingModel +} + +// NetmonConfig is the structure providing specific configuration +// for the network monitor. +type NetmonConfig struct { + Path string + Debug bool + Enable bool +} diff --git a/virtcontainers/types/network_test.go b/virtcontainers/types/network_test.go new file mode 100644 index 0000000000..9af0de0da8 --- /dev/null +++ b/virtcontainers/types/network_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package types + +import ( + "testing" +) + +func TestNetInterworkingModelIsValid(t *testing.T) { + tests := []struct { + name string + n NetInterworkingModel + want bool + }{ + {"Invalid Model", NetXConnectInvalidModel, false}, + {"Default Model", NetXConnectDefaultModel, true}, + {"Bridged Model", NetXConnectBridgedModel, true}, + {"TC Filter Model", NetXConnectTCFilterModel, true}, + {"Macvtap Model", NetXConnectMacVtapModel, true}, + {"Enlightened Model", NetXConnectEnlightenedModel, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.n.IsValid(); got != tt.want { + t.Errorf("NetInterworkingModel.IsValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNetInterworkingModelSetModel(t *testing.T) { + var n NetInterworkingModel + tests := []struct { + name string + modelName string + wantErr bool + }{ + {"Invalid Model", "Invalid", true}, + {"default Model", DefaultNetModelStr, false}, + {"bridged Model", BridgedNetModelStr, false}, + {"macvtap Model", MacvtapNetModelStr, false}, + {"enlightened Model", EnlightenedNetModelStr, false}, + {"tcfilter Model", TcFilterNetModelStr, false}, + {"none Model", NoneNetModelStr, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := n.SetModel(tt.modelName); (err != nil) != tt.wantErr { + t.Errorf("NetInterworkingModel.SetModel() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From c394517f95e38d0af3fa8d0cab69d07d1f0cf3f6 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Thu, 31 Jan 2019 15:22:45 +0100 Subject: [PATCH 3/7] virtcontainers: Move kataVSOCK to types This is a generic type, shared across the agent and hypervisor implementations. Fixes: #1229 Signed-off-by: Samuel Ortiz --- virtcontainers/fc.go | 8 ++++---- virtcontainers/kata_agent.go | 21 +++++---------------- virtcontainers/kata_agent_test.go | 2 +- virtcontainers/qemu.go | 4 ++-- virtcontainers/qemu_arch_base.go | 10 +++++----- virtcontainers/qemu_test.go | 8 ++++---- virtcontainers/types/network.go | 20 ++++++++++++++++++++ 7 files changed, 41 insertions(+), 32 deletions(-) diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 291c01741a..0d6fd48cc6 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -486,14 +486,14 @@ func (fc *firecracker) resumeSandbox() error { return nil } -func (fc *firecracker) fcAddVsock(vs kataVSOCK) error { +func (fc *firecracker) fcAddVsock(vs types.VSOCK) error { span, _ := fc.trace("fcAddVsock") defer span.Finish() vsockParams := ops.NewPutGuestVsockByIDParams() vsockID := "root" vsock := &models.Vsock{ - GuestCid: int64(vs.contextID), + GuestCid: int64(vs.ContextID), ID: &vsockID, } vsockParams.SetID(vsockID) @@ -505,7 +505,7 @@ func (fc *firecracker) fcAddVsock(vs kataVSOCK) error { //Still racy. There is no way to send an fd to the firecracker //REST API. We could release this just before we start the instance //but even that will not eliminate the race - vs.vhostFd.Close() + vs.VHostFd.Close() return nil } @@ -619,7 +619,7 @@ func (fc *firecracker) addDevice(devInfo interface{}, devType hypervisor.Device) case config.BlockDrive: fc.Logger().WithField("device-type-blockdrive", devInfo).Info("Adding device") return fc.fcAddBlockDrive(v) - case kataVSOCK: + case types.VSOCK: fc.Logger().WithField("device-type-vsock", devInfo).Info("Adding device") return fc.fcAddVsock(v) default: diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 6e1a3505c9..44a8d72f5f 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -57,7 +57,6 @@ var ( mountGuest9pTag = "kataShared" kataGuestSandboxDir = "/run/kata-containers/sandbox/" type9pFs = "9p" - vsockSocketScheme = "vsock" // port numbers below 1024 are called privileged ports. Only a process with // CAP_NET_BIND_SERVICE capability may bind to these port numbers. vSockPort = 1024 @@ -80,16 +79,6 @@ type KataAgentConfig struct { UseVSock bool } -type kataVSOCK struct { - contextID uint64 - port uint32 - vhostFd *os.File -} - -func (s *kataVSOCK) String() string { - return fmt.Sprintf("%s://%d:%d", vsockSocketScheme, s.contextID, s.port) -} - // KataAgentState is the structure describing the data stored from this // agent implementation. type KataAgentState struct { @@ -145,7 +134,7 @@ func (k *kataAgent) generateVMSocket(id string, c KataAgentConfig) error { // We want to go through VSOCK. The VM VSOCK endpoint will be our gRPC. k.Logger().Debug("agent: Using vsock VM socket endpoint") // We dont know yet the context ID - set empty vsock configuration - k.vmSocket = kataVSOCK{} + k.vmSocket = types.VSOCK{} } else { k.Logger().Debug("agent: Using unix socket form VM socket endpoint") // We need to generate a host UNIX socket path for the emulated serial port. @@ -206,7 +195,7 @@ func (k *kataAgent) agentURL() (string, error) { switch s := k.vmSocket.(type) { case types.Socket: return s.HostPath, nil - case kataVSOCK: + case types.VSOCK: return s.String(), nil default: return "", fmt.Errorf("Invalid socket type") @@ -241,13 +230,13 @@ func (k *kataAgent) configure(h hypervisor.Hypervisor, id, sharePath string, bui if err != nil { return err } - case kataVSOCK: + case types.VSOCK: var err error - s.vhostFd, s.contextID, err = utils.FindContextID() + s.VHostFd, s.ContextID, err = utils.FindContextID() if err != nil { return err } - s.port = uint32(vSockPort) + s.Port = uint32(vSockPort) if err := h.AddDevice(s, hypervisor.VSockPCIDev); err != nil { return err } diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 68daa1e592..5b062e1fef 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -649,7 +649,7 @@ func TestAgentPathAPI(t *testing.T) { c.UseVSock = true err = k2.generateVMSocket(id, c) assert.Nil(err) - _, ok = k2.vmSocket.(kataVSOCK) + _, ok = k2.vmSocket.(types.VSOCK) assert.True(ok) } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index f3d230c445..ffce3b2e11 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -1241,8 +1241,8 @@ func (q *qemu) AddDevice(devInfo interface{}, devType hypervisor.Device) error { q.qemuConfig.Devices = q.arch.append9PVolume(q.qemuConfig.Devices, v) case types.Socket: q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v) - case kataVSOCK: - q.fds = append(q.fds, v.vhostFd) + case types.VSOCK: + q.fds = append(q.fds, v.VHostFd) q.qemuConfig.Devices = q.arch.appendVSockPCI(q.qemuConfig.Devices, v) case hypervisor.Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index d98c707ecc..992231fc04 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -79,7 +79,7 @@ type qemuArch interface { appendSocket(devices []govmmQemu.Device, socket types.Socket) []govmmQemu.Device // appendVSockPCI appends a vsock PCI to devices - appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOCK) []govmmQemu.Device + appendVSockPCI(devices []govmmQemu.Device, vsock types.VSOCK) []govmmQemu.Device // appendNetwork appends a endpoint device to devices appendNetwork(devices []govmmQemu.Device, endpoint hypervisor.Endpoint) []govmmQemu.Device @@ -417,12 +417,12 @@ func (q *qemuArchBase) appendSocket(devices []govmmQemu.Device, socket types.Soc return devices } -func (q *qemuArchBase) appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOCK) []govmmQemu.Device { +func (q *qemuArchBase) appendVSockPCI(devices []govmmQemu.Device, vsock types.VSOCK) []govmmQemu.Device { devices = append(devices, govmmQemu.VSOCKDevice{ - ID: fmt.Sprintf("vsock-%d", vsock.contextID), - ContextID: vsock.contextID, - VHostFD: vsock.vhostFd, + ID: fmt.Sprintf("vsock-%d", vsock.ContextID), + ContextID: vsock.ContextID, + VHostFD: vsock.VHostFd, DisableModern: q.nestedRun, }, ) diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 9f23c21ca7..735aab586e 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -291,10 +291,10 @@ func TestQemuAddDeviceKataVSOCK(t *testing.T) { }, } - vsock := kataVSOCK{ - contextID: contextID, - port: port, - vhostFd: vHostFD, + vsock := types.VSOCK{ + ContextID: contextID, + Port: port, + VHostFd: vHostFD, } testQemuAddDevice(t, vsock, hypervisor.VSockPCIDev, expectedOut) diff --git a/virtcontainers/types/network.go b/virtcontainers/types/network.go index a784b3d88c..4bcc4d1f33 100644 --- a/virtcontainers/types/network.go +++ b/virtcontainers/types/network.go @@ -173,3 +173,23 @@ type NetmonConfig struct { Debug bool Enable bool } + +// VSOCKSocketScheme is the URL scheme for a vsock URL (vsock://). +var VSOCKSocketScheme = "vsock" + +// VSOCK represents a Linux virtual Socket (AF_VSOCK). +type VSOCK struct { + // ContextID is the VSOCK context identifier. + ContextID uint64 + + // Port is the VSOCK port. + Port uint32 + + // VHostFd is the vhost file descriptor backing the VSOCK. + VHostFd *os.File +} + +// String formats a VSOCK into its URL. +func (s *VSOCK) String() string { + return fmt.Sprintf("%s://%d:%d", VSOCKSocketScheme, s.ContextID, s.Port) +} From 28ec210e444ea1b503adf7703fc827d7e0e60cc3 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Thu, 31 Jan 2019 18:35:48 +0100 Subject: [PATCH 4/7] virtcontainers: hypervisor: Remove the QEMU specific constant Max VCPUs is a QEMU specific configuration. In order to even decouple further the hypervisor interface from downward dependencies (e.g. qemu), we leave the hypervisor max VCPUs configuration handling down to the hypervisor implementation itself. Fixes: #1229 Signed-off-by: Samuel Ortiz --- virtcontainers/api_test.go | 2 -- virtcontainers/hypervisor/hypervisor.go | 12 +----------- virtcontainers/hypervisor/hypervisor_test.go | 1 - virtcontainers/qemu.go | 19 +++++++++++++------ virtcontainers/qemu_arch_base_test.go | 4 ++-- virtcontainers/qemu_test.go | 8 +++++--- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go index c9bb40adb3..92c72f2cff 100644 --- a/virtcontainers/api_test.go +++ b/virtcontainers/api_test.go @@ -860,7 +860,6 @@ func TestStatusSandboxSuccessfulStateReady(t *testing.T) { MemorySize: hypervisor.DefaultMemSzMiB, DefaultBridges: hypervisor.DefaultBridges, BlockDeviceDriver: hypervisor.DefaultBlockDriver, - DefaultMaxVCPUs: hypervisor.DefaultMaxQemuVCPUs, Msize9p: hypervisor.DefaultMsize9p, } @@ -918,7 +917,6 @@ func TestStatusSandboxSuccessfulStateRunning(t *testing.T) { MemorySize: hypervisor.DefaultMemSzMiB, DefaultBridges: hypervisor.DefaultBridges, BlockDeviceDriver: hypervisor.DefaultBlockDriver, - DefaultMaxVCPUs: hypervisor.DefaultMaxQemuVCPUs, Msize9p: hypervisor.DefaultMsize9p, } diff --git a/virtcontainers/hypervisor/hypervisor.go b/virtcontainers/hypervisor/hypervisor.go index 416a7d2964..25b64966c5 100644 --- a/virtcontainers/hypervisor/hypervisor.go +++ b/virtcontainers/hypervisor/hypervisor.go @@ -68,16 +68,10 @@ const ( // DefaultBlockDriver is the default virtio block based driver. DefaultBlockDriver = config.VirtioSCSI - // DefaultMaxQemuVCPUs is the maximum number of vCPUs a virtcontainers VM will run with by default. - DefaultMaxQemuVCPUs uint32 = 4 - // DefaultMsize9p is the default 9pfs msize value. DefaultMsize9p = 8192 ) -// In some architectures the maximum number of vCPUs depends on the number of physical cores. -//var defaultMaxQemuVCPUs = MaxQemuVCPUs() - // Device describes a virtualized device. type Device int @@ -175,7 +169,7 @@ type Config struct { // NumVCPUs specifies default number of vCPUs for the VM. NumVCPUs uint32 - //DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. + // DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. DefaultMaxVCPUs uint32 // DefaultMem specifies default memory size in MiB for the VM. @@ -359,10 +353,6 @@ func (conf *Config) Valid() error { conf.BlockDeviceDriver = DefaultBlockDriver } - if conf.DefaultMaxVCPUs == 0 { - conf.DefaultMaxVCPUs = DefaultMaxQemuVCPUs - } - if conf.Msize9p == 0 { conf.Msize9p = DefaultMsize9p } diff --git a/virtcontainers/hypervisor/hypervisor_test.go b/virtcontainers/hypervisor/hypervisor_test.go index 710418cbd4..ffdd9f6e61 100644 --- a/virtcontainers/hypervisor/hypervisor_test.go +++ b/virtcontainers/hypervisor/hypervisor_test.go @@ -207,7 +207,6 @@ func TestConfigDefaults(t *testing.T) { MemorySize: DefaultMemSzMiB, DefaultBridges: DefaultBridges, BlockDeviceDriver: DefaultBlockDriver, - DefaultMaxVCPUs: DefaultMaxQemuVCPUs, Msize9p: DefaultMsize9p, } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index ffce3b2e11..922adc581e 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -81,6 +81,7 @@ type qemu struct { ctx context.Context nvdimmCount int + maxVCPUs uint32 } const ( @@ -147,7 +148,7 @@ func (q *qemu) kernelParameters() string { params = append(params, defaultKernelParameters...) // set the maximum number of vCPUs - params = append(params, hypervisor.Param{"nr_cpus", fmt.Sprintf("%d", q.config.DefaultMaxVCPUs)}) + params = append(params, hypervisor.Param{"nr_cpus", fmt.Sprintf("%d", q.maxVCPUs)}) // add the params specified by the provided config. As the kernel // honours the last parameter value set and since the config-provided @@ -220,6 +221,12 @@ func (q *qemu) setup(id string, hypervisorConfig *hypervisor.Config, vcStore *st q.config = *hypervisorConfig q.arch = newQemuArch(q.config) + if q.config.DefaultMaxVCPUs == 0 { + q.maxVCPUs = MaxQemuVCPUs() + } else { + q.maxVCPUs = q.config.DefaultMaxVCPUs + } + initrdPath, err := q.config.InitrdAssetPath() if err != nil { return err @@ -277,7 +284,7 @@ func (q *qemu) setup(id string, hypervisorConfig *hypervisor.Config, vcStore *st } func (q *qemu) cpuTopology() govmmQemu.SMP { - return q.arch.cpuTopology(q.config.NumVCPUs, q.config.DefaultMaxVCPUs) + return q.arch.cpuTopology(q.config.NumVCPUs, q.maxVCPUs) } func (q *qemu) hostMemMB() (uint64, error) { @@ -1055,15 +1062,15 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { // Don't fail if the number of max vCPUs is exceeded, log a warning and hot add the vCPUs needed // to reach out max vCPUs - if currentVCPUs+amount > q.config.DefaultMaxVCPUs { + if currentVCPUs+amount > q.maxVCPUs { q.Logger().Warnf("Cannot hotplug %d CPUs, currently this SB has %d CPUs and the maximum amount of CPUs is %d", - amount, currentVCPUs, q.config.DefaultMaxVCPUs) - amount = q.config.DefaultMaxVCPUs - currentVCPUs + amount, currentVCPUs, q.maxVCPUs) + amount = q.maxVCPUs - currentVCPUs } if amount == 0 { // Don't fail if no more vCPUs can be added, since cgroups still can be updated - q.Logger().Warnf("maximum number of vCPUs '%d' has been reached", q.config.DefaultMaxVCPUs) + q.Logger().Warnf("maximum number of vCPUs '%d' has been reached", q.maxVCPUs) return 0, nil } diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 665c17c3a3..d0d374ecc4 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -169,10 +169,10 @@ func TestQemuArchBaseCPUTopology(t *testing.T) { Sockets: vcpus, Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: hypervisor.DefaultMaxQemuVCPUs, + MaxCPUs: MaxQemuVCPUs(), } - smp := qemuArchBase.cpuTopology(vcpus, hypervisor.DefaultMaxQemuVCPUs) + smp := qemuArchBase.cpuTopology(vcpus, MaxQemuVCPUs()) assert.Equal(expectedSMP, smp) } diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 735aab586e..4e89a564fc 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -31,7 +31,7 @@ func newQemuConfig() hypervisor.Config { MemorySize: hypervisor.DefaultMemSzMiB, DefaultBridges: hypervisor.DefaultBridges, BlockDeviceDriver: hypervisor.DefaultBlockDriver, - DefaultMaxVCPUs: hypervisor.DefaultMaxQemuVCPUs, + DefaultMaxVCPUs: MaxQemuVCPUs(), Msize9p: hypervisor.DefaultMsize9p, } } @@ -45,8 +45,9 @@ func testQemuKernelParameters(t *testing.T, kernelParams []hypervisor.Param, exp } q := &qemu{ - config: qemuConfig, - arch: &qemuArchBase{}, + config: qemuConfig, + arch: &qemuArchBase{}, + maxVCPUs: MaxQemuVCPUs(), } params := q.kernelParameters() @@ -161,6 +162,7 @@ func TestQemuCPUTopology(t *testing.T) { NumVCPUs: uint32(vcpus), DefaultMaxVCPUs: uint32(vcpus), }, + maxVCPUs: uint32(vcpus), } expectedOut := govmmQemu.SMP{ From a88c1a7700be9a119a9b81091a692da9d3451ce8 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Thu, 31 Jan 2019 18:49:00 +0100 Subject: [PATCH 5/7] virtcontainers: hypervisor: Add qemu implementation We can now move the qemu implementation and test under the hypervisor package. Fixes: #1229 Signed-off-by: Samuel Ortiz --- pkg/katautils/config.go | 2 +- pkg/katautils/config_test.go | 2 +- virtcontainers/hack/virtc/main.go | 4 +- virtcontainers/hypervisor/hypervisor.go | 7 +- virtcontainers/hypervisor/hypervisor_test.go | 68 ++++++++++- virtcontainers/{ => hypervisor}/qemu.go | 115 +++++++++--------- virtcontainers/{ => hypervisor}/qemu_amd64.go | 9 +- .../{ => hypervisor}/qemu_amd64_test.go | 5 +- .../{ => hypervisor}/qemu_arch_base.go | 34 +++--- .../{ => hypervisor}/qemu_arch_base_test.go | 25 ++-- virtcontainers/{ => hypervisor}/qemu_arm64.go | 6 +- .../{ => hypervisor}/qemu_arm64_test.go | 4 +- .../{ => hypervisor}/qemu_ppc64le.go | 6 +- .../{ => hypervisor}/qemu_ppc64le_test.go | 4 +- virtcontainers/{ => hypervisor}/qemu_s390x.go | 4 +- .../{ => hypervisor}/qemu_s390x_test.go | 4 +- virtcontainers/{ => hypervisor}/qemu_test.go | 80 ++++++------ 17 files changed, 214 insertions(+), 165 deletions(-) rename virtcontainers/{ => hypervisor}/qemu.go (92%) rename virtcontainers/{ => hypervisor}/qemu_amd64.go (94%) rename virtcontainers/{ => hypervisor}/qemu_amd64_test.go (97%) rename virtcontainers/{ => hypervisor}/qemu_arch_base.go (94%) rename virtcontainers/{ => hypervisor}/qemu_arch_base_test.go (94%) rename virtcontainers/{ => hypervisor}/qemu_arm64.go (96%) rename virtcontainers/{ => hypervisor}/qemu_arm64_test.go (97%) rename virtcontainers/{ => hypervisor}/qemu_ppc64le.go (96%) rename virtcontainers/{ => hypervisor}/qemu_ppc64le_test.go (95%) rename virtcontainers/{ => hypervisor}/qemu_s390x.go (97%) rename virtcontainers/{ => hypervisor}/qemu_s390x_test.go (96%) rename virtcontainers/{ => hypervisor}/qemu_test.go (79%) diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 0a37065253..075d9cdf67 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -251,7 +251,7 @@ func (h hypervisor) defaultVCPUs() uint32 { func (h hypervisor) defaultMaxVCPUs() uint32 { numcpus := uint32(goruntime.NumCPU()) - maxvcpus := vc.MaxQemuVCPUs() + maxvcpus := vcHypervisor.MaxQemuVCPUs() reqVCPUs := h.DefaultMaxVCPUs //don't exceed the number of physical CPUs. If a default is not provided, use the diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index 7a57fe2862..cb4af4778e 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -935,7 +935,7 @@ func TestHypervisorDefaults(t *testing.T) { h.DefaultMaxVCPUs = uint32(numCPUs) + 1 assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") - maxvcpus := vc.MaxQemuVCPUs() + maxvcpus := vcHypervisor.MaxQemuVCPUs() h.DefaultMaxVCPUs = uint32(maxvcpus) + 1 assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") diff --git a/virtcontainers/hack/virtc/main.go b/virtcontainers/hack/virtc/main.go index cb73b51c09..2d9fb145bd 100644 --- a/virtcontainers/hack/virtc/main.go +++ b/virtcontainers/hack/virtc/main.go @@ -49,7 +49,7 @@ var sandboxConfigFlags = []cli.Flag{ cli.StringFlag{ Name: "machine-type", - Value: vc.QemuPC, + Value: hypervisor.QemuPC, Usage: "hypervisor machine type", }, @@ -156,7 +156,7 @@ func buildSandboxConfig(context *cli.Context) (vc.SandboxConfig, error) { } kernelPath := "/usr/share/clear-containers/vmlinuz.container" - if machineType == vc.QemuPCLite { + if machineType == hypervisor.QemuPCLite { kernelPath = "/usr/share/clear-containers/vmlinux.container" } diff --git a/virtcontainers/hypervisor/hypervisor.go b/virtcontainers/hypervisor/hypervisor.go index 25b64966c5..70d53eb1d9 100644 --- a/virtcontainers/hypervisor/hypervisor.go +++ b/virtcontainers/hypervisor/hypervisor.go @@ -155,7 +155,12 @@ func (t *Type) String() string { // New returns an hypervisor from and hypervisor type. func New(t Type) (Hypervisor, error) { - return nil, fmt.Errorf("Unknown hypervisor type %s", t) + switch t { + case Qemu: + return &qemu{}, nil + default: + return nil, fmt.Errorf("Unknown hypervisor type %s", t) + } } // Param is a key/value representation for hypervisor and kernel parameters. diff --git a/virtcontainers/hypervisor/hypervisor_test.go b/virtcontainers/hypervisor/hypervisor_test.go index ffdd9f6e61..95e12591a5 100644 --- a/virtcontainers/hypervisor/hypervisor_test.go +++ b/virtcontainers/hypervisor/hypervisor_test.go @@ -6,12 +6,15 @@ package hypervisor import ( + "flag" "fmt" "io/ioutil" "os" "path/filepath" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/store" ) const testKernel = "kernel" @@ -92,11 +95,11 @@ func testNewHypervisorFromType(t *testing.T, hypervisorType Type, expected Hyper } } -// func TestNewHypervisorFromQemuType(t *testing.T) { -// hypervisorType := Qemu -// expectedHypervisor := &qemu{} -// testNewHypervisorFromType(t, hypervisorType, expectedHypervisor) -// } +func TestNewHypervisorFromQemuType(t *testing.T) { + hypervisorType := Qemu + expectedHypervisor := &qemu{} + testNewHypervisorFromType(t, hypervisorType, expectedHypervisor) +} // func TestNewHypervisorFromMockType(t *testing.T) { // hypervisorType := Mock @@ -527,3 +530,58 @@ func TestRunningOnVMMNotExistingCPUInfoPathFailure(t *testing.T) { t.Fatalf("Should fail because %q file path does not exist", filePath) } } + +// TestMain is the common main function used by ALL the test functions +// for this package. +func TestMain(m *testing.M) { + var err error + + flag.Parse() + + testDir, err = ioutil.TempDir("", "vc-qemu-tmp-") + if err != nil { + panic(err) + } + + fmt.Printf("INFO: Creating hypervisor test directory %s\n", testDir) + err = os.MkdirAll(testDir, store.DirMode) + if err != nil { + fmt.Println("Could not create test directories:", err) + os.Exit(1) + } + + testQemuKernelPath = filepath.Join(testDir, testKernel) + testQemuInitrdPath = filepath.Join(testDir, testInitrd) + testQemuImagePath = filepath.Join(testDir, testImage) + testQemuPath = filepath.Join(testDir, testHypervisor) + + fmt.Printf("INFO: Creating hypervisor test kernel %s\n", testQemuKernelPath) + _, err = os.Create(testQemuKernelPath) + if err != nil { + fmt.Println("Could not create test kernel:", err) + os.RemoveAll(testDir) + os.Exit(1) + } + + fmt.Printf("INFO: Creating hypervisor test image %s\n", testQemuImagePath) + _, err = os.Create(testQemuImagePath) + if err != nil { + fmt.Println("Could not create test image:", err) + os.RemoveAll(testDir) + os.Exit(1) + } + + fmt.Printf("INFO: Creating hypervisor test hypervisor %s\n", testQemuPath) + _, err = os.Create(testQemuPath) + if err != nil { + fmt.Println("Could not create test hypervisor:", err) + os.RemoveAll(testDir) + os.Exit(1) + } + + ret := m.Run() + + os.RemoveAll(testDir) + + os.Exit(ret) +} diff --git a/virtcontainers/qemu.go b/virtcontainers/hypervisor/qemu.go similarity index 92% rename from virtcontainers/qemu.go rename to virtcontainers/hypervisor/qemu.go index 922adc581e..f5cd2a76d1 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/hypervisor/qemu.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" @@ -23,7 +23,6 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/config" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" @@ -58,13 +57,13 @@ type QemuState struct { HotplugVFIOOnRootBus bool } -// qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. +// qemu is an Hypervisor interface implementation for the Linux qemu type qemu struct { id string store *store.VCStore - config hypervisor.Config + config Config qmpMonitorCh qmpChannel @@ -101,7 +100,7 @@ var qemuMajorVersion int var qemuMinorVersion int // agnostic list of kernel parameters -var defaultKernelParameters = []hypervisor.Param{ +var defaultKernelParameters = []Param{ {"panic", "1"}, } @@ -111,7 +110,7 @@ type qmpLogger struct { func newQMPLogger() qmpLogger { return qmpLogger{ - logger: virtLog.WithField("subsystem", "qmp"), + logger: logrus.WithField("source", "virtcontainers/hypervisor").WithField("subsystem", "qmp"), } } @@ -137,7 +136,7 @@ func (l qmpLogger) Errorf(format string, v ...interface{}) { // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemu) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu") + return logrus.WithField("source", "virtcontainers/hypervisor").WithField("subsystem", "qemu") } func (q *qemu) kernelParameters() string { @@ -148,14 +147,14 @@ func (q *qemu) kernelParameters() string { params = append(params, defaultKernelParameters...) // set the maximum number of vCPUs - params = append(params, hypervisor.Param{"nr_cpus", fmt.Sprintf("%d", q.maxVCPUs)}) + params = append(params, Param{"nr_cpus", fmt.Sprintf("%d", q.maxVCPUs)}) // add the params specified by the provided config. As the kernel // honours the last parameter value set and since the config-provided // params are added here, they will take priority over the defaults. params = append(params, q.config.KernelParams...) - paramsStr := hypervisor.SerializeParams(params, "=") + paramsStr := SerializeParams(params, "=") return strings.Join(paramsStr, " ") } @@ -168,7 +167,7 @@ func (q *qemu) Capabilities() types.Capabilities { return q.arch.capabilities() } -func (q *qemu) Config() hypervisor.Config { +func (q *qemu) Config() Config { return q.config } @@ -208,7 +207,7 @@ func (q *qemu) trace(name string) (opentracing.Span, context.Context) { } // setup sets the Qemu structure up. -func (q *qemu) setup(id string, hypervisorConfig *hypervisor.Config, vcStore *store.VCStore) error { +func (q *qemu) setup(id string, hypervisorConfig *Config, vcStore *store.VCStore) error { span, _ := q.trace("setup") defer span.Finish() @@ -261,7 +260,7 @@ func (q *qemu) setup(id string, hypervisorConfig *hypervisor.Config, vcStore *st } } - nested, err := hypervisor.RunningOnVMM(hypervisor.ProcCPUInfo) + nested, err := RunningOnVMM(ProcCPUInfo) if err != nil { return err } @@ -288,7 +287,7 @@ func (q *qemu) cpuTopology() govmmQemu.SMP { } func (q *qemu) hostMemMB() (uint64, error) { - hostMemKb, err := hypervisor.GetHostMemorySizeKb(hypervisor.ProcMemInfo) + hostMemKb, err := GetHostMemorySizeKb(ProcMemInfo) if err != nil { return 0, fmt.Errorf("Unable to read memory info: %s", err) } @@ -419,7 +418,7 @@ func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) g } // CreateSandbox is the Hypervisor sandbox creation implementation for govmmQemu. -func (q *qemu) CreateSandbox(ctx context.Context, id string, hypervisorConfig *hypervisor.Config, store *store.VCStore) error { +func (q *qemu) CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, store *store.VCStore) error { // Save the tracing context q.ctx = ctx @@ -544,7 +543,7 @@ func (q *qemu) StartSandbox(timeout int) error { if q.config.Debug { params := q.arch.kernelParameters(q.config.Debug) - strParams := hypervisor.SerializeParams(params, "=") + strParams := SerializeParams(params, "=") formatted := strings.Join(strParams, " ") // The name of this field matches a similar one generated by @@ -770,7 +769,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error { return err } -func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op hypervisor.Operation, devID string) error { +func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op Operation, devID string) error { var err error if q.config.BlockDeviceDriver == config.Nvdimm { @@ -833,7 +832,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op hypervisor.Ope return nil } -func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op hypervisor.Operation) error { +func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op Operation) error { err := q.qmpSetup() if err != nil { return err @@ -841,7 +840,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op hypervisor.Operat devID := "virtio-" + drive.ID - if op == hypervisor.AddDevice { + if op == AddDevice { err = q.hotplugAddBlockDevice(drive, op, devID) } else { if q.config.BlockDeviceDriver == config.VirtioBlock { @@ -862,7 +861,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op hypervisor.Operat return err } -func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op hypervisor.Operation) error { +func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op Operation) error { err := q.qmpSetup() if err != nil { return err @@ -870,7 +869,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op hypervisor.Operation devID := device.ID - if op == hypervisor.AddDevice { + if op == AddDevice { // In case HotplugVFIOOnRootBus is true, devices are hotplugged on the root bus // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. @@ -935,7 +934,7 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) } -func (q *qemu) hotplugNetDevice(endpoint hypervisor.Endpoint, op hypervisor.Operation) error { +func (q *qemu) hotplugNetDevice(endpoint Endpoint, op Operation) error { err := q.qmpSetup() if err != nil { return err @@ -944,17 +943,17 @@ func (q *qemu) hotplugNetDevice(endpoint hypervisor.Endpoint, op hypervisor.Oper devID := "virtio-" + tap.ID switch endpoint.Type() { - case hypervisor.VethEndpointType: - drive := endpoint.(*hypervisor.VethEndpoint) + case VethEndpointType: + drive := endpoint.(*VethEndpoint) tap = drive.NetPair.TapInterface - case hypervisor.TapEndpointType: - drive := endpoint.(*hypervisor.TapEndpoint) + case TapEndpointType: + drive := endpoint.(*TapEndpoint) tap = drive.TapInterface default: return fmt.Errorf("this endpoint is not supported") } - if op == hypervisor.AddDevice { + if op == AddDevice { if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { return err @@ -992,33 +991,33 @@ func (q *qemu) hotplugNetDevice(endpoint hypervisor.Endpoint, op hypervisor.Oper return nil } -func (q *qemu) hotplugDevice(devInfo interface{}, devType hypervisor.Device, op hypervisor.Operation) (interface{}, error) { +func (q *qemu) hotplugDevice(devInfo interface{}, devType Device, op Operation) (interface{}, error) { switch devType { - case hypervisor.BlockDev: + case BlockDev: drive := devInfo.(*config.BlockDrive) return nil, q.hotplugBlockDevice(drive, op) - case hypervisor.CPUDev: + case CPUDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) - case hypervisor.VfioDev: + case VfioDev: device := devInfo.(*config.VFIODev) return nil, q.hotplugVFIODevice(device, op) - case hypervisor.MemoryDev: - memdev := devInfo.(*hypervisor.MemoryDevice) + case MemoryDev: + memdev := devInfo.(*MemoryDevice) return q.hotplugMemory(memdev, op) - case hypervisor.NetDev: - device := devInfo.(hypervisor.Endpoint) + case NetDev: + device := devInfo.(Endpoint) return nil, q.hotplugNetDevice(device, op) default: return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) } } -func (q *qemu) HotplugAddDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { +func (q *qemu) HotplugAddDevice(devInfo interface{}, devType Device) (interface{}, error) { span, _ := q.trace("hotplugAddDevice") defer span.Finish() - data, err := q.hotplugDevice(devInfo, devType, hypervisor.AddDevice) + data, err := q.hotplugDevice(devInfo, devType, AddDevice) if err != nil { return data, err } @@ -1026,11 +1025,11 @@ func (q *qemu) HotplugAddDevice(devInfo interface{}, devType hypervisor.Device) return data, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) HotplugRemoveDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { +func (q *qemu) HotplugRemoveDevice(devInfo interface{}, devType Device) (interface{}, error) { span, _ := q.trace("hotplugRemoveDevice") defer span.Finish() - data, err := q.hotplugDevice(devInfo, devType, hypervisor.RemoveDevice) + data, err := q.hotplugDevice(devInfo, devType, RemoveDevice) if err != nil { return data, err } @@ -1038,7 +1037,7 @@ func (q *qemu) HotplugRemoveDevice(devInfo interface{}, devType hypervisor.Devic return data, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugCPUs(vcpus uint32, op hypervisor.Operation) (uint32, error) { +func (q *qemu) hotplugCPUs(vcpus uint32, op Operation) (uint32, error) { if vcpus == 0 { q.Logger().Warnf("cannot hotplug 0 vCPUs") return 0, nil @@ -1049,7 +1048,7 @@ func (q *qemu) hotplugCPUs(vcpus uint32, op hypervisor.Operation) (uint32, error return 0, err } - if op == hypervisor.AddDevice { + if op == AddDevice { return q.hotplugAddCPUs(vcpus) } @@ -1151,7 +1150,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) { return amount, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugMemory(memDev *hypervisor.MemoryDevice, op hypervisor.Operation) (int, error) { +func (q *qemu) hotplugMemory(memDev *MemoryDevice, op Operation) (int, error) { if !q.arch.supportGuestMemoryHotplug() { return 0, fmt.Errorf("guest memory hotplug not supported") @@ -1175,13 +1174,13 @@ func (q *qemu) hotplugMemory(memDev *hypervisor.MemoryDevice, op hypervisor.Oper } switch op { - case hypervisor.RemoveDevice: - memLog.WithField("hypervisor.Operation", "remove").Debugf("Requested to remove memory: %d MB", memDev.SizeMB) + case RemoveDevice: + memLog.WithField("Operation", "remove").Debugf("Requested to remove memory: %d MB", memDev.SizeMB) // Dont fail but warn that this is not supported. memLog.Warn("hot-remove VM memory not supported") return 0, nil - case hypervisor.AddDevice: - memLog.WithField("hypervisor.Operation", "add").Debugf("Requested to add memory: %d MB", memDev.SizeMB) + case AddDevice: + memLog.WithField("Operation", "add").Debugf("Requested to add memory: %d MB", memDev.SizeMB) maxMem, err := q.hostMemMB() if err != nil { return 0, err @@ -1204,7 +1203,7 @@ func (q *qemu) hotplugMemory(memDev *hypervisor.MemoryDevice, op hypervisor.Oper } -func (q *qemu) hotplugAddMemory(memDev *hypervisor.MemoryDevice) (int, error) { +func (q *qemu) hotplugAddMemory(memDev *MemoryDevice) (int, error) { memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx) if err != nil { return 0, fmt.Errorf("failed to query memory devices: %v", err) @@ -1238,7 +1237,7 @@ func (q *qemu) ResumeSandbox() error { } // addDevice will add extra devices to Qemu command line. -func (q *qemu) AddDevice(devInfo interface{}, devType hypervisor.Device) error { +func (q *qemu) AddDevice(devInfo interface{}, devType Device) error { var err error span, _ := q.trace("addDevice") defer span.Finish() @@ -1251,7 +1250,7 @@ func (q *qemu) AddDevice(devInfo interface{}, devType hypervisor.Device) error { case types.VSOCK: q.fds = append(q.fds, v.VHostFd) q.qemuConfig.Devices = q.arch.appendVSockPCI(q.qemuConfig.Devices, v) - case hypervisor.Endpoint: + case Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) case config.BlockDrive: q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v) @@ -1337,7 +1336,7 @@ func (q *qemu) Disconnect() { q.qmpShutdown() } -// resizeMemory get a request to update the VM memory to reqMemMB +// ResizeMemory get a request to update the VM memory to reqMemMB // Memory update is managed with two approaches // Add memory to VM: // When memory is required to be added we hotplug memory @@ -1348,7 +1347,7 @@ func (q *qemu) Disconnect() { // the memory to remove has to be at least the size of one slot. // To return memory back we are resizing the VM memory balloon. // A longer term solution is evaluate solutions like virtio-mem -func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { +func (q *qemu) ResizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory) err := q.qmpSetup() @@ -1364,10 +1363,10 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, return currentMemory, err } - addMemDevice := &hypervisor.MemoryDevice{ + addMemDevice := &MemoryDevice{ SizeMB: int(memHotplugMB), } - data, err := q.HotplugAddDevice(addMemDevice, hypervisor.MemoryDev) + data, err := q.HotplugAddDevice(addMemDevice, MemoryDev) if err != nil { return currentMemory, err } @@ -1384,10 +1383,10 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, return currentMemory, err } - addMemDevice := &hypervisor.MemoryDevice{ + addMemDevice := &MemoryDevice{ SizeMB: int(memHotunplugMB), } - data, err := q.HotplugRemoveDevice(addMemDevice, hypervisor.MemoryDev) + data, err := q.HotplugRemoveDevice(addMemDevice, MemoryDev) if err != nil { return currentMemory, err } @@ -1490,7 +1489,7 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff return memory } -func (q *qemu) GetThreadIDs() (*hypervisor.ThreadIDs, error) { +func (q *qemu) GetThreadIDs() (*ThreadIDs, error) { span, _ := q.trace("getThreadIDs") defer span.Finish() @@ -1505,7 +1504,7 @@ func (q *qemu) GetThreadIDs() (*hypervisor.ThreadIDs, error) { return nil, err } - var tid hypervisor.ThreadIDs + var tid ThreadIDs for _, i := range cpuInfos { if i.ThreadID > 0 { tid.VCPUs = append(tid.VCPUs, i.ThreadID) @@ -1531,7 +1530,7 @@ func (q *qemu) ResizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 case currentVCPUs < reqVCPUs: //hotplug addCPUs := reqVCPUs - currentVCPUs - data, err := q.HotplugAddDevice(addCPUs, hypervisor.CPUDev) + data, err := q.HotplugAddDevice(addCPUs, CPUDev) if err != nil { return currentVCPUs, newVCPUs, err } @@ -1543,7 +1542,7 @@ func (q *qemu) ResizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 case currentVCPUs > reqVCPUs: //hotunplug removeCPUs := currentVCPUs - reqVCPUs - data, err := q.HotplugRemoveDevice(removeCPUs, hypervisor.CPUDev) + data, err := q.HotplugRemoveDevice(removeCPUs, CPUDev) if err != nil { return currentVCPUs, newVCPUs, err } diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/hypervisor/qemu_amd64.go similarity index 94% rename from virtcontainers/qemu_amd64.go rename to virtcontainers/hypervisor/qemu_amd64.go index dec02b181c..6d49f0e656 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/hypervisor/qemu_amd64.go @@ -3,12 +3,11 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "os" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" govmmQemu "github.com/intel/govmm/qemu" @@ -33,13 +32,13 @@ var qemuPaths = map[string]string{ QemuQ35: defaultQemuPath, } -var kernelRootParams = []hypervisor.Param{ +var kernelRootParams = []Param{ {"root", "/dev/pmem0p1"}, {"rootflags", "dax,data=ordered,errors=remount-ro rw"}, {"rootfstype", "ext4"}, } -var kernelParams = []hypervisor.Param{ +var kernelParams = []Param{ {"tsc", "reliable"}, {"no_timer_check", ""}, {"rcupdate.rcu_expedited", "1"}, @@ -81,7 +80,7 @@ func MaxQemuVCPUs() uint32 { return uint32(240) } -func newQemuArch(config hypervisor.Config) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_amd64_test.go b/virtcontainers/hypervisor/qemu_amd64_test.go similarity index 97% rename from virtcontainers/qemu_amd64_test.go rename to virtcontainers/hypervisor/qemu_amd64_test.go index 3b62a7184c..390558be10 100644 --- a/virtcontainers/qemu_amd64_test.go +++ b/virtcontainers/hypervisor/qemu_amd64_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -12,13 +12,12 @@ import ( "testing" govmmQemu "github.com/intel/govmm/qemu" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) func newTestQemu(machineType string) qemuArch { - config := hypervisor.Config{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/hypervisor/qemu_arch_base.go similarity index 94% rename from virtcontainers/qemu_arch_base.go rename to virtcontainers/hypervisor/qemu_arch_base.go index 992231fc04..b9c12823ff 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/hypervisor/qemu_arch_base.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "encoding/hex" @@ -14,7 +14,6 @@ import ( govmmQemu "github.com/intel/govmm/qemu" "github.com/kata-containers/runtime/virtcontainers/device/config" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -43,7 +42,7 @@ type qemuArch interface { // kernelParameters returns the kernel parameters // if debug is true then kernel debug parameters are included - kernelParameters(debug bool) []hypervisor.Param + kernelParameters(debug bool) []Param //capabilities returns the capabilities supported by QEMU capabilities() types.Capabilities @@ -82,7 +81,7 @@ type qemuArch interface { appendVSockPCI(devices []govmmQemu.Device, vsock types.VSOCK) []govmmQemu.Device // appendNetwork appends a endpoint device to devices - appendNetwork(devices []govmmQemu.Device, endpoint hypervisor.Endpoint) []govmmQemu.Device + appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device // appendBlockDevice appends a block drive to devices appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device @@ -97,7 +96,7 @@ type qemuArch interface { appendRNGDevice(devices []govmmQemu.Device, rngDevice config.RNGDev) []govmmQemu.Device // handleImagePath handles the Hypervisor Config image path - handleImagePath(config hypervisor.Config) + handleImagePath(config Config) // supportGuestMemoryHotplug returns if the guest supports memory hotplug supportGuestMemoryHotplug() bool @@ -111,9 +110,9 @@ type qemuArchBase struct { networkIndex int qemuPaths map[string]string supportedQemuMachines []govmmQemu.Machine - kernelParamsNonDebug []hypervisor.Param - kernelParamsDebug []hypervisor.Param - kernelParams []hypervisor.Param + kernelParamsNonDebug []Param + kernelParamsDebug []Param + kernelParams []Param } const ( @@ -122,7 +121,6 @@ const ( defaultCPUModel = "host" defaultBridgeBus = "pcie.0" maxDevIDSize = 31 - defaultMsize9p = 8192 ) // This is the PCI start address assigned to the first bridge that @@ -153,27 +151,27 @@ const ( // kernelParamsNonDebug is a list of the default kernel // parameters that will be used in standard (non-debug) mode. -var kernelParamsNonDebug = []hypervisor.Param{ +var kernelParamsNonDebug = []Param{ {"quiet", ""}, } // kernelParamsSystemdNonDebug is a list of the default systemd related // kernel parameters that will be used in standard (non-debug) mode. -var kernelParamsSystemdNonDebug = []hypervisor.Param{ +var kernelParamsSystemdNonDebug = []Param{ {"systemd.show_status", "false"}, } // kernelParamsDebug is a list of the default kernel // parameters that will be used in debug mode (as much boot output as // possible). -var kernelParamsDebug = []hypervisor.Param{ +var kernelParamsDebug = []Param{ {"debug", ""}, } // kernelParamsSystemdDebug is a list of the default systemd related kernel // parameters that will be used in debug mode (as much boot output as // possible). -var kernelParamsSystemdDebug = []hypervisor.Param{ +var kernelParamsSystemdDebug = []Param{ {"systemd.show_status", "true"}, {"systemd.log_level", "debug"}, } @@ -217,7 +215,7 @@ func (q *qemuArchBase) qemuPath() (string, error) { return p, nil } -func (q *qemuArchBase) kernelParameters(debug bool) []hypervisor.Param { +func (q *qemuArchBase) kernelParameters(debug bool) []Param { params := q.kernelParams if debug { @@ -448,9 +446,9 @@ func networkModelToQemuType(model types.NetInterworkingModel) govmmQemu.NetDevic } } -func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint hypervisor.Endpoint) []govmmQemu.Device { +func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device { switch ep := endpoint.(type) { - case *hypervisor.VethEndpoint, *hypervisor.BridgedMacvlanEndpoint, *hypervisor.IPVlanEndpoint: + case *VethEndpoint, *BridgedMacvlanEndpoint, *IPVlanEndpoint: netPair := ep.NetworkPair() devices = append(devices, govmmQemu.NetDevice{ @@ -468,7 +466,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint hyperv }, ) q.networkIndex++ - case *hypervisor.MacvtapEndpoint: + case *MacvtapEndpoint: devices = append(devices, govmmQemu.NetDevice{ Type: govmmQemu.MACVTAP, @@ -561,7 +559,7 @@ func (q *qemuArchBase) appendRNGDevice(devices []govmmQemu.Device, rngDev config return devices } -func (q *qemuArchBase) handleImagePath(config hypervisor.Config) { +func (q *qemuArchBase) handleImagePath(config Config) { if config.ImagePath != "" { q.kernelParams = append(q.kernelParams, kernelRootParams...) q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...) diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/hypervisor/qemu_arch_base_test.go similarity index 94% rename from virtcontainers/qemu_arch_base_test.go rename to virtcontainers/hypervisor/qemu_arch_base_test.go index d0d374ecc4..9bf5bd0c35 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/hypervisor/qemu_arch_base_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -16,7 +16,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/kata-containers/runtime/virtcontainers/device/config" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -30,18 +29,18 @@ var qemuArchBaseQemuPaths = map[string]string{ qemuArchBaseMachineType: qemuArchBaseQemuPath, } -var qemuArchBaseKernelParamsNonDebug = []hypervisor.Param{ +var qemuArchBaseKernelParamsNonDebug = []Param{ {"quiet", ""}, {"systemd.show_status", "false"}, } -var qemuArchBaseKernelParamsDebug = []hypervisor.Param{ +var qemuArchBaseKernelParamsDebug = []Param{ {"debug", ""}, {"systemd.show_status", "true"}, {"systemd.log_level", "debug"}, } -var qemuArchBaseKernelParams = []hypervisor.Param{ +var qemuArchBaseKernelParams = []Param{ {"root", "/dev/vda"}, {"rootfstype", "ext4"}, } @@ -121,15 +120,15 @@ func TestQemuArchBaseKernelParameters(t *testing.T) { qemuArchBase := newQemuArchBase() // with debug params - expectedParams := []hypervisor.Param(qemuArchBaseKernelParams) - debugParams := []hypervisor.Param(qemuArchBaseKernelParamsDebug) + expectedParams := []Param(qemuArchBaseKernelParams) + debugParams := []Param(qemuArchBaseKernelParamsDebug) expectedParams = append(expectedParams, debugParams...) p := qemuArchBase.kernelParameters(true) assert.Equal(expectedParams, p) // with non-debug params - expectedParams = []hypervisor.Param(qemuArchBaseKernelParams) - nonDebugParams := []hypervisor.Param(qemuArchBaseKernelParamsNonDebug) + expectedParams = []Param(qemuArchBaseKernelParams) + nonDebugParams := []Param(qemuArchBaseKernelParamsNonDebug) expectedParams = append(expectedParams, nonDebugParams...) p = qemuArchBase.kernelParameters(false) assert.Equal(expectedParams, p) @@ -446,7 +445,7 @@ func TestQemuArchBaseAppendNetwork(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} - macvlanEp := &hypervisor.BridgedMacvlanEndpoint{ + macvlanEp := &BridgedMacvlanEndpoint{ NetPair: types.NetworkInterfacePair{ TapInterface: types.TapInterface{ ID: "uniqueTestID-4", @@ -461,11 +460,11 @@ func TestQemuArchBaseAppendNetwork(t *testing.T) { }, NetInterworkingModel: types.DefaultNetInterworkingModel, }, - EndpointType: hypervisor.BridgedMacvlanEndpointType, + EndpointType: BridgedMacvlanEndpointType, } - macvtapEp := &hypervisor.MacvtapEndpoint{ - EndpointType: hypervisor.MacvtapEndpointType, + macvtapEp := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, EndpointProperties: types.NetworkInfo{ Iface: types.NetlinkIface{ Type: "macvtap", diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/hypervisor/qemu_arm64.go similarity index 96% rename from virtcontainers/qemu_arm64.go rename to virtcontainers/hypervisor/qemu_arm64.go index bc72b0fa79..451e98a8f2 100644 --- a/virtcontainers/qemu_arm64.go +++ b/virtcontainers/hypervisor/qemu_arm64.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "io/ioutil" @@ -51,7 +51,7 @@ var supportedQemuMachines = []govmmQemu.Machine{ // Logger returns a logrus logger appropriate for logging qemu-aarch64 messages func qemuArmLogger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu-aarch64") + return logrus.WithField("subsystem", "hypervisor/qemu-aarch64") } // On ARM platform, we have different GIC interrupt controllers. Different @@ -127,7 +127,7 @@ func MaxQemuVCPUs() uint32 { return uint32(runtime.NumCPU()) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/hypervisor/qemu_arm64_test.go similarity index 97% rename from virtcontainers/qemu_arm64_test.go rename to virtcontainers/hypervisor/qemu_arm64_test.go index 30f3568010..c98bce596e 100644 --- a/virtcontainers/qemu_arm64_test.go +++ b/virtcontainers/hypervisor/qemu_arm64_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -18,7 +18,7 @@ import ( ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_ppc64le.go b/virtcontainers/hypervisor/qemu_ppc64le.go similarity index 96% rename from virtcontainers/qemu_ppc64le.go rename to virtcontainers/hypervisor/qemu_ppc64le.go index d986e87963..85296fe694 100644 --- a/virtcontainers/qemu_ppc64le.go +++ b/virtcontainers/hypervisor/qemu_ppc64le.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "encoding/hex" @@ -58,7 +58,7 @@ var supportedQemuMachines = []govmmQemu.Machine{ // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemuPPC64le) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu") + return logrus.WithField("subsystem", "hypervisor/qemu-ppc64le") } // MaxQemuVCPUs returns the maximum number of vCPUs supported @@ -66,7 +66,7 @@ func MaxQemuVCPUs() uint32 { return uint32(128) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_ppc64le_test.go b/virtcontainers/hypervisor/qemu_ppc64le_test.go similarity index 95% rename from virtcontainers/qemu_ppc64le_test.go rename to virtcontainers/hypervisor/qemu_ppc64le_test.go index ddb25fc006..a88983cb1e 100644 --- a/virtcontainers/qemu_ppc64le_test.go +++ b/virtcontainers/hypervisor/qemu_ppc64le_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -14,7 +14,7 @@ import ( ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_s390x.go b/virtcontainers/hypervisor/qemu_s390x.go similarity index 97% rename from virtcontainers/qemu_s390x.go rename to virtcontainers/hypervisor/qemu_s390x.go index 1ac865cbfa..56a103cbad 100644 --- a/virtcontainers/qemu_s390x.go +++ b/virtcontainers/hypervisor/qemu_s390x.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -55,7 +55,7 @@ func MaxQemuVCPUs() uint32 { return uint32(248) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_s390x_test.go b/virtcontainers/hypervisor/qemu_s390x_test.go similarity index 96% rename from virtcontainers/qemu_s390x_test.go rename to virtcontainers/hypervisor/qemu_s390x_test.go index 9bb740ca31..73af6f7bef 100644 --- a/virtcontainers/qemu_s390x_test.go +++ b/virtcontainers/hypervisor/qemu_s390x_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -15,7 +15,7 @@ import ( ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_test.go b/virtcontainers/hypervisor/qemu_test.go similarity index 79% rename from virtcontainers/qemu_test.go rename to virtcontainers/hypervisor/qemu_test.go index 4e89a564fc..c631620414 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/hypervisor/qemu_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" @@ -15,28 +15,34 @@ import ( "testing" govmmQemu "github.com/intel/govmm/qemu" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) -func newQemuConfig() hypervisor.Config { - return hypervisor.Config{ +const sandboxID = "123456789" + +var testQemuKernelPath = "" +var testQemuInitrdPath = "" +var testQemuImagePath = "" +var testQemuPath = "" + +func newQemuConfig() Config { + return Config{ KernelPath: testQemuKernelPath, ImagePath: testQemuImagePath, InitrdPath: testQemuInitrdPath, HypervisorPath: testQemuPath, - NumVCPUs: hypervisor.DefaultVCPUs, - MemorySize: hypervisor.DefaultMemSzMiB, - DefaultBridges: hypervisor.DefaultBridges, - BlockDeviceDriver: hypervisor.DefaultBlockDriver, + NumVCPUs: DefaultVCPUs, + MemorySize: DefaultMemSzMiB, + DefaultBridges: DefaultBridges, + BlockDeviceDriver: DefaultBlockDriver, DefaultMaxVCPUs: MaxQemuVCPUs(), - Msize9p: hypervisor.DefaultMsize9p, + Msize9p: DefaultMsize9p, } } -func testQemuKernelParameters(t *testing.T, kernelParams []hypervisor.Param, expected string, debug bool) { +func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected string, debug bool) { qemuConfig := newQemuConfig() qemuConfig.KernelParams = kernelParams @@ -58,7 +64,7 @@ func testQemuKernelParameters(t *testing.T, kernelParams []hypervisor.Param, exp func TestQemuKernelParameters(t *testing.T) { expectedOut := fmt.Sprintf("panic=1 nr_cpus=%d foo=foo bar=bar", MaxQemuVCPUs()) - params := []hypervisor.Param{ + params := []Param{ { Key: "foo", Value: "foo", @@ -77,19 +83,12 @@ func TestQemuCreateSandbox(t *testing.T) { qemuConfig := newQemuConfig() q := &qemu{} - sandbox := &Sandbox{ - ctx: context.Background(), - id: "testSandbox", - config: &SandboxConfig{ - HypervisorConfig: qemuConfig, - }, - } + sandboxID := "testSandbox" - vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) + vcStore, err := store.NewVCSandboxStore(context.Background(), sandboxID) if err != nil { t.Fatal(err) } - sandbox.store = vcStore // Create the hypervisor fake binary testQemuPath := filepath.Join(testDir, testHypervisor) @@ -98,13 +97,13 @@ func TestQemuCreateSandbox(t *testing.T) { t.Fatalf("Could not create hypervisor file %s: %v", testQemuPath, err) } - // Create parent dir path for hypervisor.json - parentDir := store.SandboxConfigurationRootPath(sandbox.id) + // Create parent dir path for json + parentDir := store.SandboxConfigurationRootPath(sandboxID) if err := os.MkdirAll(parentDir, store.DirMode); err != nil { t.Fatalf("Could not create parent directory %s: %v", parentDir, err) } - if err := q.CreateSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + if err := q.CreateSandbox(context.Background(), sandboxID, &qemuConfig, vcStore); err != nil { t.Fatal(err) } @@ -121,19 +120,12 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) { qemuConfig := newQemuConfig() q := &qemu{} - sandbox := &Sandbox{ - ctx: context.Background(), - id: "testSandbox", - config: &SandboxConfig{ - HypervisorConfig: qemuConfig, - }, - } + sandboxID := "testSandbox" - vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) + vcStore, err := store.NewVCSandboxStore(context.Background(), sandboxID) if err != nil { t.Fatal(err) } - sandbox.store = vcStore // Create the hypervisor fake binary testQemuPath := filepath.Join(testDir, testHypervisor) @@ -142,13 +134,13 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) { t.Fatalf("Could not create hypervisor file %s: %v", testQemuPath, err) } - // Ensure parent dir path for hypervisor.json does not exist. - parentDir := store.SandboxConfigurationRootPath(sandbox.id) + // Ensure parent dir path for json does not exist. + parentDir := store.SandboxConfigurationRootPath(sandboxID) if err := os.RemoveAll(parentDir); err != nil { t.Fatal(err) } - if err := q.CreateSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + if err := q.CreateSandbox(context.Background(), sandboxID, &qemuConfig, vcStore); err != nil { t.Fatalf("Qemu createSandbox() is not expected to fail because of missing parent directory for storage: %v", err) } } @@ -158,7 +150,7 @@ func TestQemuCPUTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, - config: hypervisor.Config{ + config: Config{ NumVCPUs: uint32(vcpus), DefaultMaxVCPUs: uint32(vcpus), }, @@ -186,13 +178,13 @@ func TestQemuMemoryTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, - config: hypervisor.Config{ + config: Config{ MemorySize: mem, MemSlots: slots, }, } - hostMemKb, err := hypervisor.GetHostMemorySizeKb(hypervisor.ProcMemInfo) + hostMemKb, err := GetHostMemorySizeKb(ProcMemInfo) if err != nil { t.Fatal(err) } @@ -214,7 +206,7 @@ func TestQemuMemoryTopology(t *testing.T) { } } -func testQemuAddDevice(t *testing.T, devInfo interface{}, devType hypervisor.Device, expected []govmmQemu.Device) { +func testQemuAddDevice(t *testing.T, devInfo interface{}, devType Device, expected []govmmQemu.Device) { q := &qemu{ ctx: context.Background(), arch: &qemuArchBase{}, @@ -250,7 +242,7 @@ func TestQemuAddDeviceFsDev(t *testing.T) { HostPath: hostPath, } - testQemuAddDevice(t, volume, hypervisor.FsDev, expectedOut) + testQemuAddDevice(t, volume, FsDev, expectedOut) } func TestQemuAddDeviceSerialPortDev(t *testing.T) { @@ -277,7 +269,7 @@ func TestQemuAddDeviceSerialPortDev(t *testing.T) { Name: name, } - testQemuAddDevice(t, socket, hypervisor.SerialPortDev, expectedOut) + testQemuAddDevice(t, socket, SerialPortDev, expectedOut) } func TestQemuAddDeviceKataVSOCK(t *testing.T) { @@ -299,7 +291,7 @@ func TestQemuAddDeviceKataVSOCK(t *testing.T) { VHostFd: vHostFD, } - testQemuAddDevice(t, vsock, hypervisor.VSockPCIDev, expectedOut) + testQemuAddDevice(t, vsock, VSockPCIDev, expectedOut) } func TestQemuGetSandboxConsole(t *testing.T) { @@ -395,9 +387,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) { } q.store = vcStore - _, err = q.HotplugAddDevice(&hypervisor.MemoryDevice{0, 128}, hypervisor.FsDev) + _, err = q.HotplugAddDevice(&MemoryDevice{0, 128}, FsDev) assert.Error(err) - _, err = q.HotplugRemoveDevice(&hypervisor.MemoryDevice{0, 128}, hypervisor.FsDev) + _, err = q.HotplugRemoveDevice(&MemoryDevice{0, 128}, FsDev) assert.Error(err) } From 5e24af4730b72f139dd7738328e0fd635ea7247d Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Fri, 1 Feb 2019 15:38:26 +0100 Subject: [PATCH 6/7] virtcontainers: hypervisor: Add mock implementation We can now move the mock implementation and test under the hypervisor package. Fixes: #1229 Signed-off-by: Samuel Ortiz --- virtcontainers/cgroups_test.go | 3 +- virtcontainers/container_test.go | 4 +- virtcontainers/factory/cache/cache_test.go | 2 +- virtcontainers/factory/direct/direct_test.go | 2 +- virtcontainers/factory/factory_test.go | 12 +- .../factory/template/template_test.go | 2 +- virtcontainers/hyperstart_agent_test.go | 3 +- virtcontainers/hypervisor/hypervisor.go | 2 + virtcontainers/hypervisor/mock.go | 107 ++++++++++++++++++ .../mock_test.go} | 44 ++++--- .../hypervisor/physical_endpoint_test.go | 4 +- .../hypervisor/vhostuser_endpoint_test.go | 6 +- virtcontainers/kata_agent_test.go | 6 +- virtcontainers/mock_hypervisor.go | 103 ----------------- virtcontainers/noop_agent_test.go | 3 +- virtcontainers/sandbox_test.go | 6 +- virtcontainers/vm_test.go | 2 +- 17 files changed, 157 insertions(+), 154 deletions(-) create mode 100644 virtcontainers/hypervisor/mock.go rename virtcontainers/{mock_hypervisor_test.go => hypervisor/mock_test.go} (58%) delete mode 100644 virtcontainers/mock_hypervisor.go diff --git a/virtcontainers/cgroups_test.go b/virtcontainers/cgroups_test.go index 1a89690fb4..646a2cb2cc 100644 --- a/virtcontainers/cgroups_test.go +++ b/virtcontainers/cgroups_test.go @@ -19,6 +19,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ) @@ -127,7 +128,7 @@ func TestSetupCgroups(t *testing.T) { s := &Sandbox{ id: "test-sandbox", - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), config: &SandboxConfig{ Containers: []ContainerConfig{ { diff --git a/virtcontainers/container_test.go b/virtcontainers/container_test.go index badaed9fce..1ae3fc5619 100644 --- a/virtcontainers/container_test.go +++ b/virtcontainers/container_test.go @@ -113,7 +113,7 @@ func TestContainerRemoveDrive(t *testing.T) { // test should pass without a hypervisor created for the container's sandbox. assert.Nil(t, err, "remove drive should succeed") - sandbox.hypervisor = &mockHypervisor{} + sandbox.hypervisor = hypervisor.NewMock() path := "/dev/hda" deviceInfo := config.DeviceInfo{ HostPath: path, @@ -218,7 +218,7 @@ func TestContainerAddDriveDir(t *testing.T) { ctx: context.Background(), id: testSandboxID, devManager: manager.NewDeviceManager(manager.VirtioSCSI, nil), - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), agent: &noopAgent{}, config: &SandboxConfig{ HypervisorConfig: hypervisor.Config{ diff --git a/virtcontainers/factory/cache/cache_test.go b/virtcontainers/factory/cache/cache_test.go index 18bff0e9d9..d19d4925f7 100644 --- a/virtcontainers/factory/cache/cache_test.go +++ b/virtcontainers/factory/cache/cache_test.go @@ -26,7 +26,7 @@ func TestTemplateFactory(t *testing.T) { ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, HypervisorConfig: hyperConfig, diff --git a/virtcontainers/factory/direct/direct_test.go b/virtcontainers/factory/direct/direct_test.go index 0396dea734..656b7beb71 100644 --- a/virtcontainers/factory/direct/direct_test.go +++ b/virtcontainers/factory/direct/direct_test.go @@ -25,7 +25,7 @@ func TestTemplateFactory(t *testing.T) { ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, HypervisorConfig: hyperConfig, diff --git a/virtcontainers/factory/factory_test.go b/virtcontainers/factory/factory_test.go index 9a43965dd1..3f97601374 100644 --- a/virtcontainers/factory/factory_test.go +++ b/virtcontainers/factory/factory_test.go @@ -30,7 +30,7 @@ func TestNewFactory(t *testing.T) { assert.Error(err) config.VMConfig = vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, } @@ -105,7 +105,7 @@ func TestVMConfigValid(t *testing.T) { testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") config := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, @@ -135,11 +135,11 @@ func TestCheckVMConfig(t *testing.T) { err := checkVMConfig(config1, config2) assert.Nil(err) - config1.HypervisorType = vc.MockHypervisor + config1.HypervisorType = hypervisor.Mock err = checkVMConfig(config1, config2) assert.Error(err) - config2.HypervisorType = vc.MockHypervisor + config2.HypervisorType = hypervisor.Mock err = checkVMConfig(config1, config2) assert.Nil(err) @@ -176,7 +176,7 @@ func TestFactoryGetVM(t *testing.T) { ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hyperConfig, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, @@ -321,7 +321,7 @@ func TestDeepCompare(t *testing.T) { var err error ctx := context.Background() config.VMConfig = vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, } diff --git a/virtcontainers/factory/template/template_test.go b/virtcontainers/factory/template/template_test.go index d042aa700b..d9971fdaa5 100644 --- a/virtcontainers/factory/template/template_test.go +++ b/virtcontainers/factory/template/template_test.go @@ -29,7 +29,7 @@ func TestTemplateFactory(t *testing.T) { ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hyperConfig, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, diff --git a/virtcontainers/hyperstart_agent_test.go b/virtcontainers/hyperstart_agent_test.go index cf914550f7..5fee5ca58b 100644 --- a/virtcontainers/hyperstart_agent_test.go +++ b/virtcontainers/hyperstart_agent_test.go @@ -13,6 +13,7 @@ import ( "reflect" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -187,7 +188,7 @@ func TestHyperConfigure(t *testing.T) { assert.Nil(err) h := &hyper{} - m := &mockHypervisor{} + m := hypervisor.NewMock() c := HyperConfig{} id := "foobar" diff --git a/virtcontainers/hypervisor/hypervisor.go b/virtcontainers/hypervisor/hypervisor.go index 70d53eb1d9..a2d1862c03 100644 --- a/virtcontainers/hypervisor/hypervisor.go +++ b/virtcontainers/hypervisor/hypervisor.go @@ -158,6 +158,8 @@ func New(t Type) (Hypervisor, error) { switch t { case Qemu: return &qemu{}, nil + case Mock: + return &mock{}, nil default: return nil, fmt.Errorf("Unknown hypervisor type %s", t) } diff --git a/virtcontainers/hypervisor/mock.go b/virtcontainers/hypervisor/mock.go new file mode 100644 index 0000000000..a680887be0 --- /dev/null +++ b/virtcontainers/hypervisor/mock.go @@ -0,0 +1,107 @@ +// Copyright (c) 2016 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + "context" + "os" + + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" +) + +type mock struct { +} + +// NewMock returns a new mock hypervisor instance. +func NewMock() Hypervisor { + return &mock{} +} + +func (m *mock) Capabilities() types.Capabilities { + return types.Capabilities{} +} + +func (m *mock) Config() Config { + return Config{} +} + +func (m *mock) CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, store *store.VCStore) error { + err := hypervisorConfig.Valid() + if err != nil { + return err + } + + return nil +} + +func (m *mock) StartSandbox(timeout int) error { + return nil +} + +func (m *mock) StopSandbox() error { + return nil +} + +func (m *mock) PauseSandbox() error { + return nil +} + +func (m *mock) ResumeSandbox() error { + return nil +} + +func (m *mock) SaveSandbox() error { + return nil +} + +func (m *mock) AddDevice(devInfo interface{}, devType Device) error { + return nil +} + +func (m *mock) HotplugAddDevice(devInfo interface{}, devType Device) (interface{}, error) { + switch devType { + case CPUDev: + return devInfo.(uint32), nil + case MemoryDev: + memdev := devInfo.(*MemoryDevice) + return memdev.SizeMB, nil + } + return nil, nil +} + +func (m *mock) HotplugRemoveDevice(devInfo interface{}, devType Device) (interface{}, error) { + switch devType { + case CPUDev: + return devInfo.(uint32), nil + case MemoryDev: + return 0, nil + } + return nil, nil +} + +func (m *mock) GetSandboxConsole(sandboxID string) (string, error) { + return "", nil +} + +func (m *mock) ResizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { + return 0, nil +} +func (m *mock) ResizeVCPUs(cpus uint32) (uint32, uint32, error) { + return 0, 0, nil +} + +func (m *mock) Disconnect() { +} + +func (m *mock) GetThreadIDs() (*ThreadIDs, error) { + vcpus := []int{os.Getpid()} + return &ThreadIDs{vcpus}, nil +} + +func (m *mock) Cleanup() error { + return nil +} diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/hypervisor/mock_test.go similarity index 58% rename from virtcontainers/mock_hypervisor_test.go rename to virtcontainers/hypervisor/mock_test.go index d651f018e4..2d156c187c 100644 --- a/virtcontainers/mock_hypervisor_test.go +++ b/virtcontainers/hypervisor/mock_test.go @@ -3,58 +3,52 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" "fmt" "testing" - - "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestMockHypervisorCreateSandbox(t *testing.T) { - var m *mockHypervisor - - sandbox := &Sandbox{ - config: &SandboxConfig{ - ID: "mock_sandbox", - HypervisorConfig: hypervisor.Config{ - KernelPath: "", - ImagePath: "", - HypervisorPath: "", - }, - }, + var m *mock + sandboxID := "mock_sandbox" + + hypervisorConfig := Config{ + KernelPath: "", + ImagePath: "", + HypervisorPath: "", } ctx := context.Background() // wrong config - if err := m.CreateSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err == nil { + if err := m.CreateSandbox(ctx, sandboxID, &hypervisorConfig, nil); err == nil { t.Fatal() } - sandbox.config.HypervisorConfig = hypervisor.Config{ + validHypervisorConfig := Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - if err := m.CreateSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err != nil { + if err := m.CreateSandbox(ctx, sandboxID, &validHypervisorConfig, nil); err != nil { t.Fatal(err) } } func TestMockHypervisorStartSandbox(t *testing.T) { - var m *mockHypervisor + var m *mock - if err := m.StartSandbox(vmStartTimeout); err != nil { + if err := m.StartSandbox(10); err != nil { t.Fatal(err) } } func TestMockHypervisorStopSandbox(t *testing.T) { - var m *mockHypervisor + var m *mock if err := m.StopSandbox(); err != nil { t.Fatal(err) @@ -62,15 +56,15 @@ func TestMockHypervisorStopSandbox(t *testing.T) { } func TestMockHypervisorAddDevice(t *testing.T) { - var m *mockHypervisor + var m *mock - if err := m.AddDevice(nil, hypervisor.ImgDev); err != nil { + if err := m.AddDevice(nil, ImgDev); err != nil { t.Fatal(err) } } func TestMockHypervisorGetSandboxConsole(t *testing.T) { - var m *mockHypervisor + var m *mock expected := "" @@ -85,7 +79,7 @@ func TestMockHypervisorGetSandboxConsole(t *testing.T) { } func TestMockHypervisorSaveSandbox(t *testing.T) { - var m *mockHypervisor + var m *mock if err := m.SaveSandbox(); err != nil { t.Fatal(err) @@ -93,7 +87,7 @@ func TestMockHypervisorSaveSandbox(t *testing.T) { } func TestMockHypervisorDisconnect(t *testing.T) { - var m *mockHypervisor + var m *mock m.Disconnect() } diff --git a/virtcontainers/hypervisor/physical_endpoint_test.go b/virtcontainers/hypervisor/physical_endpoint_test.go index d00cd0ad87..c51a194d6b 100644 --- a/virtcontainers/hypervisor/physical_endpoint_test.go +++ b/virtcontainers/hypervisor/physical_endpoint_test.go @@ -23,7 +23,7 @@ func TestPhysicalEndpoint_HotAttach(t *testing.T) { HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), } - h := &mockHypervisor{} + h := &mock{} err := v.HotAttach(h) assert.Error(err) @@ -36,7 +36,7 @@ func TestPhysicalEndpoint_HotDetach(t *testing.T) { HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), } - h := &mockHypervisor{} + h := &mock{} err := v.HotDetach(h, true, "") assert.Error(err) diff --git a/virtcontainers/hypervisor/vhostuser_endpoint_test.go b/virtcontainers/hypervisor/vhostuser_endpoint_test.go index e13eddb013..7270b5926f 100644 --- a/virtcontainers/hypervisor/vhostuser_endpoint_test.go +++ b/virtcontainers/hypervisor/vhostuser_endpoint_test.go @@ -96,7 +96,7 @@ func TestVhostUserEndpointAttach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + h := &mock{} err := v.Attach(h) if err != nil { @@ -112,7 +112,7 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + h := &mock{} err := v.HotAttach(h) assert.Error(err) @@ -126,7 +126,7 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + h := &mock{} err := v.HotDetach(h, true, "") assert.Error(err) diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 5b062e1fef..b0d69168ce 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -660,7 +660,7 @@ func TestAgentConfigure(t *testing.T) { assert.Nil(err) k := &kataAgent{} - h := &mockHypervisor{} + h := hypervisor.NewMock() c := KataAgentConfig{} id := "foobar" @@ -737,7 +737,7 @@ func TestAgentCreateContainer(t *testing.T) { ImagePath: "bar", }, }, - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), } vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) @@ -788,7 +788,7 @@ func TestAgentCreateContainer(t *testing.T) { dir, err := ioutil.TempDir("", "kata-agent-test") assert.Nil(err) - err = k.configure(&mockHypervisor{}, sandbox.id, dir, true, KataAgentConfig{}) + err = k.configure(hypervisor.NewMock(), sandbox.id, dir, true, KataAgentConfig{}) assert.Nil(err) // We'll fail on container metadata file creation, but it helps increasing coverage... diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go deleted file mode 100644 index fc204473c2..0000000000 --- a/virtcontainers/mock_hypervisor.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2016 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "context" - "os" - - "github.com/kata-containers/runtime/virtcontainers/hypervisor" - "github.com/kata-containers/runtime/virtcontainers/store" - "github.com/kata-containers/runtime/virtcontainers/types" -) - -type mockHypervisor struct { -} - -func (m *mockHypervisor) Capabilities() types.Capabilities { - return types.Capabilities{} -} - -func (m *mockHypervisor) Config() hypervisor.Config { - return hypervisor.Config{} -} - -func (m *mockHypervisor) CreateSandbox(ctx context.Context, id string, hypervisorConfig *hypervisor.Config, store *store.VCStore) error { - err := hypervisorConfig.Valid() - if err != nil { - return err - } - - return nil -} - -func (m *mockHypervisor) StartSandbox(timeout int) error { - return nil -} - -func (m *mockHypervisor) StopSandbox() error { - return nil -} - -func (m *mockHypervisor) PauseSandbox() error { - return nil -} - -func (m *mockHypervisor) ResumeSandbox() error { - return nil -} - -func (m *mockHypervisor) SaveSandbox() error { - return nil -} - -func (m *mockHypervisor) AddDevice(devInfo interface{}, devType hypervisor.Device) error { - return nil -} - -func (m *mockHypervisor) HotplugAddDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { - switch devType { - case hypervisor.CPUDev: - return devInfo.(uint32), nil - case hypervisor.MemoryDev: - memdev := devInfo.(*hypervisor.MemoryDevice) - return memdev.SizeMB, nil - } - return nil, nil -} - -func (m *mockHypervisor) HotplugRemoveDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { - switch devType { - case hypervisor.CPUDev: - return devInfo.(uint32), nil - case hypervisor.MemoryDev: - return 0, nil - } - return nil, nil -} - -func (m *mockHypervisor) GetSandboxConsole(sandboxID string) (string, error) { - return "", nil -} - -func (m *mockHypervisor) ResizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { - return 0, nil -} -func (m *mockHypervisor) ResizeVCPUs(cpus uint32) (uint32, uint32, error) { - return 0, 0, nil -} - -func (m *mockHypervisor) Disconnect() { -} - -func (m *mockHypervisor) GetThreadIDs() (*hypervisor.ThreadIDs, error) { - vcpus := []int{os.Getpid()} - return &hypervisor.ThreadIDs{vcpus}, nil -} - -func (m *mockHypervisor) Cleanup() error { - return nil -} diff --git a/virtcontainers/noop_agent_test.go b/virtcontainers/noop_agent_test.go index ad471f4647..6a0aae6cbf 100644 --- a/virtcontainers/noop_agent_test.go +++ b/virtcontainers/noop_agent_test.go @@ -10,6 +10,7 @@ import ( "context" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) @@ -166,7 +167,7 @@ func TestNoopAgentResumeContainer(t *testing.T) { func TestNoopAgentConfigure(t *testing.T) { n := &noopAgent{} - h := &mockHypervisor{} + h := hypervisor.NewMock() id := "foobar" sharePath := "foobarDir" err := n.configure(h, id, sharePath, true, nil) diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index b831aecab6..bd2f58a94d 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -928,7 +928,7 @@ func TestSandboxAttachDevicesVFIO(t *testing.T) { sandbox := Sandbox{ id: "100", containers: containers, - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), devManager: dm, ctx: context.Background(), } @@ -1302,7 +1302,7 @@ func TestContainerProcessIOStream(t *testing.T) { } func TestAttachBlockDevice(t *testing.T) { - h := &mockHypervisor{} + h := hypervisor.NewMock() hConfig := hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, @@ -1390,7 +1390,7 @@ func TestAttachBlockDevice(t *testing.T) { } func TestPreAddDevice(t *testing.T) { - h := &mockHypervisor{} + h := hypervisor.NewMock() hConfig := hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go index 53cf47897a..a92bb807bc 100644 --- a/virtcontainers/vm_test.go +++ b/virtcontainers/vm_test.go @@ -99,7 +99,7 @@ func TestSetupProxy(t *testing.T) { AgentType: NoopAgentType, } - hypervisor := &mockHypervisor{} + hypervisor := hypervisor.NewMock() agent := &noopAgent{} // wrong proxy type From c9fd15929be1da347e7fc7fbe3044a87d45d88c7 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Fri, 1 Feb 2019 15:44:41 +0100 Subject: [PATCH 7/7] virtcontainers: hypervisor: Add firecracker implementation We can now move the firecracker implementation and test under the hypervisor package. Fixes: #1229 Signed-off-by: Samuel Ortiz --- virtcontainers/{ => hypervisor}/fc.go | 75 ++++++++++++------------- virtcontainers/hypervisor/hypervisor.go | 2 + 2 files changed, 39 insertions(+), 38 deletions(-) rename virtcontainers/{ => hypervisor}/fc.go (87%) diff --git a/virtcontainers/fc.go b/virtcontainers/hypervisor/fc.go similarity index 87% rename from virtcontainers/fc.go rename to virtcontainers/hypervisor/fc.go index 0d6fd48cc6..0dfc163c6f 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/hypervisor/fc.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" @@ -22,7 +22,6 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/config" - "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -101,19 +100,19 @@ type firecracker struct { socketPath string store *store.VCStore - config hypervisor.Config + config Config pendingDevices []firecrackerDevice // Devices to be added when the FC API is ready ctx context.Context } type firecrackerDevice struct { dev interface{} - devType hypervisor.Device + devType Device } // Logger returns a logrus logger appropriate for logging firecracker messages func (fc *firecracker) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "firecracker") + return hLog.WithField("subsystem", "firecracker") } func (fc *firecracker) trace(name string) (opentracing.Span, context.Context) { @@ -130,9 +129,9 @@ func (fc *firecracker) trace(name string) (opentracing.Span, context.Context) { return span, ctx } -// For firecracker this call only sets the internal structure up. +// CreateSandbox only sets the internal structure up. This is specific to Firecracker. // The sandbox will be created and started through startSandbox(). -func (fc *firecracker) createSandbox(ctx context.Context, id string, hypervisorConfig *hypervisor.Config, vcStore *store.VCStore) error { +func (fc *firecracker) CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, vcStore *store.VCStore) error { fc.ctx = ctx span, _ := fc.trace("createSandbox") @@ -336,10 +335,10 @@ func (fc *firecracker) fcStartVM() error { return nil } -// startSandbox will start the hypervisor for the given sandbox. +// StartSandbox will start the hypervisor for the given sandbox. // In the context of firecracker, this will start the hypervisor, // for configuration, but not yet start the actual virtual machine -func (fc *firecracker) startSandbox(timeout int) error { +func (fc *firecracker) StartSandbox(timeout int) error { span, _ := fc.trace("startSandbox") defer span.Finish() @@ -353,7 +352,7 @@ func (fc *firecracker) startSandbox(timeout int) error { return err } - strParams := hypervisor.SerializeParams(fc.config.KernelParams, "=") + strParams := SerializeParams(fc.config.KernelParams, "=") formattedParams := strings.Join(strParams, " ") fc.fcSetBootSource(kernelPath, formattedParams) @@ -374,7 +373,7 @@ func (fc *firecracker) startSandbox(timeout int) error { fc.createDiskPool() for _, d := range fc.pendingDevices { - if err = fc.addDevice(d.dev, d.devType); err != nil { + if err = fc.AddDevice(d.dev, d.devType); err != nil { return err } } @@ -425,8 +424,8 @@ func (fc *firecracker) createDiskPool() error { return nil } -// stopSandbox will stop the Sandbox's VM. -func (fc *firecracker) stopSandbox() (err error) { +// StopSandbox will stop the Sandbox's VM. +func (fc *firecracker) StopSandbox() (err error) { span, _ := fc.trace("stopSandbox") defer span.Finish() @@ -474,15 +473,15 @@ func (fc *firecracker) stopSandbox() (err error) { return syscall.Kill(pid, syscall.SIGKILL) } -func (fc *firecracker) pauseSandbox() error { +func (fc *firecracker) PauseSandbox() error { return nil } -func (fc *firecracker) saveSandbox() error { +func (fc *firecracker) SaveSandbox() error { return nil } -func (fc *firecracker) resumeSandbox() error { +func (fc *firecracker) ResumeSandbox() error { return nil } @@ -509,7 +508,7 @@ func (fc *firecracker) fcAddVsock(vs types.VSOCK) error { return nil } -func (fc *firecracker) fcAddNetDevice(endpoint hypervisor.Endpoint) error { +func (fc *firecracker) fcAddNetDevice(endpoint Endpoint) error { span, _ := fc.trace("fcAddNetDevice") defer span.Finish() @@ -593,9 +592,9 @@ func (fc *firecracker) fcUpdateBlockDrive(drive config.BlockDrive) error { return nil } -// addDevice will add extra devices to firecracker. Limited to configure before the +// AddDevice will add extra devices to firecracker. Limited to configure before the // virtual machine starts. Devices include drivers and network interfaces only. -func (fc *firecracker) addDevice(devInfo interface{}, devType hypervisor.Device) error { +func (fc *firecracker) AddDevice(devInfo interface{}, devType Device) error { span, _ := fc.trace("addDevice") defer span.Finish() @@ -613,7 +612,7 @@ func (fc *firecracker) addDevice(devInfo interface{}, devType hypervisor.Device) } switch v := devInfo.(type) { - case hypervisor.Endpoint: + case Endpoint: fc.Logger().WithField("device-type-endpoint", devInfo).Info("Adding device") return fc.fcAddNetDevice(v) case config.BlockDrive: @@ -630,43 +629,43 @@ func (fc *firecracker) addDevice(devInfo interface{}, devType hypervisor.Device) return nil } -// hotplugAddDevice supported in Firecracker VMM -func (fc *firecracker) hotplugAddDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { +// HotplugAddDevice supported in Firecracker VMM +func (fc *firecracker) HotplugAddDevice(devInfo interface{}, devType Device) (interface{}, error) { span, _ := fc.trace("hotplugAddDevice") defer span.Finish() switch devType { - case hypervisor.BlockDev: + case BlockDev: //The drive placeholder has to exist prior to Update return nil, fc.fcUpdateBlockDrive(*devInfo.(*config.BlockDrive)) default: fc.Logger().WithFields(logrus.Fields{"devInfo": devInfo, - "hypervisor.Device": devType}).Warn("hotplugAddDevice: unsupported device") - return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, hypervisor.Device%v", + "Device": devType}).Warn("hotplugAddDevice: unsupported device") + return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, Device%v", devInfo, devType) } } -// hotplugRemoveDevice supported in Firecracker VMM, but no-op -func (fc *firecracker) hotplugRemoveDevice(devInfo interface{}, devType hypervisor.Device) (interface{}, error) { +// HotplugRemoveDevice supported in Firecracker VMM, but no-op +func (fc *firecracker) HotplugRemoveDevice(devInfo interface{}, devType Device) (interface{}, error) { return nil, nil } -// getSandboxConsole builds the path of the console where we can read +// GetSandboxConsole builds the path of the console where we can read // logs coming from the sandbox. // // we can get logs from firecracker itself; WIP on enabling. Who needs // logs when you're just hacking? -func (fc *firecracker) getSandboxConsole(id string) (string, error) { +func (fc *firecracker) GetSandboxConsole(id string) (string, error) { return "", nil } -func (fc *firecracker) disconnect() { +func (fc *firecracker) Disconnect() { fc.state.set(notReady) } -// Adds all capabilities supported by firecracker implementation of hypervisor interface -func (fc *firecracker) capabilities() types.Capabilities { +// Capabilities adds all capabilities supported by firecracker implementation of hypervisor interface +func (fc *firecracker) Capabilities() types.Capabilities { span, _ := fc.trace("capabilities") defer span.Finish() var caps types.Capabilities @@ -676,30 +675,30 @@ func (fc *firecracker) capabilities() types.Capabilities { return caps } -func (fc *firecracker) hypervisorConfig() hypervisor.Config { +func (fc *firecracker) Config() Config { return fc.config } -func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { +func (fc *firecracker) ResizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { return 0, nil } -func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { +func (fc *firecracker) ResizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { return 0, 0, nil } -// this is used to apply cgroup information on the host. not sure how necessary this +// GetThreadIDs is used to apply cgroup information on the host. not sure how necessary this // is in the first pass. // // Need to see if there's an easy way to ask firecracker for thread ids associated with // the vCPUs. Issue opened to ask for per vCPU thread IDs: // https://github.com/firecracker-microvm/firecracker/issues/718 -func (fc *firecracker) getThreadIDs() (*hypervisor.ThreadIDs, error) { +func (fc *firecracker) GetThreadIDs() (*ThreadIDs, error) { //TODO: this may not be exactly supported in Firecracker. Closest is cpu-template as part // of get /machine-config return nil, nil } -func (fc *firecracker) cleanup() error { +func (fc *firecracker) Cleanup() error { return nil } diff --git a/virtcontainers/hypervisor/hypervisor.go b/virtcontainers/hypervisor/hypervisor.go index a2d1862c03..d1c63daa55 100644 --- a/virtcontainers/hypervisor/hypervisor.go +++ b/virtcontainers/hypervisor/hypervisor.go @@ -160,6 +160,8 @@ func New(t Type) (Hypervisor, error) { return &qemu{}, nil case Mock: return &mock{}, nil + case Firecracker: + return &firecracker{}, nil default: return nil, fmt.Errorf("Unknown hypervisor type %s", t) }