diff --git a/cli/factory_test.go b/cli/factory_test.go index faf5028d73..cfe194e0d1 100644 --- a/cli/factory_test.go +++ b/cli/factory_test.go @@ -15,6 +15,7 @@ import ( "github.com/urfave/cli" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestFactoryCLIFunctionNoRuntimeConfig(t *testing.T) { @@ -64,7 +65,7 @@ func TestFactoryCLIFunctionInit(t *testing.T) { // With template runtimeConfig.FactoryConfig.Template = true - runtimeConfig.HypervisorType = vc.MockHypervisor + runtimeConfig.HypervisorType = hypervisor.Mock runtimeConfig.AgentType = vc.NoopAgentType ctx.App.Metadata["runtimeConfig"] = runtimeConfig fn, ok = initFactoryCommand.Action.(func(context *cli.Context) error) @@ -99,7 +100,7 @@ func TestFactoryCLIFunctionDestroy(t *testing.T) { // With template runtimeConfig.FactoryConfig.Template = true - runtimeConfig.HypervisorType = vc.MockHypervisor + runtimeConfig.HypervisorType = hypervisor.Mock runtimeConfig.AgentType = vc.NoopAgentType ctx.App.Metadata["runtimeConfig"] = runtimeConfig fn, ok = destroyFactoryCommand.Action.(func(context *cli.Context) error) @@ -135,7 +136,7 @@ func TestFactoryCLIFunctionStatus(t *testing.T) { // With template runtimeConfig.FactoryConfig.Template = true - runtimeConfig.HypervisorType = vc.MockHypervisor + runtimeConfig.HypervisorType = hypervisor.Mock runtimeConfig.AgentType = vc.NoopAgentType ctx.App.Metadata["runtimeConfig"] = runtimeConfig err = fn(ctx) diff --git a/cli/kata-check.go b/cli/kata-check.go index d47328867c..187c3253c8 100644 --- a/cli/kata-check.go +++ b/cli/kata-check.go @@ -24,7 +24,7 @@ import ( "syscall" "github.com/kata-containers/runtime/pkg/katautils" - vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -186,7 +186,7 @@ type kernelParamHandler func(onVMM bool, fields logrus.Fields, msg string) bool // the number of module errors (all of which are logged by this // function). Only fatal errors result in an error return. func checkKernelModules(modules map[string]kernelModule, handler kernelParamHandler) (count uint32, err error) { - onVMM, err := vc.RunningOnVMM(procCPUInfo) + onVMM, err := vcHypervisor.RunningOnVMM(procCPUInfo) if err != nil { return 0, err } diff --git a/cli/kata-check_amd64.go b/cli/kata-check_amd64.go index 2aa34f34e9..f8cfec2805 100644 --- a/cli/kata-check_amd64.go +++ b/cli/kata-check_amd64.go @@ -11,7 +11,7 @@ import ( "io/ioutil" "strings" - vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) const ( @@ -55,7 +55,7 @@ func setCPUtype() error { return fmt.Errorf("Unknow CPU Type") } else if cpuType == cpuTypeIntel { var kvmIntelParams map[string]string - onVMM, err := vc.RunningOnVMM(procCPUInfo) + onVMM, err := hypervisor.RunningOnVMM(procCPUInfo) if err != nil && !onVMM { kvmIntelParams = map[string]string{ // "VMX Unrestricted mode support". This is used diff --git a/cli/kata-env.go b/cli/kata-env.go index 8bf2d05760..0e5d436af8 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -16,6 +16,7 @@ import ( "github.com/BurntSushi/toml" "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" vcUtils "github.com/kata-containers/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -368,7 +369,7 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e kernel := KernelInfo{ Path: config.HypervisorConfig.KernelPath, - Parameters: strings.Join(vc.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), + Parameters: strings.Join(vcHypervisor.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), } initrd := InitrdInfo{ diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go index a3b6a5970a..47672ae824 100644 --- a/cli/kata-env_test.go +++ b/cli/kata-env_test.go @@ -20,6 +20,7 @@ import ( "github.com/BurntSushi/toml" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcUtils "github.com/kata-containers/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/urfave/cli" @@ -344,7 +345,7 @@ func getExpectedImage(config oci.RuntimeConfig) ImageInfo { func getExpectedKernel(config oci.RuntimeConfig) KernelInfo { return KernelInfo{ Path: config.HypervisorConfig.KernelPath, - Parameters: strings.Join(vc.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), + Parameters: strings.Join(hypervisor.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "), } } diff --git a/cli/list.go b/cli/list.go index 26dc7391b3..5f6a4a26bb 100644 --- a/cli/list.go +++ b/cli/list.go @@ -20,7 +20,7 @@ import ( "github.com/urfave/cli" "github.com/kata-containers/runtime/pkg/katautils" - vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" oci "github.com/kata-containers/runtime/virtcontainers/pkg/oci" ) @@ -360,7 +360,7 @@ func getContainers(ctx context.Context, context *cli.Context) ([]fullContainerSt // getHypervisorDetails returns details of the latest version of the // hypervisor and the associated assets. -func getHypervisorDetails(hypervisorConfig *vc.HypervisorConfig) hypervisorDetails { +func getHypervisorDetails(hypervisorConfig *vcHypervisor.Config) hypervisorDetails { hypervisorPath, err := hypervisorConfig.HypervisorAssetPath() if err != nil { hypervisorPath = hypervisorConfig.HypervisorPath diff --git a/cli/main_test.go b/cli/main_test.go index f9a65fce81..56d6a941fc 100644 --- a/cli/main_test.go +++ b/cli/main_test.go @@ -24,6 +24,7 @@ import ( "github.com/dlespiau/covertool/pkg/cover" "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" "github.com/kata-containers/runtime/virtcontainers/types" @@ -206,7 +207,7 @@ func grep(pattern, file string) error { // // Note: no parameter validation in case caller wishes to create an invalid // object. -func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) { +func newTestHypervisorConfig(dir string, create bool) (hypervisor.Config, error) { kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") hypervisorPath := path.Join(dir, "hypervisor") @@ -215,12 +216,12 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro for _, file := range []string{kernelPath, imagePath, hypervisorPath} { err := createEmptyFile(file) if err != nil { - return vc.HypervisorConfig{}, err + return hypervisor.Config{}, err } } } - return vc.HypervisorConfig{ + return hypervisor.Config{ KernelPath: kernelPath, ImagePath: imagePath, HypervisorPath: hypervisorPath, @@ -240,7 +241,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf } return oci.RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.KataContainersAgent, ProxyType: vc.CCProxyType, diff --git a/containerd-shim-v2/utils_test.go b/containerd-shim-v2/utils_test.go index 4ae413ef93..cf0f75a8c5 100644 --- a/containerd-shim-v2/utils_test.go +++ b/containerd-shim-v2/utils_test.go @@ -19,6 +19,7 @@ import ( "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" ) @@ -160,7 +161,7 @@ func createEmptyFile(path string) (err error) { // // Note: no parameter validation in case caller wishes to create an invalid // object. -func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) { +func newTestHypervisorConfig(dir string, create bool) (hypervisor.Config, error) { kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") hypervisorPath := path.Join(dir, "hypervisor") @@ -169,12 +170,12 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro for _, file := range []string{kernelPath, imagePath, hypervisorPath} { err := createEmptyFile(file) if err != nil { - return vc.HypervisorConfig{}, err + return hypervisor.Config{}, err } } } - return vc.HypervisorConfig{ + return hypervisor.Config{ KernelPath: kernelPath, ImagePath: imagePath, HypervisorPath: hypervisorPath, @@ -194,7 +195,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf } return oci.RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.KataContainersAgent, ProxyType: vc.KataBuiltInProxyType, diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index b286ef1fe2..075d9cdf67 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -16,13 +16,15 @@ import ( "github.com/BurntSushi/toml" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/config" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" ) const ( - defaultHypervisor = vc.QemuHypervisor + defaultHypervisor = vcHypervisor.Qemu defaultAgent = vc.KataContainersAgent ) @@ -249,7 +251,7 @@ func (h hypervisor) defaultVCPUs() uint32 { func (h hypervisor) defaultMaxVCPUs() uint32 { numcpus := uint32(goruntime.NumCPU()) - maxvcpus := vc.MaxQemuVCPUs() + maxvcpus := vcHypervisor.MaxQemuVCPUs() reqVCPUs := h.DefaultMaxVCPUs //don't exceed the number of physical CPUs. If a default is not provided, use the @@ -401,45 +403,45 @@ func (n netmon) debug() bool { return n.Debug } -func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { +func newFirecrackerHypervisorConfig(h hypervisor) (vcHypervisor.Config, error) { hypervisor, err := h.path() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } kernel, err := h.kernel() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } initrd, image, err := h.getInitrdAndImage() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } firmware, err := h.firmware() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } kernelParams := h.kernelParams() blockDriver, err := h.blockDeviceDriver() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } if !utils.SupportsVsocks() { - return vc.HypervisorConfig{}, errors.New("No vsock support, firecracker cannot be used") + return vcHypervisor.Config{}, errors.New("No vsock support, firecracker cannot be used") } - return vc.HypervisorConfig{ + return vcHypervisor.Config{ HypervisorPath: hypervisor, KernelPath: kernel, InitrdPath: initrd, ImagePath: image, FirmwarePath: firmware, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + KernelParams: vcHypervisor.DeserializeParams(strings.Fields(kernelParams)), NumVCPUs: h.defaultVCPUs(), DefaultMaxVCPUs: h.defaultMaxVCPUs(), MemorySize: h.defaultMemSz(), @@ -458,35 +460,35 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } -func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { +func newQemuHypervisorConfig(h hypervisor) (vcHypervisor.Config, error) { hypervisor, err := h.path() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } kernel, err := h.kernel() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } initrd, image, err := h.getInitrdAndImage() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } if image != "" && initrd != "" { - return vc.HypervisorConfig{}, + return vcHypervisor.Config{}, errors.New("having both an image and an initrd defined in the configuration file is not supported") } if image == "" && initrd == "" { - return vc.HypervisorConfig{}, + return vcHypervisor.Config{}, errors.New("either image or initrd must be defined in the configuration file") } firmware, err := h.firmware() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } machineAccelerators := h.machineAccelerators() @@ -495,7 +497,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { blockDriver, err := h.blockDeviceDriver() if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } useVSock := false @@ -508,14 +510,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { } } - return vc.HypervisorConfig{ + return vcHypervisor.Config{ HypervisorPath: hypervisor, KernelPath: kernel, InitrdPath: initrd, ImagePath: image, FirmwarePath: firmware, MachineAccelerators: machineAccelerators, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + KernelParams: vcHypervisor.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, NumVCPUs: h.defaultVCPUs(), DefaultMaxVCPUs: h.defaultMaxVCPUs(), @@ -563,14 +565,14 @@ func newShimConfig(s shim) (vc.ShimConfig, error) { func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig) error { for k, hypervisor := range tomlConf.Hypervisor { var err error - var hConfig vc.HypervisorConfig + var hConfig vcHypervisor.Config switch k { case firecrackerHypervisorTableType: - config.HypervisorType = vc.FirecrackerHypervisor + config.HypervisorType = vcHypervisor.Firecracker hConfig, err = newFirecrackerHypervisorConfig(hypervisor) case qemuHypervisorTableType: - config.HypervisorType = vc.QemuHypervisor + config.HypervisorType = vcHypervisor.Qemu hConfig, err = newQemuHypervisorConfig(hypervisor) } @@ -645,7 +647,7 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error { defaultKernelParams := GetKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig)) if runtimeConfig.HypervisorConfig.Debug { - strParams := vc.SerializeParams(defaultKernelParams, "=") + strParams := vcHypervisor.SerializeParams(defaultKernelParams, "=") formatted := strings.Join(strParams, " ") kataUtilsLogger.WithField("default-kernel-parameters", formatted).Debug() @@ -655,7 +657,7 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error { userKernelParams := runtimeConfig.HypervisorConfig.KernelParams // reset - runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{} + runtimeConfig.HypervisorConfig.KernelParams = []vcHypervisor.Param{} // first, add default values for _, p := range defaultKernelParams { @@ -714,7 +716,7 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run func initConfig() (config oci.RuntimeConfig, err error) { var defaultAgentConfig interface{} - defaultHypervisorConfig := vc.HypervisorConfig{ + defaultHypervisorConfig := vcHypervisor.Config{ HypervisorPath: defaultHypervisorPath, KernelPath: defaultKernelPath, ImagePath: defaultImagePath, @@ -893,7 +895,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error { if config.NetmonConfig.Enable { return fmt.Errorf("config disable_new_netns conflicts with enable_netmon") } - if config.InterNetworkModel != vc.NetXConnectNoneModel { + if config.InterNetworkModel != types.NetXConnectNoneModel { return fmt.Errorf("config disable_new_netns only works with 'none' internetworking_model") } } else if config.ShimConfig.(vc.ShimConfig).Trace { @@ -923,7 +925,7 @@ func checkFactoryConfig(config oci.RuntimeConfig) error { // checkHypervisorConfig performs basic "sanity checks" on the hypervisor // config. -func checkHypervisorConfig(config vc.HypervisorConfig) error { +func checkHypervisorConfig(config vcHypervisor.Config) error { type image struct { path string initrd bool diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index 12bac15979..cb4af4778e 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -21,7 +21,9 @@ import ( "testing" vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/stretchr/testify/assert" ) @@ -84,7 +86,6 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath } func createConfig(configPath string, fileData string) error { - err := ioutil.WriteFile(configPath, []byte(fileData), testFileMode) if err != nil { fmt.Fprintf(os.Stderr, "Unable to create config file %s %v\n", configPath, err) @@ -147,11 +148,11 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf } } - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := vcHypervisor.Config{ HypervisorPath: hypervisorPath, KernelPath: kernelPath, ImagePath: imagePath, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + KernelParams: vcHypervisor.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, NumVCPUs: defaultVCPUCount, DefaultMaxVCPUs: uint32(goruntime.NumCPU()), @@ -593,7 +594,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { t.Fatal(err) } - expectedHypervisorConfig := vc.HypervisorConfig{ + expectedHypervisorConfig := vcHypervisor.Config{ HypervisorPath: defaultHypervisorPath, KernelPath: defaultKernelPath, ImagePath: defaultImagePath, @@ -934,7 +935,7 @@ func TestHypervisorDefaults(t *testing.T) { h.DefaultMaxVCPUs = uint32(numCPUs) + 1 assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") - maxvcpus := vc.MaxQemuVCPUs() + maxvcpus := vcHypervisor.MaxQemuVCPUs() h.DefaultMaxVCPUs = uint32(maxvcpus) + 1 assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") @@ -1401,8 +1402,8 @@ func TestUpdateRuntimeConfigurationInvalidKernelParams(t *testing.T) { GetKernelParamsFunc = savedFunc }() - GetKernelParamsFunc = func(needSystemd bool) []vc.Param { - return []vc.Param{ + GetKernelParamsFunc = func(needSystemd bool) []vcHypervisor.Param { + return []vcHypervisor.Param{ { Key: "", Value: "", @@ -1493,7 +1494,7 @@ func TestCheckHypervisorConfig(t *testing.T) { // capture output to buffer kataUtilsLogger.Logger.Out = logBuf - config := vc.HypervisorConfig{ + config := vcHypervisor.Config{ ImagePath: d.imagePath, InitrdPath: d.initrdPath, MemorySize: d.memBytes, @@ -1532,7 +1533,7 @@ func TestCheckNetNsConfig(t *testing.T) { config = oci.RuntimeConfig{ DisableNewNetNs: true, - InterNetworkModel: vc.NetXConnectDefaultModel, + InterNetworkModel: types.NetXConnectDefaultModel, } err = checkNetNsConfig(config) assert.Error(err) @@ -1559,7 +1560,7 @@ func TestCheckFactoryConfig(t *testing.T) { for i, d := range data { config := oci.RuntimeConfig{ - HypervisorConfig: vc.HypervisorConfig{ + HypervisorConfig: vcHypervisor.Config{ ImagePath: d.imagePath, InitrdPath: d.initrdPath, }, @@ -1584,18 +1585,18 @@ func TestCheckNetNsConfigShimTrace(t *testing.T) { type testData struct { disableNetNs bool - networkModel vc.NetInterworkingModel + networkModel types.NetInterworkingModel shimTrace bool expectError bool } data := []testData{ - {false, vc.NetXConnectMacVtapModel, false, false}, - {false, vc.NetXConnectMacVtapModel, true, true}, - {true, vc.NetXConnectMacVtapModel, true, true}, - {true, vc.NetXConnectMacVtapModel, false, true}, - {true, vc.NetXConnectNoneModel, false, false}, - {true, vc.NetXConnectNoneModel, true, false}, + {false, types.NetXConnectMacVtapModel, false, false}, + {false, types.NetXConnectMacVtapModel, true, true}, + {true, types.NetXConnectMacVtapModel, true, true}, + {true, types.NetXConnectMacVtapModel, false, true}, + {true, types.NetXConnectNoneModel, false, false}, + {true, types.NetXConnectNoneModel, true, false}, } for i, d := range data { diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go index 0e07e8003c..ad29676ca2 100644 --- a/pkg/katautils/create.go +++ b/pkg/katautils/create.go @@ -12,13 +12,14 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" vf "github.com/kata-containers/runtime/virtcontainers/factory" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" ) // GetKernelParamsFunc use a variable to allow tests to modify its value var GetKernelParamsFunc = getKernelParams -var systemdKernelParam = []vc.Param{ +var systemdKernelParam = []vcHypervisor.Param{ { Key: "init", Value: "/usr/lib/systemd/systemd", @@ -37,8 +38,8 @@ var systemdKernelParam = []vc.Param{ }, } -func getKernelParams(needSystemd bool) []vc.Param { - p := []vc.Param{} +func getKernelParams(needSystemd bool) []vcHypervisor.Param { + p := []vcHypervisor.Param{} if needSystemd { p = append(p, systemdKernelParam...) @@ -47,7 +48,7 @@ func getKernelParams(needSystemd bool) []vc.Param { return p } -func needSystemd(config vc.HypervisorConfig) bool { +func needSystemd(config vcHypervisor.Config) bool { return config.ImagePath != "" } diff --git a/pkg/katautils/create_test.go b/pkg/katautils/create_test.go index 77701838e9..e72d286833 100644 --- a/pkg/katautils/create_test.go +++ b/pkg/katautils/create_test.go @@ -19,6 +19,7 @@ import ( "testing" vc "github.com/kata-containers/runtime/virtcontainers" + vcHypervisor "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" "github.com/opencontainers/runtime-spec/specs-go" @@ -117,7 +118,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf } return oci.RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: vcHypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.KataContainersAgent, ProxyType: vc.CCProxyType, @@ -132,7 +133,7 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf // // Note: no parameter validation in case caller wishes to create an invalid // object. -func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) { +func newTestHypervisorConfig(dir string, create bool) (vcHypervisor.Config, error) { kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") hypervisorPath := path.Join(dir, "hypervisor") @@ -141,12 +142,12 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro for _, file := range []string{kernelPath, imagePath, hypervisorPath} { err := createEmptyFile(file) if err != nil { - return vc.HypervisorConfig{}, err + return vcHypervisor.Config{}, err } } } - return vc.HypervisorConfig{ + return vcHypervisor.Config{ KernelPath: kernelPath, ImagePath: imagePath, HypervisorPath: hypervisorPath, @@ -155,7 +156,7 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro } // return the value of the *last* param with the specified key -func findLastParam(key string, params []vc.Param) (string, error) { +func findLastParam(key string, params []vcHypervisor.Param) (string, error) { if key == "" { return "", errors.New("ERROR: need non-nil key") } @@ -218,12 +219,12 @@ func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) { ipName := "ip" ipValue := "127.0.0.1" - params := []vc.Param{ + params := []vcHypervisor.Param{ {Key: initName, Value: initValue}, {Key: ipName, Value: ipValue}, } - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := vcHypervisor.Config{ KernelParams: params, } diff --git a/pkg/katautils/network.go b/pkg/katautils/network.go index 036b611dbb..089e7436be 100644 --- a/pkg/katautils/network.go +++ b/pkg/katautils/network.go @@ -15,7 +15,7 @@ import ( "strings" "github.com/containernetworking/plugins/pkg/ns" - vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/types" "golang.org/x/sys/unix" ) @@ -52,7 +52,7 @@ func EnterNetNS(netNSPath string, cb func() error) error { } // SetupNetworkNamespace create a network namespace -func SetupNetworkNamespace(config *vc.NetworkConfig) error { +func SetupNetworkNamespace(config *types.NetworkConfig) error { if config.DisableNewNetNs { kataUtilsLogger.Info("DisableNewNetNs is on, shim and hypervisor are running in the host netns") return nil diff --git a/pkg/katautils/network_test.go b/pkg/katautils/network_test.go index 857568ce50..4a1d30d77d 100644 --- a/pkg/katautils/network_test.go +++ b/pkg/katautils/network_test.go @@ -14,7 +14,7 @@ import ( "testing" "github.com/containernetworking/plugins/pkg/ns" - vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" "golang.org/x/sys/unix" ) @@ -108,14 +108,14 @@ func TestSetupNetworkNamespace(t *testing.T) { assert := assert.New(t) // Network namespace same as the host - config := &vc.NetworkConfig{ + config := &types.NetworkConfig{ NetNSPath: "/proc/self/ns/net", } err := SetupNetworkNamespace(config) assert.Error(err) // Non-existent netns path - config = &vc.NetworkConfig{ + config = &types.NetworkConfig{ NetNSPath: "/proc/123456789/ns/net", } err = SetupNetworkNamespace(config) @@ -124,7 +124,7 @@ func TestSetupNetworkNamespace(t *testing.T) { // Existent netns path n, err := ns.NewNS() assert.NoError(err) - config = &vc.NetworkConfig{ + config = &types.NetworkConfig{ NetNSPath: n.Path(), } err = SetupNetworkNamespace(config) @@ -132,7 +132,7 @@ func TestSetupNetworkNamespace(t *testing.T) { n.Close() // Empty netns path - config = &vc.NetworkConfig{} + config = &types.NetworkConfig{} err = SetupNetworkNamespace(config) assert.NoError(err) n, err = ns.GetNS(config.NetNSPath) @@ -144,7 +144,7 @@ func TestSetupNetworkNamespace(t *testing.T) { os.RemoveAll(config.NetNSPath) // Config with DisableNewNetNs - config = &vc.NetworkConfig{DisableNewNetNs: true} + config = &types.NetworkConfig{DisableNewNetNs: true} err = SetupNetworkNamespace(config) assert.NoError(err) } diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go index 9cfce0f349..8964af18af 100644 --- a/virtcontainers/agent.go +++ b/virtcontainers/agent.go @@ -11,6 +11,7 @@ import ( "time" "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/mitchellh/mapstructure" @@ -223,7 +224,7 @@ type agent interface { resumeContainer(sandbox *Sandbox, c Container) error // configure will update agent settings based on provided arguments - configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error + configure(h hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error // getVMPath will return the agent vm socket's directory path getVMPath(id string) string diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go index ec19e72acc..92c72f2cff 100644 --- a/virtcontainers/api_test.go +++ b/virtcontainers/api_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/mock" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/store" @@ -67,7 +68,7 @@ func newTestSandboxConfigNoop() SandboxConfig { } // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -75,7 +76,7 @@ func newTestSandboxConfigNoop() SandboxConfig { sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: NoopAgentType, @@ -98,7 +99,7 @@ func newTestSandboxConfigHyperstartAgent() SandboxConfig { } // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -111,7 +112,7 @@ func newTestSandboxConfigHyperstartAgent() SandboxConfig { sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: HyperstartAgent, @@ -134,7 +135,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { } // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -145,12 +146,12 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { SockTtyName: testHyperstartTtySocket, } - netConfig := NetworkConfig{} + netConfig := types.NetworkConfig{} sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: HyperstartAgent, @@ -167,7 +168,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { func newTestSandboxConfigKataAgent() SandboxConfig { // Sets the hypervisor configuration. - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -175,7 +176,7 @@ func newTestSandboxConfigKataAgent() SandboxConfig { sandboxConfig := SandboxConfig{ ID: testSandboxID, - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hypervisorConfig, AgentType: KataContainersAgent, @@ -851,16 +852,15 @@ func TestStatusSandboxSuccessfulStateReady(t *testing.T) { cleanUp() config := newTestSandboxConfigNoop() - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: hypervisor.DefaultVCPUs, + MemorySize: hypervisor.DefaultMemSzMiB, + DefaultBridges: hypervisor.DefaultBridges, + BlockDeviceDriver: hypervisor.DefaultBlockDriver, + Msize9p: hypervisor.DefaultMsize9p, } expectedStatus := SandboxStatus{ @@ -868,7 +868,7 @@ func TestStatusSandboxSuccessfulStateReady(t *testing.T) { State: types.State{ State: types.StateReady, }, - Hypervisor: MockHypervisor, + Hypervisor: hypervisor.Mock, HypervisorConfig: hypervisorConfig, Agent: NoopAgentType, Annotations: sandboxAnnotations, @@ -909,16 +909,15 @@ func TestStatusSandboxSuccessfulStateRunning(t *testing.T) { cleanUp() config := newTestSandboxConfigNoop() - hypervisorConfig := HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: hypervisor.DefaultVCPUs, + MemorySize: hypervisor.DefaultMemSzMiB, + DefaultBridges: hypervisor.DefaultBridges, + BlockDeviceDriver: hypervisor.DefaultBlockDriver, + Msize9p: hypervisor.DefaultMsize9p, } expectedStatus := SandboxStatus{ @@ -926,7 +925,7 @@ func TestStatusSandboxSuccessfulStateRunning(t *testing.T) { State: types.State{ State: types.StateRunning, }, - Hypervisor: MockHypervisor, + Hypervisor: hypervisor.Mock, HypervisorConfig: hypervisorConfig, Agent: NoopAgentType, Annotations: sandboxAnnotations, @@ -1993,14 +1992,14 @@ func TestProcessListContainer(t *testing.T) { * Benchmarks */ -func createNewSandboxConfig(hType HypervisorType, aType AgentType, aConfig interface{}) SandboxConfig { - hypervisorConfig := HypervisorConfig{ +func createNewSandboxConfig(hType hypervisor.Type, aType AgentType, aConfig interface{}) SandboxConfig { + hypervisorConfig := hypervisor.Config{ KernelPath: "/usr/share/kata-containers/vmlinux.container", ImagePath: "/usr/share/kata-containers/kata-containers.img", HypervisorPath: "/usr/bin/qemu-system-x86_64", } - netConfig := NetworkConfig{} + netConfig := types.NetworkConfig{} return SandboxConfig{ ID: testSandboxID, @@ -2159,7 +2158,7 @@ func createStartStopDeleteContainers(b *testing.B, sandboxConfig SandboxConfig, func BenchmarkCreateStartStopDeleteSandboxQemuHypervisorHyperstartAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, HyperstartAgent, HyperConfig{}) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, HyperstartAgent, HyperConfig{}) sockDir, err := testGenerateCCProxySockDir() if err != nil { @@ -2180,21 +2179,21 @@ func BenchmarkCreateStartStopDeleteSandboxQemuHypervisorHyperstartAgentNetworkNo func BenchmarkCreateStartStopDeleteSandboxQemuHypervisorNoopAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, NoopAgentType, nil) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, NoopAgentType, nil) createStartStopDeleteSandbox(b, sandboxConfig) } } -func BenchmarkCreateStartStopDeleteSandboxMockHypervisorNoopAgentNetworkNoop(b *testing.B) { +func BenchmarkCreateStartStopDeleteSandboxHypervisorMockNoopAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(MockHypervisor, NoopAgentType, nil) + sandboxConfig := createNewSandboxConfig(hypervisor.Mock, NoopAgentType, nil) createStartStopDeleteSandbox(b, sandboxConfig) } } func BenchmarkStartStop1ContainerQemuHypervisorHyperstartAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, HyperstartAgent, HyperConfig{}) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, HyperstartAgent, HyperConfig{}) contConfigs := createNewContainerConfigs(1) sockDir, err := testGenerateCCProxySockDir() @@ -2216,7 +2215,7 @@ func BenchmarkStartStop1ContainerQemuHypervisorHyperstartAgentNetworkNoop(b *tes func BenchmarkStartStop10ContainerQemuHypervisorHyperstartAgentNetworkNoop(b *testing.B) { for i := 0; i < b.N; i++ { - sandboxConfig := createNewSandboxConfig(QemuHypervisor, HyperstartAgent, HyperConfig{}) + sandboxConfig := createNewSandboxConfig(hypervisor.Qemu, HyperstartAgent, HyperConfig{}) contConfigs := createNewContainerConfigs(10) sockDir, err := testGenerateCCProxySockDir() @@ -2420,7 +2419,7 @@ func TestNetworkOperation(t *testing.T) { defer deleteNetNS(netNSPath) config := newTestSandboxConfigNoop() - config.NetworkConfig = NetworkConfig{ + config.NetworkConfig = types.NetworkConfig{ NetNSPath: netNSPath, } diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go index 20fa6b07bc..fd11a9a8f8 100644 --- a/virtcontainers/cgroups.go +++ b/virtcontainers/cgroups.go @@ -99,7 +99,7 @@ func (s *Sandbox) applyCPUCgroup(rc *specs.LinuxResources) error { // when new container joins, new CPU could be hotplugged, so we // have to query fresh vcpu info from hypervisor for every time. - tids, err := s.hypervisor.getThreadIDs() + tids, err := s.hypervisor.GetThreadIDs() if err != nil { return fmt.Errorf("failed to get thread ids from hypervisor: %v", err) } @@ -113,15 +113,15 @@ func (s *Sandbox) applyCPUCgroup(rc *specs.LinuxResources) error { // use Add() to add vcpu thread to s.cgroup, it will write thread id to // `cgroup.procs` which will move all threads in qemu process to this cgroup // immediately as default behaviour. - if len(tids.vcpus) > 0 { + if len(tids.VCPUs) > 0 { if err := s.cgroup.sandboxSub.Add(cgroups.Process{ - Pid: tids.vcpus[0], + Pid: tids.VCPUs[0], }); err != nil { return err } } - for _, i := range tids.vcpus { + for _, i := range tids.VCPUs { if i <= 0 { continue } diff --git a/virtcontainers/cgroups_test.go b/virtcontainers/cgroups_test.go index 1a89690fb4..646a2cb2cc 100644 --- a/virtcontainers/cgroups_test.go +++ b/virtcontainers/cgroups_test.go @@ -19,6 +19,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ) @@ -127,7 +128,7 @@ func TestSetupCgroups(t *testing.T) { s := &Sandbox{ id: "test-sandbox", - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), config: &SandboxConfig{ Containers: []ContainerConfig{ { diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 7952dfbd61..887ede7ad3 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -431,7 +431,7 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s // copy file to contaier's rootfs if filesystem sharing is not supported, otherwise // bind mount it in the shared directory. - caps := c.sandbox.hypervisor.capabilities() + caps := c.sandbox.hypervisor.Capabilities() if !caps.IsFsSharingSupported() { c.Logger().Debug("filesystem sharing is not supported, files will be copied") @@ -713,7 +713,7 @@ func (c *Container) rollbackFailingContainerCreation() { func (c *Container) checkBlockDeviceSupport() bool { if !c.sandbox.config.HypervisorConfig.DisableBlockDeviceUse { agentCaps := c.sandbox.agent.capabilities() - hypervisorCaps := c.sandbox.hypervisor.capabilities() + hypervisorCaps := c.sandbox.hypervisor.Capabilities() if agentCaps.IsBlockDeviceSupported() && hypervisorCaps.IsBlockDeviceHotplugSupported() { return true diff --git a/virtcontainers/container_test.go b/virtcontainers/container_test.go index e97d699215..1ae3fc5619 100644 --- a/virtcontainers/container_test.go +++ b/virtcontainers/container_test.go @@ -20,6 +20,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" @@ -112,7 +113,7 @@ func TestContainerRemoveDrive(t *testing.T) { // test should pass without a hypervisor created for the container's sandbox. assert.Nil(t, err, "remove drive should succeed") - sandbox.hypervisor = &mockHypervisor{} + sandbox.hypervisor = hypervisor.NewMock() path := "/dev/hda" deviceInfo := config.DeviceInfo{ HostPath: path, @@ -217,10 +218,10 @@ func TestContainerAddDriveDir(t *testing.T) { ctx: context.Background(), id: testSandboxID, devManager: manager.NewDeviceManager(manager.VirtioSCSI, nil), - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), agent: &noopAgent{}, config: &SandboxConfig{ - HypervisorConfig: HypervisorConfig{ + HypervisorConfig: hypervisor.Config{ DisableBlockDeviceUse: false, }, }, diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go deleted file mode 100644 index 978da2dc10..0000000000 --- a/virtcontainers/endpoint.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "fmt" -) - -// Endpoint represents a physical or virtual network interface. -type Endpoint interface { - Properties() NetworkInfo - Name() string - HardwareAddr() string - Type() EndpointType - PciAddr() string - NetworkPair() *NetworkInterfacePair - - SetProperties(NetworkInfo) - SetPciAddr(string) - Attach(hypervisor) error - Detach(netNsCreated bool, netNsPath string) error - HotAttach(h hypervisor) error - HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error -} - -// EndpointType identifies the type of the network endpoint. -type EndpointType string - -const ( - // PhysicalEndpointType is the physical network interface. - PhysicalEndpointType EndpointType = "physical" - - // VethEndpointType is the virtual network interface. - VethEndpointType EndpointType = "virtual" - - // VhostUserEndpointType is the vhostuser network interface. - VhostUserEndpointType EndpointType = "vhost-user" - - // BridgedMacvlanEndpointType is macvlan network interface. - BridgedMacvlanEndpointType EndpointType = "macvlan" - - // MacvtapEndpointType is macvtap network interface. - MacvtapEndpointType EndpointType = "macvtap" - - // TapEndpointType is tap network interface. - TapEndpointType EndpointType = "tap" - - // IPVlanEndpointType is ipvlan network interface. - IPVlanEndpointType EndpointType = "ipvlan" -) - -// Set sets an endpoint type based on the input string. -func (endpointType *EndpointType) Set(value string) error { - switch value { - case "physical": - *endpointType = PhysicalEndpointType - return nil - case "virtual": - *endpointType = VethEndpointType - return nil - case "vhost-user": - *endpointType = VhostUserEndpointType - return nil - case "macvlan": - *endpointType = BridgedMacvlanEndpointType - return nil - case "macvtap": - *endpointType = MacvtapEndpointType - return nil - case "tap": - *endpointType = TapEndpointType - return nil - case "ipvlan": - *endpointType = IPVlanEndpointType - return nil - default: - return fmt.Errorf("Unknown endpoint type %s", value) - } -} - -// String converts an endpoint type to a string. -func (endpointType *EndpointType) String() string { - switch *endpointType { - case PhysicalEndpointType: - return string(PhysicalEndpointType) - case VethEndpointType: - return string(VethEndpointType) - case VhostUserEndpointType: - return string(VhostUserEndpointType) - case BridgedMacvlanEndpointType: - return string(BridgedMacvlanEndpointType) - case MacvtapEndpointType: - return string(MacvtapEndpointType) - case TapEndpointType: - return string(TapEndpointType) - case IPVlanEndpointType: - return string(IPVlanEndpointType) - default: - return "" - } -} diff --git a/virtcontainers/example_pod_run_test.go b/virtcontainers/example_pod_run_test.go index 5b93479079..d3844605dd 100644 --- a/virtcontainers/example_pod_run_test.go +++ b/virtcontainers/example_pod_run_test.go @@ -11,6 +11,7 @@ import ( "strings" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -40,7 +41,7 @@ func Example_createAndStartSandbox() { } // Sets the hypervisor configuration. - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: "/usr/share/kata-containers/vmlinux.container", ImagePath: "/usr/share/kata-containers/kata-containers.img", HypervisorPath: "/usr/bin/qemu-system-x86_64", @@ -55,7 +56,7 @@ func Example_createAndStartSandbox() { // - Hypervisor is QEMU // - Agent is hyperstart sandboxConfig := vc.SandboxConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: vc.HyperstartAgent, diff --git a/virtcontainers/factory/cache/cache_test.go b/virtcontainers/factory/cache/cache_test.go index 2189360d8e..d19d4925f7 100644 --- a/virtcontainers/factory/cache/cache_test.go +++ b/virtcontainers/factory/cache/cache_test.go @@ -14,18 +14,19 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory/direct" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestTemplateFactory(t *testing.T) { assert := assert.New(t) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, HypervisorConfig: hyperConfig, diff --git a/virtcontainers/factory/direct/direct_test.go b/virtcontainers/factory/direct/direct_test.go index cca51cea97..656b7beb71 100644 --- a/virtcontainers/factory/direct/direct_test.go +++ b/virtcontainers/factory/direct/direct_test.go @@ -13,18 +13,19 @@ import ( "github.com/stretchr/testify/assert" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestTemplateFactory(t *testing.T) { assert := assert.New(t) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, HypervisorConfig: hyperConfig, diff --git a/virtcontainers/factory/factory_test.go b/virtcontainers/factory/factory_test.go index deaa02c20e..3f97601374 100644 --- a/virtcontainers/factory/factory_test.go +++ b/virtcontainers/factory/factory_test.go @@ -15,6 +15,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory/base" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestNewFactory(t *testing.T) { @@ -29,7 +30,7 @@ func TestNewFactory(t *testing.T) { assert.Error(err) config.VMConfig = vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, } @@ -39,7 +40,7 @@ func TestNewFactory(t *testing.T) { testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + config.VMConfig.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } @@ -84,7 +85,7 @@ func TestFactorySetLogger(t *testing.T) { SetLogger(context.Background(), testLog) var config Config - config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + config.VMConfig.HypervisorConfig = hypervisor.Config{ KernelPath: "foo", ImagePath: "bar", } @@ -104,8 +105,8 @@ func TestVMConfigValid(t *testing.T) { testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") config := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, - HypervisorConfig: vc.HypervisorConfig{ + HypervisorType: hypervisor.Mock, + HypervisorConfig: hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, }, @@ -134,11 +135,11 @@ func TestCheckVMConfig(t *testing.T) { err := checkVMConfig(config1, config2) assert.Nil(err) - config1.HypervisorType = vc.MockHypervisor + config1.HypervisorType = hypervisor.Mock err = checkVMConfig(config1, config2) assert.Error(err) - config2.HypervisorType = vc.MockHypervisor + config2.HypervisorType = hypervisor.Mock err = checkVMConfig(config1, config2) assert.Nil(err) @@ -151,14 +152,14 @@ func TestCheckVMConfig(t *testing.T) { assert.Nil(err) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config1.HypervisorConfig = vc.HypervisorConfig{ + config1.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } err = checkVMConfig(config1, config2) assert.Error(err) - config2.HypervisorConfig = vc.HypervisorConfig{ + config2.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } @@ -170,12 +171,12 @@ func TestFactoryGetVM(t *testing.T) { assert := assert.New(t) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hyperConfig, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, @@ -287,11 +288,11 @@ func TestDeepCompare(t *testing.T) { assert.True(deepCompare(foo, bar)) // slice - foo.HypervisorConfig.KernelParams = []vc.Param{} + foo.HypervisorConfig.KernelParams = []hypervisor.Param{} assert.True(deepCompare(foo, bar)) - foo.HypervisorConfig.KernelParams = append(foo.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"}) + foo.HypervisorConfig.KernelParams = append(foo.HypervisorConfig.KernelParams, hypervisor.Param{Key: "key", Value: "value"}) assert.False(deepCompare(foo, bar)) - bar.HypervisorConfig.KernelParams = append(bar.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"}) + bar.HypervisorConfig.KernelParams = append(bar.HypervisorConfig.KernelParams, hypervisor.Param{Key: "key", Value: "value"}) assert.True(deepCompare(foo, bar)) // map @@ -320,12 +321,12 @@ func TestDeepCompare(t *testing.T) { var err error ctx := context.Background() config.VMConfig = vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, } testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + config.VMConfig.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } diff --git a/virtcontainers/factory/template/template_test.go b/virtcontainers/factory/template/template_test.go index d08a2ab69e..d9971fdaa5 100644 --- a/virtcontainers/factory/template/template_test.go +++ b/virtcontainers/factory/template/template_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" ) func TestTemplateFactory(t *testing.T) { @@ -23,12 +24,12 @@ func TestTemplateFactory(t *testing.T) { templateWaitForAgent = 1 * time.Microsecond testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, HypervisorConfig: hyperConfig, AgentType: vc.NoopAgentType, ProxyType: vc.NoopProxyType, diff --git a/virtcontainers/hack/virtc/main.go b/virtcontainers/hack/virtc/main.go index 2c25d43031..2d9fb145bd 100644 --- a/virtcontainers/hack/virtc/main.go +++ b/virtcontainers/hack/virtc/main.go @@ -13,6 +13,7 @@ import ( "strings" "text/tabwriter" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/sirupsen/logrus" @@ -48,7 +49,7 @@ var sandboxConfigFlags = []cli.Flag{ cli.StringFlag{ Name: "machine-type", - Value: vc.QemuPC, + Value: hypervisor.QemuPC, Usage: "hypervisor machine type", }, @@ -101,7 +102,7 @@ var sandboxConfigFlags = []cli.Flag{ }, } -var ccKernelParams = []vc.Param{ +var ccKernelParams = []hypervisor.Param{ { Key: "init", Value: "/usr/lib/systemd/systemd", @@ -120,7 +121,7 @@ var ccKernelParams = []vc.Param{ }, } -func buildKernelParams(config *vc.HypervisorConfig) error { +func buildKernelParams(config *hypervisor.Config) error { for _, p := range ccKernelParams { if err := config.AddKernelParam(p); err != nil { return err @@ -155,11 +156,11 @@ func buildSandboxConfig(context *cli.Context) (vc.SandboxConfig, error) { } kernelPath := "/usr/share/clear-containers/vmlinuz.container" - if machineType == vc.QemuPCLite { + if machineType == hypervisor.QemuPCLite { kernelPath = "/usr/share/clear-containers/vmlinux.container" } - hypervisorConfig := vc.HypervisorConfig{ + hypervisorConfig := hypervisor.Config{ KernelPath: kernelPath, ImagePath: "/usr/share/clear-containers/clear-containers.img", HypervisorMachineType: machineType, @@ -170,7 +171,7 @@ func buildSandboxConfig(context *cli.Context) (vc.SandboxConfig, error) { return vc.SandboxConfig{}, err } - netConfig := vc.NetworkConfig{} + netConfig := types.NetworkConfig{} switch *agentType { case vc.HyperstartAgent: @@ -195,7 +196,7 @@ func buildSandboxConfig(context *cli.Context) (vc.SandboxConfig, error) { sandboxConfig := vc.SandboxConfig{ ID: id, - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, HypervisorConfig: hypervisorConfig, AgentType: *agentType, diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index da6b9ecb69..93b665d415 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -20,6 +20,7 @@ import ( proxyClient "github.com/clearcontainers/proxy/client" "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" @@ -152,8 +153,8 @@ func (h *hyper) processHyperRoute(route netlink.Route, deviceName string) *hyper destination = "" } else { destination = route.Dst.String() - if destination == defaultRouteDest { - destination = defaultRouteLabel + if destination == types.DefaultRouteDest { + destination = types.DefaultRouteLabel } // Skip IPv6 because not supported by hyperstart @@ -305,9 +306,9 @@ func (h *hyper) getSharePath(id string) string { return filepath.Join(defaultSharedDir, id) } -func (h *hyper) configure(hv hypervisor, id, sharePath string, builtin bool, config interface{}) error { +func (h *hyper) configure(hv hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error { for _, socket := range h.sockets { - err := hv.addDevice(socket, serialPortDev) + err := hv.AddDevice(socket, hypervisor.SerialPortDev) if err != nil { return err } @@ -324,7 +325,7 @@ func (h *hyper) configure(hv hypervisor, id, sharePath string, builtin bool, con return err } - return hv.addDevice(sharedVolume, fsDev) + return hv.AddDevice(sharedVolume, hypervisor.FsDev) } func (h *hyper) createSandbox(sandbox *Sandbox) (err error) { @@ -815,7 +816,7 @@ func (h *hyper) register() error { } defer h.disconnect() - console, err := h.sandbox.hypervisor.getSandboxConsole(h.sandbox.id) + console, err := h.sandbox.hypervisor.GetSandboxConsole(h.sandbox.id) if err != nil { return err } diff --git a/virtcontainers/hyperstart_agent_test.go b/virtcontainers/hyperstart_agent_test.go index 69d75e3b73..5fee5ca58b 100644 --- a/virtcontainers/hyperstart_agent_test.go +++ b/virtcontainers/hyperstart_agent_test.go @@ -13,6 +13,7 @@ import ( "reflect" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -130,12 +131,12 @@ func TestProcessHyperRouteEmptyGWSuccessful(t *testing.T) { func TestProcessHyperRouteEmptyDestSuccessful(t *testing.T) { expected := &hyperstart.Route{ - Dest: defaultRouteLabel, + Dest: types.DefaultRouteLabel, Gateway: testRouteGateway, Device: testRouteDeviceName, } - _, dest, err := net.ParseCIDR(defaultRouteDest) + _, dest, err := net.ParseCIDR(types.DefaultRouteDest) if err != nil { t.Fatal(err) } @@ -187,7 +188,7 @@ func TestHyperConfigure(t *testing.T) { assert.Nil(err) h := &hyper{} - m := &mockHypervisor{} + m := hypervisor.NewMock() c := HyperConfig{} id := "foobar" diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/hypervisor/bridgedmacvlan_endpoint.go similarity index 80% rename from virtcontainers/bridgedmacvlan_endpoint.go rename to virtcontainers/hypervisor/bridgedmacvlan_endpoint.go index d2efd56f53..8a741cfb37 100644 --- a/virtcontainers/bridgedmacvlan_endpoint.go +++ b/virtcontainers/hypervisor/bridgedmacvlan_endpoint.go @@ -3,23 +3,24 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/types" ) // BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM type BridgedMacvlanEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo + NetPair types.NetworkInterfacePair + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } -func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { +func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel types.NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { if idx < 0 { return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) } @@ -41,7 +42,7 @@ func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingMod } // Properties returns properties of the interface. -func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo { +func (endpoint *BridgedMacvlanEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -62,7 +63,7 @@ func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType { } // SetProperties sets the properties for the endpoint. -func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } @@ -77,19 +78,19 @@ func (endpoint *BridgedMacvlanEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *types.NetworkInterfacePair { return &endpoint.NetPair } // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error { +func (endpoint *BridgedMacvlanEndpoint) Attach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, NetDev) } // Detach for the virtual endpoint tears down the tap and bridge @@ -101,17 +102,17 @@ func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath stri return nil } - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }) } // HotAttach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error { +func (endpoint *BridgedMacvlanEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *BridgedMacvlanEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") } diff --git a/virtcontainers/bridgedmacvlan_endpoint_test.go b/virtcontainers/hypervisor/bridgedmacvlan_endpoint_test.go similarity index 69% rename from virtcontainers/bridgedmacvlan_endpoint_test.go rename to virtcontainers/hypervisor/bridgedmacvlan_endpoint_test.go index 0cc4a94012..13003feee6 100644 --- a/virtcontainers/bridgedmacvlan_endpoint_test.go +++ b/virtcontainers/hypervisor/bridgedmacvlan_endpoint_test.go @@ -3,36 +3,38 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateBridgedMacvlanEndpoint(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &BridgedMacvlanEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth4", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, EndpointType: BridgedMacvlanEndpointType, } - result, err := createBridgedMacvlanNetworkEndpoint(4, "", DefaultNetInterworkingModel) + result, err := createBridgedMacvlanNetworkEndpoint(4, "", types.DefaultNetInterworkingModel) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/hypervisor/endpoint.go b/virtcontainers/hypervisor/endpoint.go new file mode 100644 index 0000000000..d24ec35073 --- /dev/null +++ b/virtcontainers/hypervisor/endpoint.go @@ -0,0 +1,794 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + "encoding/json" + "fmt" + "net" + "sort" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + + "github.com/kata-containers/runtime/virtcontainers/types" +) + +// Endpoint represents a physical or virtual network interface. +type Endpoint interface { + Properties() types.NetworkInfo + Name() string + HardwareAddr() string + Type() EndpointType + PciAddr() string + NetworkPair() *types.NetworkInterfacePair + + SetProperties(types.NetworkInfo) + SetPciAddr(string) + Attach(Hypervisor) error + Detach(netNsCreated bool, netNsPath string) error + HotAttach(Hypervisor) error + HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error +} + +// EndpointType identifies the type of the network endpoint. +type EndpointType string + +const ( + // PhysicalEndpointType is the physical network interface. + PhysicalEndpointType EndpointType = "physical" + + // VethEndpointType is the virtual network interface. + VethEndpointType EndpointType = "virtual" + + // VhostUserEndpointType is the vhostuser network interface. + VhostUserEndpointType EndpointType = "vhost-user" + + // BridgedMacvlanEndpointType is macvlan network interface. + BridgedMacvlanEndpointType EndpointType = "macvlan" + + // MacvtapEndpointType is macvtap network interface. + MacvtapEndpointType EndpointType = "macvtap" + + // TapEndpointType is tap network interface. + TapEndpointType EndpointType = "tap" + + // IPVlanEndpointType is ipvlan network interface. + IPVlanEndpointType EndpointType = "ipvlan" +) + +// Set sets an endpoint type based on the input string. +func (endpointType *EndpointType) Set(value string) error { + switch value { + case "physical": + *endpointType = PhysicalEndpointType + return nil + case "virtual": + *endpointType = VethEndpointType + return nil + case "vhost-user": + *endpointType = VhostUserEndpointType + return nil + case "macvlan": + *endpointType = BridgedMacvlanEndpointType + return nil + case "macvtap": + *endpointType = MacvtapEndpointType + return nil + case "tap": + *endpointType = TapEndpointType + return nil + case "ipvlan": + *endpointType = IPVlanEndpointType + return nil + default: + return fmt.Errorf("Unknown endpoint type %s", value) + } +} + +// String converts an endpoint type to a string. +func (endpointType *EndpointType) String() string { + switch *endpointType { + case PhysicalEndpointType: + return string(PhysicalEndpointType) + case VethEndpointType: + return string(VethEndpointType) + case VhostUserEndpointType: + return string(VhostUserEndpointType) + case BridgedMacvlanEndpointType: + return string(BridgedMacvlanEndpointType) + case MacvtapEndpointType: + return string(MacvtapEndpointType) + case TapEndpointType: + return string(TapEndpointType) + case IPVlanEndpointType: + return string(IPVlanEndpointType) + default: + return "" + } +} + +func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.Link, error) { + var link netlink.Link + + switch ep := endpoint.(type) { + case *VethEndpoint: + link = &netlink.Veth{} + case *BridgedMacvlanEndpoint: + link = &netlink.Macvlan{} + case *IPVlanEndpoint: + link = &netlink.IPVlan{} + default: + return nil, fmt.Errorf("Unexpected endpointType %s", ep.Type()) + } + + return getLinkByName(netHandle, endpoint.NetworkPair().VirtIface.Name, link) +} + +func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + attrs := link.Attrs() + + // Attach the macvtap interface to the underlying container + // interface. Also picks relevant attributes from the parent + tapLink, err := createMacVtap(netHandle, netPair.TAPIface.Name, + &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + LinkAttrs: netlink.LinkAttrs{ + TxQLen: attrs.TxQLen, + ParentIndex: attrs.Index, + }, + }, + }, queues) + + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } + + // Save the veth MAC address to the TAP so that it can later be used + // to build the hypervisor command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + tapHardAddr := attrs.HardwareAddr + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() + + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) + } + + hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetHardwareAddr(tapLink, tapHardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) + } + + // Clear the IP addresses from the veth interface to prevent ARP conflict + netPair.VirtIface.Addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return fmt.Errorf("Unable to obtain veth IP addresses: %s", err) + } + + if err := clearIPs(link, netPair.VirtIface.Addrs); err != nil { + return fmt.Errorf("Unable to clear veth IP addresses: %s", err) + } + + if err := netHandle.LinkSetUp(link); err != nil { + return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) + } + + // Note: The underlying interfaces need to be up prior to fd creation. + + netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, queues) + if err != nil { + return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err) + } + + if !disableVhostNet { + vhostFds, err := createVhostFds(queues) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds + } + + return nil +} + +func bridgeNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } + netPair.VMFds = fds + + if !disableVhostNet { + vhostFds, err := createVhostFds(queues) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds + } + + var attrs *netlink.LinkAttrs + var link netlink.Link + + link, err = getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + attrs = link.Attrs() + + // Save the veth MAC address to the TAP so that it can later be used + // to build the hypervisor command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() + + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) + } + + hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + mcastSnoop := false + bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, queues) + if err != nil { + return fmt.Errorf("Could not create bridge: %s", err) + } + + if err := netHandle.LinkSetMaster(tapLink, bridgeLink.(*netlink.Bridge)); err != nil { + return fmt.Errorf("Could not attach TAP %s to the bridge %s: %s", + netPair.TAPIface.Name, netPair.Name, err) + } + + if err := netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkSetMaster(link, bridgeLink.(*netlink.Bridge)); err != nil { + return fmt.Errorf("Could not attach veth %s to the bridge %s: %s", + netPair.VirtIface.Name, netPair.Name, err) + } + + if err := netHandle.LinkSetUp(link); err != nil { + return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetUp(bridgeLink); err != nil { + return fmt.Errorf("Could not enable bridge %s: %s", netPair.Name, err) + } + + return nil +} + +func setupTCFiltering(endpoint Endpoint, queues int, disableVhostNet bool) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } + netPair.VMFds = fds + + if !disableVhostNet { + vhostFds, err := createVhostFds(queues) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds + } + + var attrs *netlink.LinkAttrs + var link netlink.Link + + link, err = getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + attrs = link.Attrs() + + // Save the veth MAC address to the TAP so that it can later be used + // to build the hypervisor command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() + + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) + } + + if err := netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) + } + + tapAttrs := tapLink.Attrs() + + if err := addQdiscIngress(tapAttrs.Index); err != nil { + return err + } + + if err := addQdiscIngress(attrs.Index); err != nil { + return err + } + + if err := addRedirectTCFilter(attrs.Index, tapAttrs.Index); err != nil { + return err + } + + if err := addRedirectTCFilter(tapAttrs.Index, attrs.Index); err != nil { + return err + } + + return nil +} + +func untapNetworkPair(endpoint Endpoint) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Macvtap{}) + if err != nil { + return fmt.Errorf("Could not get TAP interface %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkDel(tapLink); err != nil { + return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) + } + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + + // Restore the IPs that were cleared + err = setIPs(link, netPair.VirtIface.Addrs) + return err +} + +func unBridgeNetworkPair(endpoint Endpoint) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) + if err != nil { + return fmt.Errorf("Could not get TAP interface: %s", err) + } + + bridgeLink, err := getLinkByName(netHandle, netPair.Name, &netlink.Bridge{}) + if err != nil { + return fmt.Errorf("Could not get bridge interface: %s", err) + } + + if err := netHandle.LinkSetDown(bridgeLink); err != nil { + return fmt.Errorf("Could not disable bridge %s: %s", netPair.Name, err) + } + + if err := netHandle.LinkSetDown(tapLink); err != nil { + return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkSetNoMaster(tapLink); err != nil { + return fmt.Errorf("Could not detach TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkDel(bridgeLink); err != nil { + return fmt.Errorf("Could not remove bridge %s: %s", netPair.Name, err) + } + + if err := netHandle.LinkDel(tapLink); err != nil { + return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) + } + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) + if err != nil { + return err + } + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { + return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", + netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetNoMaster(link); err != nil { + return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) + } + + return nil +} + +func removeTCFiltering(endpoint Endpoint) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + netPair := endpoint.NetworkPair() + + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) + if err != nil { + return fmt.Errorf("Could not get TAP interface: %s", err) + } + + if err := netHandle.LinkSetDown(tapLink); err != nil { + return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) + } + + if err := netHandle.LinkDel(tapLink); err != nil { + return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) + } + + link, err := getLinkForEndpoint(endpoint, netHandle) + if err != nil { + return err + } + + if err := removeRedirectTCFilter(link); err != nil { + return err + } + + if err := removeQdiscIngress(link); err != nil { + return err + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + + return nil +} + +// The endpoint type should dictate how the connection needs to happen. +func xConnectVMNetwork(endpoint Endpoint, h Hypervisor) error { + netPair := endpoint.NetworkPair() + + queues := 0 + caps := h.Capabilities() + if caps.IsMultiQueueSupported() { + queues = int(h.Config().NumVCPUs) + } + + disableVhostNet := h.Config().DisableVhostNet + + if netPair.NetInterworkingModel == types.NetXConnectDefaultModel { + netPair.NetInterworkingModel = types.DefaultNetInterworkingModel + } + + switch netPair.NetInterworkingModel { + case types.NetXConnectBridgedModel: + return bridgeNetworkPair(endpoint, queues, disableVhostNet) + case types.NetXConnectMacVtapModel: + return tapNetworkPair(endpoint, queues, disableVhostNet) + case types.NetXConnectTCFilterModel: + return setupTCFiltering(endpoint, queues, disableVhostNet) + case types.NetXConnectEnlightenedModel: + return fmt.Errorf("Unsupported networking model") + default: + return fmt.Errorf("Invalid internetworking model") + } +} + +// The endpoint type should dictate how the disconnection needs to happen. +func xDisconnectVMNetwork(endpoint Endpoint) error { + netPair := endpoint.NetworkPair() + + if netPair.NetInterworkingModel == types.NetXConnectDefaultModel { + netPair.NetInterworkingModel = types.DefaultNetInterworkingModel + } + + switch netPair.NetInterworkingModel { + case types.NetXConnectBridgedModel: + return unBridgeNetworkPair(endpoint) + case types.NetXConnectMacVtapModel: + return untapNetworkPair(endpoint) + case types.NetXConnectTCFilterModel: + return removeTCFiltering(endpoint) + case types.NetXConnectEnlightenedModel: + return fmt.Errorf("Unsupported networking model") + default: + return fmt.Errorf("Invalid internetworking model") + } +} + +// CreateEndpoint creates an Endpoint from a network info and interworking model. +func CreateEndpoint(netInfo types.NetworkInfo, idx int, model types.NetInterworkingModel) (Endpoint, error) { + var endpoint Endpoint + // TODO: This is the incoming interface + // based on the incoming interface we should create + // an appropriate EndPoint based on interface type + // This should be a switch + + // Check if interface is a physical interface. Do not create + // tap interface/bridge if it is. + isPhysical, err := isPhysicalIface(netInfo.Iface.Name) + if err != nil { + return nil, err + } + + if isPhysical { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") + endpoint, err = createPhysicalEndpoint(netInfo) + } else { + var socketPath string + + // Check if this is a dummy interface which has a vhost-user socket associated with it + socketPath, err = vhostUserSocketPath(netInfo) + if err != nil { + return nil, err + } + + if socketPath != "" { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") + endpoint, err = createVhostUserEndpoint(netInfo, socketPath) + } else if netInfo.Iface.Type == "macvlan" { + networkLogger().Infof("macvlan interface found") + endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) + } else if netInfo.Iface.Type == "macvtap" { + networkLogger().Infof("macvtap interface found") + endpoint, err = createMacvtapNetworkEndpoint(netInfo) + } else if netInfo.Iface.Type == "tap" { + networkLogger().Info("tap interface found") + endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) + } else if netInfo.Iface.Type == "veth" { + endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) + } else if netInfo.Iface.Type == "ipvlan" { + endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) + } else { + return nil, fmt.Errorf("Unsupported network interface") + } + } + + return endpoint, err +} + +// CreateEndpointsFromScan creates a slice of Endpoints from a network namespace scan. +func CreateEndpointsFromScan(networkNSPath string, config *types.NetworkConfig) ([]Endpoint, error) { + var endpoints []Endpoint + + netnsHandle, err := netns.GetFromPath(networkNSPath) + if err != nil { + return []Endpoint{}, err + } + defer netnsHandle.Close() + + netlinkHandle, err := netlink.NewHandleAt(netnsHandle) + if err != nil { + return []Endpoint{}, err + } + defer netlinkHandle.Delete() + + linkList, err := netlinkHandle.LinkList() + if err != nil { + return []Endpoint{}, err + } + + idx := 0 + for _, link := range linkList { + var ( + endpoint Endpoint + errCreate error + ) + + netInfo, err := networkInfoFromLink(netlinkHandle, link) + if err != nil { + return []Endpoint{}, err + } + + // Ignore unconfigured network interfaces. These are + // either base tunnel devices that are not namespaced + // like gre0, gretap0, sit0, ipip0, tunl0 or incorrectly + // setup interfaces. + if len(netInfo.Addrs) == 0 { + continue + } + + // Skip any loopback interfaces: + if (netInfo.Iface.Flags & net.FlagLoopback) != 0 { + continue + } + + if err := DoNetNS(networkNSPath, func(_ ns.NetNS) error { + endpoint, errCreate = CreateEndpoint(netInfo, idx, config.InterworkingModel) + return errCreate + }); err != nil { + return []Endpoint{}, err + } + + endpoint.SetProperties(netInfo) + endpoints = append(endpoints, endpoint) + + idx++ + } + + sort.Slice(endpoints, func(i, j int) bool { + return endpoints[i].Name() < endpoints[j].Name() + }) + + networkLogger().WithField("endpoints", endpoints).Info("Endpoints found after scan") + + return endpoints, nil +} + +// TypedJSONEndpoint is used as an intermediate representation for +// marshalling and unmarshalling Endpoint objects. +type TypedJSONEndpoint struct { + Type EndpointType + Data json.RawMessage +} + +// UnmarshalEndpoints unmarshalls a slice of typed Endpoints into a slice of regular ones. +func UnmarshalEndpoints(typedEndpoints []TypedJSONEndpoint) ([]Endpoint, error) { + var endpoints []Endpoint + + for _, e := range typedEndpoints { + switch e.Type { + case PhysicalEndpointType: + var endpoint PhysicalEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "physical", + }).Info("endpoint unmarshalled") + + case VethEndpointType: + var endpoint VethEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "virtual", + }).Info("endpoint unmarshalled") + + case VhostUserEndpointType: + var endpoint VhostUserEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "vhostuser", + }).Info("endpoint unmarshalled") + + case BridgedMacvlanEndpointType: + var endpoint BridgedMacvlanEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "macvlan", + }).Info("endpoint unmarshalled") + + case MacvtapEndpointType: + var endpoint MacvtapEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "macvtap", + }).Info("endpoint unmarshalled") + + case TapEndpointType: + var endpoint TapEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return nil, err + } + + endpoints = append(endpoints, &endpoint) + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "tap", + }).Info("endpoint unmarshalled") + + default: + networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type") + } + } + return endpoints, nil +} diff --git a/virtcontainers/endpoint_test.go b/virtcontainers/hypervisor/endpoint_test.go similarity index 98% rename from virtcontainers/endpoint_test.go rename to virtcontainers/hypervisor/endpoint_test.go index e5473a96c7..63a919fba3 100644 --- a/virtcontainers/endpoint_test.go +++ b/virtcontainers/hypervisor/endpoint_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import "testing" diff --git a/virtcontainers/fc.go b/virtcontainers/hypervisor/fc.go similarity index 88% rename from virtcontainers/fc.go rename to virtcontainers/hypervisor/fc.go index 9111fff653..0dfc163c6f 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/hypervisor/fc.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" @@ -100,19 +100,19 @@ type firecracker struct { socketPath string store *store.VCStore - config HypervisorConfig + config Config pendingDevices []firecrackerDevice // Devices to be added when the FC API is ready ctx context.Context } type firecrackerDevice struct { dev interface{} - devType deviceType + devType Device } // Logger returns a logrus logger appropriate for logging firecracker messages func (fc *firecracker) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "firecracker") + return hLog.WithField("subsystem", "firecracker") } func (fc *firecracker) trace(name string) (opentracing.Span, context.Context) { @@ -129,9 +129,9 @@ func (fc *firecracker) trace(name string) (opentracing.Span, context.Context) { return span, ctx } -// For firecracker this call only sets the internal structure up. +// CreateSandbox only sets the internal structure up. This is specific to Firecracker. // The sandbox will be created and started through startSandbox(). -func (fc *firecracker) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (fc *firecracker) CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, vcStore *store.VCStore) error { fc.ctx = ctx span, _ := fc.trace("createSandbox") @@ -335,10 +335,10 @@ func (fc *firecracker) fcStartVM() error { return nil } -// startSandbox will start the hypervisor for the given sandbox. +// StartSandbox will start the hypervisor for the given sandbox. // In the context of firecracker, this will start the hypervisor, // for configuration, but not yet start the actual virtual machine -func (fc *firecracker) startSandbox(timeout int) error { +func (fc *firecracker) StartSandbox(timeout int) error { span, _ := fc.trace("startSandbox") defer span.Finish() @@ -373,7 +373,7 @@ func (fc *firecracker) startSandbox(timeout int) error { fc.createDiskPool() for _, d := range fc.pendingDevices { - if err = fc.addDevice(d.dev, d.devType); err != nil { + if err = fc.AddDevice(d.dev, d.devType); err != nil { return err } } @@ -424,8 +424,8 @@ func (fc *firecracker) createDiskPool() error { return nil } -// stopSandbox will stop the Sandbox's VM. -func (fc *firecracker) stopSandbox() (err error) { +// StopSandbox will stop the Sandbox's VM. +func (fc *firecracker) StopSandbox() (err error) { span, _ := fc.trace("stopSandbox") defer span.Finish() @@ -473,26 +473,26 @@ func (fc *firecracker) stopSandbox() (err error) { return syscall.Kill(pid, syscall.SIGKILL) } -func (fc *firecracker) pauseSandbox() error { +func (fc *firecracker) PauseSandbox() error { return nil } -func (fc *firecracker) saveSandbox() error { +func (fc *firecracker) SaveSandbox() error { return nil } -func (fc *firecracker) resumeSandbox() error { +func (fc *firecracker) ResumeSandbox() error { return nil } -func (fc *firecracker) fcAddVsock(vs kataVSOCK) error { +func (fc *firecracker) fcAddVsock(vs types.VSOCK) error { span, _ := fc.trace("fcAddVsock") defer span.Finish() vsockParams := ops.NewPutGuestVsockByIDParams() vsockID := "root" vsock := &models.Vsock{ - GuestCid: int64(vs.contextID), + GuestCid: int64(vs.ContextID), ID: &vsockID, } vsockParams.SetID(vsockID) @@ -504,7 +504,7 @@ func (fc *firecracker) fcAddVsock(vs kataVSOCK) error { //Still racy. There is no way to send an fd to the firecracker //REST API. We could release this just before we start the instance //but even that will not eliminate the race - vs.vhostFd.Close() + vs.VHostFd.Close() return nil } @@ -592,9 +592,9 @@ func (fc *firecracker) fcUpdateBlockDrive(drive config.BlockDrive) error { return nil } -// addDevice will add extra devices to firecracker. Limited to configure before the +// AddDevice will add extra devices to firecracker. Limited to configure before the // virtual machine starts. Devices include drivers and network interfaces only. -func (fc *firecracker) addDevice(devInfo interface{}, devType deviceType) error { +func (fc *firecracker) AddDevice(devInfo interface{}, devType Device) error { span, _ := fc.trace("addDevice") defer span.Finish() @@ -618,7 +618,7 @@ func (fc *firecracker) addDevice(devInfo interface{}, devType deviceType) error case config.BlockDrive: fc.Logger().WithField("device-type-blockdrive", devInfo).Info("Adding device") return fc.fcAddBlockDrive(v) - case kataVSOCK: + case types.VSOCK: fc.Logger().WithField("device-type-vsock", devInfo).Info("Adding device") return fc.fcAddVsock(v) default: @@ -629,43 +629,43 @@ func (fc *firecracker) addDevice(devInfo interface{}, devType deviceType) error return nil } -// hotplugAddDevice supported in Firecracker VMM -func (fc *firecracker) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +// HotplugAddDevice supported in Firecracker VMM +func (fc *firecracker) HotplugAddDevice(devInfo interface{}, devType Device) (interface{}, error) { span, _ := fc.trace("hotplugAddDevice") defer span.Finish() switch devType { - case blockDev: + case BlockDev: //The drive placeholder has to exist prior to Update return nil, fc.fcUpdateBlockDrive(*devInfo.(*config.BlockDrive)) default: fc.Logger().WithFields(logrus.Fields{"devInfo": devInfo, - "deviceType": devType}).Warn("hotplugAddDevice: unsupported device") - return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, deviceType%v", + "Device": devType}).Warn("hotplugAddDevice: unsupported device") + return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, Device%v", devInfo, devType) } } -// hotplugRemoveDevice supported in Firecracker VMM, but no-op -func (fc *firecracker) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +// HotplugRemoveDevice supported in Firecracker VMM, but no-op +func (fc *firecracker) HotplugRemoveDevice(devInfo interface{}, devType Device) (interface{}, error) { return nil, nil } -// getSandboxConsole builds the path of the console where we can read +// GetSandboxConsole builds the path of the console where we can read // logs coming from the sandbox. // // we can get logs from firecracker itself; WIP on enabling. Who needs // logs when you're just hacking? -func (fc *firecracker) getSandboxConsole(id string) (string, error) { +func (fc *firecracker) GetSandboxConsole(id string) (string, error) { return "", nil } -func (fc *firecracker) disconnect() { +func (fc *firecracker) Disconnect() { fc.state.set(notReady) } -// Adds all capabilities supported by firecracker implementation of hypervisor interface -func (fc *firecracker) capabilities() types.Capabilities { +// Capabilities adds all capabilities supported by firecracker implementation of hypervisor interface +func (fc *firecracker) Capabilities() types.Capabilities { span, _ := fc.trace("capabilities") defer span.Finish() var caps types.Capabilities @@ -675,30 +675,30 @@ func (fc *firecracker) capabilities() types.Capabilities { return caps } -func (fc *firecracker) hypervisorConfig() HypervisorConfig { +func (fc *firecracker) Config() Config { return fc.config } -func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { +func (fc *firecracker) ResizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { return 0, nil } -func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { +func (fc *firecracker) ResizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { return 0, 0, nil } -// this is used to apply cgroup information on the host. not sure how necessary this +// GetThreadIDs is used to apply cgroup information on the host. not sure how necessary this // is in the first pass. // // Need to see if there's an easy way to ask firecracker for thread ids associated with // the vCPUs. Issue opened to ask for per vCPU thread IDs: // https://github.com/firecracker-microvm/firecracker/issues/718 -func (fc *firecracker) getThreadIDs() (*threadIDs, error) { +func (fc *firecracker) GetThreadIDs() (*ThreadIDs, error) { //TODO: this may not be exactly supported in Firecracker. Closest is cpu-template as part // of get /machine-config return nil, nil } -func (fc *firecracker) cleanup() error { +func (fc *firecracker) Cleanup() error { return nil } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor/hypervisor.go similarity index 71% rename from virtcontainers/hypervisor.go rename to virtcontainers/hypervisor/hypervisor.go index dd37f8706d..d1c63daa55 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor/hypervisor.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "bufio" @@ -17,99 +17,122 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/sirupsen/logrus" ) -// HypervisorType describes an hypervisor type. -type HypervisorType string +var hLog = logrus.WithField("source", "virtcontainers/hypervisor") -type operation int +// Type describes an hypervisor type. +type Type string const ( - // FirecrackerHypervisor is the FC hypervisor. - FirecrackerHypervisor HypervisorType = "firecracker" + // Firecracker is the Firecracker hypervisor. + Firecracker Type = "firecracker" - // QemuHypervisor is the QEMU hypervisor. - QemuHypervisor HypervisorType = "qemu" + // Qemu is the QEMU hypervisor. + Qemu Type = "qemu" - // MockHypervisor is a mock hypervisor for testing purposes - MockHypervisor HypervisorType = "mock" + // Mock is a mock hypervisor for testing purposes + Mock Type = "mock" ) +// Operation represents a hypervisor device operation +type Operation int + const ( - procMemInfo = "/proc/meminfo" - procCPUInfo = "/proc/cpuinfo" + // AddDevice adds a device to a guest. + AddDevice Operation = iota + + // RemoveDevice removes a device from a guest. + RemoveDevice ) const ( - defaultVCPUs = 1 - // 2 GiB - defaultMemSzMiB = 2048 + // ProcMemInfo is the /proc/meminfo path + ProcMemInfo = "/proc/meminfo" - defaultBridges = 1 - - defaultBlockDriver = config.VirtioSCSI + // ProcCPUInfo is the /proc/cpuinfo path + ProcCPUInfo = "/proc/cpuinfo" ) -// In some architectures the maximum number of vCPUs depends on the number of physical cores. -var defaultMaxQemuVCPUs = MaxQemuVCPUs() +const ( + // DefaultVCPUs is the number of vCPUs a virtcontainers VM will run with by default. + DefaultVCPUs = 1 -// deviceType describes a virtualized device type. -type deviceType int + // DefaultMemSzMiB is the amount of memory a virtcontainers VM will run with by default. + DefaultMemSzMiB = 2048 + + // DefaultBridges is the number of PCI bridges a virtcontainers VM will run with by default. + DefaultBridges = 1 + + // DefaultBlockDriver is the default virtio block based driver. + DefaultBlockDriver = config.VirtioSCSI + + // DefaultMsize9p is the default 9pfs msize value. + DefaultMsize9p = 8192 +) + +// Device describes a virtualized device. +type Device int const ( // ImgDev is the image device type. - imgDev deviceType = iota + ImgDev Device = iota // FsDev is the filesystem device type. - fsDev + FsDev // NetDev is the network device type. - netDev + NetDev // SerialDev is the serial device type. - serialDev // nolint: varcheck,unused + SerialDev // nolint: varcheck,unused // BlockDev is the block device type. - blockDev + BlockDev // ConsoleDev is the console device type. - consoleDev // nolint: varcheck,unused + ConsoleDev // nolint: varcheck,unused // SerialPortDev is the serial port device type. - serialPortDev + SerialPortDev - // vSockPCIDev is the vhost vsock PCI device type. - vSockPCIDev + // VSockPCIDev is the vhost vsock PCI device type. + VSockPCIDev - // VFIODevice is VFIO device type - vfioDev + // VfioDev is VFIO device type + VfioDev - // vhostuserDev is a Vhost-user device type - vhostuserDev + // VhostuserDev is a Vhost-user device type + VhostuserDev - // CPUDevice is CPU device type - cpuDev + // CPUDev is CPU device type + CPUDev - // memoryDevice is memory device type - memoryDev + // MemoryDev is memory device type + MemoryDev ) -type memoryDevice struct { - slot int - sizeMB int +// MemoryDevice represents a memory slot +type MemoryDevice struct { + // Slot is the memory slot ID + Slot int + + // SizeMB is the memory slot size in MBytes. + SizeMB int } // Set sets an hypervisor type based on the input string. -func (hType *HypervisorType) Set(value string) error { +func (t *Type) Set(value string) error { switch value { case "qemu": - *hType = QemuHypervisor + *t = Qemu return nil case "firecracker": - *hType = FirecrackerHypervisor + *t = Firecracker return nil case "mock": - *hType = MockHypervisor + *t = Mock return nil default: return fmt.Errorf("Unknown hypervisor type %s", value) @@ -117,30 +140,30 @@ func (hType *HypervisorType) Set(value string) error { } // String converts an hypervisor type to a string. -func (hType *HypervisorType) String() string { - switch *hType { - case QemuHypervisor: - return string(QemuHypervisor) - case FirecrackerHypervisor: - return string(FirecrackerHypervisor) - case MockHypervisor: - return string(MockHypervisor) +func (t *Type) String() string { + switch *t { + case Qemu: + return string(Qemu) + case Firecracker: + return string(Firecracker) + case Mock: + return string(Mock) default: return "" } } -// newHypervisor returns an hypervisor from and hypervisor type. -func newHypervisor(hType HypervisorType) (hypervisor, error) { - switch hType { - case QemuHypervisor: +// New returns an hypervisor from and hypervisor type. +func New(t Type) (Hypervisor, error) { + switch t { + case Qemu: return &qemu{}, nil - case FirecrackerHypervisor: + case Mock: + return &mock{}, nil + case Firecracker: return &firecracker{}, nil - case MockHypervisor: - return &mockHypervisor{}, nil default: - return nil, fmt.Errorf("Unknown hypervisor type %s", hType) + return nil, fmt.Errorf("Unknown hypervisor type %s", t) } } @@ -150,12 +173,12 @@ type Param struct { Value string } -// HypervisorConfig is the hypervisor configuration. -type HypervisorConfig struct { +// Config is the hypervisor configuration. +type Config struct { // NumVCPUs specifies default number of vCPUs for the VM. NumVCPUs uint32 - //DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. + // DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. DefaultMaxVCPUs uint32 // DefaultMem specifies default memory size in MiB for the VM. @@ -285,11 +308,13 @@ type HypervisorConfig struct { GuestHookPath string } -type threadIDs struct { - vcpus []int +// ThreadIDs represent a set of threads vCPU IDs. +type ThreadIDs struct { + // VCPUs is a slice of vCPU IDs. + VCPUs []int } -func (conf *HypervisorConfig) checkTemplateConfig() error { +func (conf *Config) checkTemplateConfig() error { if conf.BootToBeTemplate && conf.BootFromTemplate { return fmt.Errorf("Cannot set both 'to be' and 'from' vm tempate") } @@ -307,7 +332,8 @@ func (conf *HypervisorConfig) checkTemplateConfig() error { return nil } -func (conf *HypervisorConfig) valid() error { +// Valid checks if a hypervisor configuration is valid. +func (conf *Config) Valid() error { if conf.KernelPath == "" { return fmt.Errorf("Missing kernel path") } @@ -321,27 +347,23 @@ func (conf *HypervisorConfig) valid() error { } if conf.NumVCPUs == 0 { - conf.NumVCPUs = defaultVCPUs + conf.NumVCPUs = DefaultVCPUs } if conf.MemorySize == 0 { - conf.MemorySize = defaultMemSzMiB + conf.MemorySize = DefaultMemSzMiB } if conf.DefaultBridges == 0 { - conf.DefaultBridges = defaultBridges + conf.DefaultBridges = DefaultBridges } if conf.BlockDeviceDriver == "" { - conf.BlockDeviceDriver = defaultBlockDriver - } - - if conf.DefaultMaxVCPUs == 0 { - conf.DefaultMaxVCPUs = defaultMaxQemuVCPUs + conf.BlockDeviceDriver = DefaultBlockDriver } if conf.Msize9p == 0 { - conf.Msize9p = defaultMsize9p + conf.Msize9p = DefaultMsize9p } return nil @@ -349,7 +371,7 @@ func (conf *HypervisorConfig) valid() error { // AddKernelParam allows the addition of new kernel parameters to an existing // hypervisor configuration. -func (conf *HypervisorConfig) AddKernelParam(p Param) error { +func (conf *Config) AddKernelParam(p Param) error { if p.Key == "" { return fmt.Errorf("Empty kernel parameter") } @@ -359,7 +381,8 @@ func (conf *HypervisorConfig) AddKernelParam(p Param) error { return nil } -func (conf *HypervisorConfig) addCustomAsset(a *types.Asset) error { +// AddCustomAsset adds a custom asset to a hypervisor configuration +func (conf *Config) AddCustomAsset(a *types.Asset) error { if a == nil || a.Path() == "" { // We did not get a custom asset, we will use the default one. return nil @@ -369,7 +392,7 @@ func (conf *HypervisorConfig) addCustomAsset(a *types.Asset) error { return fmt.Errorf("Invalid %s at %s", a.Type(), a.Path()) } - virtLog.Debugf("Using custom %v asset %s", a.Type(), a.Path()) + hLog.Debugf("Using custom %v asset %s", a.Type(), a.Path()) if conf.customAssets == nil { conf.customAssets = make(map[types.AssetType]*types.Asset) @@ -380,7 +403,7 @@ func (conf *HypervisorConfig) addCustomAsset(a *types.Asset) error { return nil } -func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) { +func (conf *Config) assetPath(t types.AssetType) (string, error) { // Custom assets take precedence over the configured ones a, ok := conf.customAssets[t] if ok { @@ -405,7 +428,7 @@ func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) { } } -func (conf *HypervisorConfig) isCustomAsset(t types.AssetType) bool { +func (conf *Config) isCustomAsset(t types.AssetType) bool { _, ok := conf.customAssets[t] if ok { return true @@ -415,52 +438,52 @@ func (conf *HypervisorConfig) isCustomAsset(t types.AssetType) bool { } // KernelAssetPath returns the guest kernel path -func (conf *HypervisorConfig) KernelAssetPath() (string, error) { +func (conf *Config) KernelAssetPath() (string, error) { return conf.assetPath(types.KernelAsset) } // CustomKernelAsset returns true if the kernel asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomKernelAsset() bool { +func (conf *Config) CustomKernelAsset() bool { return conf.isCustomAsset(types.KernelAsset) } // ImageAssetPath returns the guest image path -func (conf *HypervisorConfig) ImageAssetPath() (string, error) { +func (conf *Config) ImageAssetPath() (string, error) { return conf.assetPath(types.ImageAsset) } // CustomImageAsset returns true if the image asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomImageAsset() bool { +func (conf *Config) CustomImageAsset() bool { return conf.isCustomAsset(types.ImageAsset) } // InitrdAssetPath returns the guest initrd path -func (conf *HypervisorConfig) InitrdAssetPath() (string, error) { +func (conf *Config) InitrdAssetPath() (string, error) { return conf.assetPath(types.InitrdAsset) } // CustomInitrdAsset returns true if the initrd asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomInitrdAsset() bool { +func (conf *Config) CustomInitrdAsset() bool { return conf.isCustomAsset(types.InitrdAsset) } // HypervisorAssetPath returns the VM hypervisor path -func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) { +func (conf *Config) HypervisorAssetPath() (string, error) { return conf.assetPath(types.HypervisorAsset) } // CustomHypervisorAsset returns true if the hypervisor asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomHypervisorAsset() bool { +func (conf *Config) CustomHypervisorAsset() bool { return conf.isCustomAsset(types.HypervisorAsset) } // FirmwareAssetPath returns the guest firmware path -func (conf *HypervisorConfig) FirmwareAssetPath() (string, error) { +func (conf *Config) FirmwareAssetPath() (string, error) { return conf.assetPath(types.FirmwareAsset) } // CustomFirmwareAsset returns true if the firmware asset is a custom one, false otherwise. -func (conf *HypervisorConfig) CustomFirmwareAsset() bool { +func (conf *Config) CustomFirmwareAsset() bool { return conf.isCustomAsset(types.FirmwareAsset) } @@ -509,7 +532,8 @@ func DeserializeParams(parameters []string) []Param { return params } -func getHostMemorySizeKb(memInfoPath string) (uint64, error) { +// GetHostMemorySizeKb return the host memory size in KBytes. +func GetHostMemorySizeKb(memInfoPath string) (uint64, error) { f, err := os.Open(memInfoPath) if err != nil { return 0, err @@ -545,7 +569,7 @@ func getHostMemorySizeKb(memInfoPath string) (uint64, error) { // RunningOnVMM checks if the system is running inside a VM. func RunningOnVMM(cpuInfoPath string) (bool, error) { if runtime.GOARCH == "arm64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x" { - virtLog.Info("Unable to know if the system is running inside a VM") + hLog.Info("Unable to know if the system is running inside a VM") return false, nil } @@ -589,24 +613,24 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) { return false, fmt.Errorf("Couldn't find %q from %q output", flagsField, cpuInfoPath) } -// hypervisor is the virtcontainers hypervisor interface. +// Hypervisor is the virtcontainers hypervisor interface. // The default hypervisor implementation is Qemu. -type hypervisor interface { - createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error - startSandbox(timeout int) error - stopSandbox() error - pauseSandbox() error - saveSandbox() error - resumeSandbox() error - addDevice(devInfo interface{}, devType deviceType) error - hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) - hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) - resizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error) - resizeVCPUs(vcpus uint32) (uint32, uint32, error) - getSandboxConsole(sandboxID string) (string, error) - disconnect() - capabilities() types.Capabilities - hypervisorConfig() HypervisorConfig - getThreadIDs() (*threadIDs, error) - cleanup() error +type Hypervisor interface { + CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, store *store.VCStore) error + StartSandbox(timeout int) error + StopSandbox() error + PauseSandbox() error + SaveSandbox() error + ResumeSandbox() error + AddDevice(devInfo interface{}, dev Device) error + HotplugAddDevice(devInfo interface{}, dev Device) (interface{}, error) + HotplugRemoveDevice(devInfo interface{}, dev Device) (interface{}, error) + ResizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error) + ResizeVCPUs(vcpus uint32) (uint32, uint32, error) + GetSandboxConsole(sandboxID string) (string, error) + Disconnect() + Capabilities() types.Capabilities + Config() Config + GetThreadIDs() (*ThreadIDs, error) + Cleanup() error } diff --git a/virtcontainers/hypervisor_test.go b/virtcontainers/hypervisor/hypervisor_test.go similarity index 66% rename from virtcontainers/hypervisor_test.go rename to virtcontainers/hypervisor/hypervisor_test.go index 0a1b2514f4..95e12591a5 100644 --- a/virtcontainers/hypervisor_test.go +++ b/virtcontainers/hypervisor/hypervisor_test.go @@ -3,19 +3,31 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( + "flag" "fmt" "io/ioutil" "os" "path/filepath" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/store" ) -func testSetHypervisorType(t *testing.T, value string, expected HypervisorType) { - var hypervisorType HypervisorType +const testKernel = "kernel" +const testInitrd = "initrd" +const testImage = "image" +const testHypervisor = "hypervisor" + +var testDir = "" + +const testDisabledAsNonRoot = "Test disabled as requires root privileges" + +func testSetType(t *testing.T, value string, expected Type) { + var hypervisorType Type err := (&hypervisorType).Set(value) if err != nil { @@ -27,52 +39,53 @@ func testSetHypervisorType(t *testing.T, value string, expected HypervisorType) } } -func TestSetQemuHypervisorType(t *testing.T) { - testSetHypervisorType(t, "qemu", QemuHypervisor) +func TestSetQemuType(t *testing.T) { + testSetType(t, "qemu", Qemu) } -func TestSetMockHypervisorType(t *testing.T) { - testSetHypervisorType(t, "mock", MockHypervisor) +func TestSetMockType(t *testing.T) { + testSetType(t, "mock", Mock) } -func TestSetUnknownHypervisorType(t *testing.T) { - var hypervisorType HypervisorType +func TestSetUnknownType(t *testing.T) { + var hypervisorType Type err := (&hypervisorType).Set("unknown") if err == nil { t.Fatal() } - if hypervisorType == QemuHypervisor || - hypervisorType == MockHypervisor { + if hypervisorType == Qemu || + hypervisorType == Mock || + hypervisorType == Firecracker { t.Fatal() } } -func testStringFromHypervisorType(t *testing.T, hypervisorType HypervisorType, expected string) { +func testStringFromType(t *testing.T, hypervisorType Type, expected string) { hypervisorTypeStr := (&hypervisorType).String() if hypervisorTypeStr != expected { t.Fatal() } } -func TestStringFromQemuHypervisorType(t *testing.T) { - hypervisorType := QemuHypervisor - testStringFromHypervisorType(t, hypervisorType, "qemu") +func TestStringFromQemuType(t *testing.T) { + hypervisorType := Qemu + testStringFromType(t, hypervisorType, "qemu") } -func TestStringFromMockHypervisorType(t *testing.T) { - hypervisorType := MockHypervisor - testStringFromHypervisorType(t, hypervisorType, "mock") +func TestStringFromMockType(t *testing.T) { + hypervisorType := Mock + testStringFromType(t, hypervisorType, "mock") } -func TestStringFromUnknownHypervisorType(t *testing.T) { - var hypervisorType HypervisorType - testStringFromHypervisorType(t, hypervisorType, "") +func TestStringFromUnknownType(t *testing.T) { + var hypervisorType Type + testStringFromType(t, hypervisorType, "") } -func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType HypervisorType, expected hypervisor) { - hy, err := newHypervisor(hypervisorType) +func testNewHypervisorFromType(t *testing.T, hypervisorType Type, expected Hypervisor) { + hy, err := New(hypervisorType) if err != nil { t.Fatal(err) } @@ -82,22 +95,22 @@ func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType Hypervisor } } -func TestNewHypervisorFromQemuHypervisorType(t *testing.T) { - hypervisorType := QemuHypervisor +func TestNewHypervisorFromQemuType(t *testing.T) { + hypervisorType := Qemu expectedHypervisor := &qemu{} - testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor) + testNewHypervisorFromType(t, hypervisorType, expectedHypervisor) } -func TestNewHypervisorFromMockHypervisorType(t *testing.T) { - hypervisorType := MockHypervisor - expectedHypervisor := &mockHypervisor{} - testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor) -} +// func TestNewHypervisorFromMockType(t *testing.T) { +// hypervisorType := Mock +// expectedHypervisor := &mockHypervisor{} +// testNewHypervisorFromType(t, hypervisorType, expectedHypervisor) +// } -func TestNewHypervisorFromUnknownHypervisorType(t *testing.T) { - var hypervisorType HypervisorType +func TestNewHypervisorFromUnknownType(t *testing.T) { + var hypervisorType Type - hy, err := newHypervisor(hypervisorType) + hy, err := New(hypervisorType) if err == nil { t.Fatal() } @@ -107,8 +120,8 @@ func TestNewHypervisorFromUnknownHypervisorType(t *testing.T) { } } -func testHypervisorConfigValid(t *testing.T, hypervisorConfig *HypervisorConfig, success bool) { - err := hypervisorConfig.valid() +func testConfigValid(t *testing.T, hypervisorConfig *Config, success bool) { + err := hypervisorConfig.Valid() if success && err != nil { t.Fatal() } @@ -117,88 +130,87 @@ func testHypervisorConfigValid(t *testing.T, hypervisorConfig *HypervisorConfig, } } -func TestHypervisorConfigNoKernelPath(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigNoKernelPath(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: "", ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) } -func TestHypervisorConfigNoImagePath(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigNoImagePath(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: "", HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) } -func TestHypervisorConfigNoHypervisorPath(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigNoHypervisorPath(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: "", } - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) } -func TestHypervisorConfigIsValid(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigIsValid(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) } -func TestHypervisorConfigValidTemplateConfig(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigValidTemplateConfig(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), BootToBeTemplate: true, BootFromTemplate: true, } - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) hypervisorConfig.BootToBeTemplate = false - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) hypervisorConfig.MemoryPath = "foobar" - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) hypervisorConfig.DevicesStatePath = "foobar" - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) hypervisorConfig.BootFromTemplate = false hypervisorConfig.BootToBeTemplate = true - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) hypervisorConfig.MemoryPath = "" - testHypervisorConfigValid(t, hypervisorConfig, false) + testConfigValid(t, hypervisorConfig, false) } -func TestHypervisorConfigDefaults(t *testing.T) { - hypervisorConfig := &HypervisorConfig{ +func TestConfigDefaults(t *testing.T) { + hypervisorConfig := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: "", } - testHypervisorConfigValid(t, hypervisorConfig, true) + testConfigValid(t, hypervisorConfig, true) - hypervisorConfigDefaultsExpected := &HypervisorConfig{ + hypervisorConfigDefaultsExpected := &Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: "", - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: DefaultVCPUs, + MemorySize: DefaultMemSzMiB, + DefaultBridges: DefaultBridges, + BlockDeviceDriver: DefaultBlockDriver, + Msize9p: DefaultMsize9p, } if reflect.DeepEqual(hypervisorConfig, hypervisorConfigDefaultsExpected) == false { @@ -353,7 +365,7 @@ func TestDeserializeParams(t *testing.T) { } func TestAddKernelParamValid(t *testing.T) { - var config HypervisorConfig + var config Config expected := []Param{ {"foo", "bar"}, @@ -366,7 +378,7 @@ func TestAddKernelParamValid(t *testing.T) { } func TestAddKernelParamInvalid(t *testing.T) { - var config HypervisorConfig + var config Config invalid := []Param{ {"", "bar"}, @@ -415,7 +427,7 @@ func TestGetHostMemorySizeKb(t *testing.T) { defer os.RemoveAll(dir) file := filepath.Join(dir, "meminfo") - if _, err := getHostMemorySizeKb(file); err == nil { + if _, err := GetHostMemorySizeKb(file); err == nil { t.Fatalf("expected failure as file %q does not exist", file) } @@ -425,7 +437,7 @@ func TestGetHostMemorySizeKb(t *testing.T) { } defer os.Remove(file) - hostMemKb, err := getHostMemorySizeKb(file) + hostMemKb, err := GetHostMemorySizeKb(file) if (d.expectError && err == nil) || (!d.expectError && err != nil) { t.Fatalf("got %d, input %v", hostMemKb, d) @@ -518,3 +530,58 @@ func TestRunningOnVMMNotExistingCPUInfoPathFailure(t *testing.T) { t.Fatalf("Should fail because %q file path does not exist", filePath) } } + +// TestMain is the common main function used by ALL the test functions +// for this package. +func TestMain(m *testing.M) { + var err error + + flag.Parse() + + testDir, err = ioutil.TempDir("", "vc-qemu-tmp-") + if err != nil { + panic(err) + } + + fmt.Printf("INFO: Creating hypervisor test directory %s\n", testDir) + err = os.MkdirAll(testDir, store.DirMode) + if err != nil { + fmt.Println("Could not create test directories:", err) + os.Exit(1) + } + + testQemuKernelPath = filepath.Join(testDir, testKernel) + testQemuInitrdPath = filepath.Join(testDir, testInitrd) + testQemuImagePath = filepath.Join(testDir, testImage) + testQemuPath = filepath.Join(testDir, testHypervisor) + + fmt.Printf("INFO: Creating hypervisor test kernel %s\n", testQemuKernelPath) + _, err = os.Create(testQemuKernelPath) + if err != nil { + fmt.Println("Could not create test kernel:", err) + os.RemoveAll(testDir) + os.Exit(1) + } + + fmt.Printf("INFO: Creating hypervisor test image %s\n", testQemuImagePath) + _, err = os.Create(testQemuImagePath) + if err != nil { + fmt.Println("Could not create test image:", err) + os.RemoveAll(testDir) + os.Exit(1) + } + + fmt.Printf("INFO: Creating hypervisor test hypervisor %s\n", testQemuPath) + _, err = os.Create(testQemuPath) + if err != nil { + fmt.Println("Could not create test hypervisor:", err) + os.RemoveAll(testDir) + os.Exit(1) + } + + ret := m.Run() + + os.RemoveAll(testDir) + + os.Exit(ret) +} diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/hypervisor/ipvlan_endpoint.go similarity index 80% rename from virtcontainers/ipvlan_endpoint.go rename to virtcontainers/hypervisor/ipvlan_endpoint.go index b71bbd0fbe..6d3c642678 100644 --- a/virtcontainers/ipvlan_endpoint.go +++ b/virtcontainers/hypervisor/ipvlan_endpoint.go @@ -3,18 +3,19 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/types" ) // IPVlanEndpoint represents a ipvlan endpoint that is bridged to the VM type IPVlanEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo + NetPair types.NetworkInterfacePair + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } @@ -26,7 +27,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error // Use tc filtering for ipvlan, since the other inter networking models will // not work for ipvlan. - interworkingModel := NetXConnectTCFilterModel + interworkingModel := types.NetXConnectTCFilterModel netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) if err != nil { return nil, err @@ -44,7 +45,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error } // Properties returns properties of the interface. -func (endpoint *IPVlanEndpoint) Properties() NetworkInfo { +func (endpoint *IPVlanEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -65,7 +66,7 @@ func (endpoint *IPVlanEndpoint) Type() EndpointType { } // SetProperties sets the properties for the endpoint. -func (endpoint *IPVlanEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *IPVlanEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } @@ -80,19 +81,19 @@ func (endpoint *IPVlanEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *IPVlanEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *IPVlanEndpoint) NetworkPair() *types.NetworkInterfacePair { return &endpoint.NetPair } // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *IPVlanEndpoint) Attach(h hypervisor) error { +func (endpoint *IPVlanEndpoint) Attach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, NetDev) } // Detach for the virtual endpoint tears down the tap and bridge @@ -104,17 +105,17 @@ func (endpoint *IPVlanEndpoint) Detach(netNsCreated bool, netNsPath string) erro return nil } - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }) } // HotAttach for physical endpoint not supported yet -func (endpoint *IPVlanEndpoint) HotAttach(h hypervisor) error { +func (endpoint *IPVlanEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("IPVlanEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *IPVlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *IPVlanEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("IPVlanEndpoint does not support Hot detach") } diff --git a/virtcontainers/ipvlan_endpoint_test.go b/virtcontainers/hypervisor/ipvlan_endpoint_test.go similarity index 76% rename from virtcontainers/ipvlan_endpoint_test.go rename to virtcontainers/hypervisor/ipvlan_endpoint_test.go index 945427c80b..ed5bb7b64f 100644 --- a/virtcontainers/ipvlan_endpoint_test.go +++ b/virtcontainers/hypervisor/ipvlan_endpoint_test.go @@ -3,32 +3,34 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateIPVlanEndpoint(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &IPVlanEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-5", Name: "br5_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap5_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth5", HardAddr: macAddr.String(), }, - NetInterworkingModel: NetXConnectTCFilterModel, + NetInterworkingModel: types.NetXConnectTCFilterModel, }, EndpointType: IPVlanEndpointType, } diff --git a/virtcontainers/macvtap_endpoint.go b/virtcontainers/hypervisor/macvtap_endpoint.go similarity index 74% rename from virtcontainers/macvtap_endpoint.go rename to virtcontainers/hypervisor/macvtap_endpoint.go index d35d9c70cb..9f34d00b6f 100644 --- a/virtcontainers/macvtap_endpoint.go +++ b/virtcontainers/hypervisor/macvtap_endpoint.go @@ -3,23 +3,25 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "os" + + "github.com/kata-containers/runtime/virtcontainers/types" ) // MacvtapEndpoint represents a macvtap endpoint type MacvtapEndpoint struct { - EndpointProperties NetworkInfo + EndpointProperties types.NetworkInfo EndpointType EndpointType VMFds []*os.File VhostFds []*os.File PCIAddr string } -func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) { +func createMacvtapNetworkEndpoint(netInfo types.NetworkInfo) (*MacvtapEndpoint, error) { endpoint := &MacvtapEndpoint{ EndpointType: MacvtapEndpointType, EndpointProperties: netInfo, @@ -29,7 +31,7 @@ func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) } // Properties returns the properties of the macvtap interface. -func (endpoint *MacvtapEndpoint) Properties() NetworkInfo { +func (endpoint *MacvtapEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -49,28 +51,28 @@ func (endpoint *MacvtapEndpoint) Type() EndpointType { } // SetProperties sets the properties of the macvtap endpoint. -func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *MacvtapEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } -// Attach for macvtap endpoint passes macvtap device to the hypervisor. -func (endpoint *MacvtapEndpoint) Attach(h hypervisor) error { +// Attach for macvtap endpoint passes macvtap device to the hypervisir. +func (endpoint *MacvtapEndpoint) Attach(h Hypervisor) error { var err error - endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs)) + endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.Config().NumVCPUs)) if err != nil { return fmt.Errorf("Could not setup macvtap fds %s: %s", endpoint.EndpointProperties.Iface.Name, err) } - if !h.hypervisorConfig().DisableVhostNet { - vhostFds, err := createVhostFds(int(h.hypervisorConfig().NumVCPUs)) + if !h.Config().DisableVhostNet { + vhostFds, err := createVhostFds(int(h.Config().NumVCPUs)) if err != nil { return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.EndpointProperties.Iface.Name, err) } endpoint.VhostFds = vhostFds } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, NetDev) } // Detach for macvtap endpoint does nothing. @@ -79,12 +81,12 @@ func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) err } // HotAttach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor) error { +func (endpoint *MacvtapEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("MacvtapEndpoint does not support Hot attach") } // HotDetach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *MacvtapEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("MacvtapEndpoint does not support Hot detach") } @@ -99,6 +101,6 @@ func (endpoint *MacvtapEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *MacvtapEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } diff --git a/virtcontainers/macvtap_endpoint_test.go b/virtcontainers/hypervisor/macvtap_endpoint_test.go similarity index 78% rename from virtcontainers/macvtap_endpoint_test.go rename to virtcontainers/hypervisor/macvtap_endpoint_test.go index fbef06f30e..e0fb2df3f0 100644 --- a/virtcontainers/macvtap_endpoint_test.go +++ b/virtcontainers/hypervisor/macvtap_endpoint_test.go @@ -3,16 +3,18 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateMacvtapEndpoint(t *testing.T) { - netInfo := NetworkInfo{ - Iface: NetlinkIface{ + netInfo := types.NetworkInfo{ + Iface: types.NetlinkIface{ Type: "macvtap", }, } diff --git a/virtcontainers/hypervisor/mock.go b/virtcontainers/hypervisor/mock.go new file mode 100644 index 0000000000..a680887be0 --- /dev/null +++ b/virtcontainers/hypervisor/mock.go @@ -0,0 +1,107 @@ +// Copyright (c) 2016 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + "context" + "os" + + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" +) + +type mock struct { +} + +// NewMock returns a new mock hypervisor instance. +func NewMock() Hypervisor { + return &mock{} +} + +func (m *mock) Capabilities() types.Capabilities { + return types.Capabilities{} +} + +func (m *mock) Config() Config { + return Config{} +} + +func (m *mock) CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, store *store.VCStore) error { + err := hypervisorConfig.Valid() + if err != nil { + return err + } + + return nil +} + +func (m *mock) StartSandbox(timeout int) error { + return nil +} + +func (m *mock) StopSandbox() error { + return nil +} + +func (m *mock) PauseSandbox() error { + return nil +} + +func (m *mock) ResumeSandbox() error { + return nil +} + +func (m *mock) SaveSandbox() error { + return nil +} + +func (m *mock) AddDevice(devInfo interface{}, devType Device) error { + return nil +} + +func (m *mock) HotplugAddDevice(devInfo interface{}, devType Device) (interface{}, error) { + switch devType { + case CPUDev: + return devInfo.(uint32), nil + case MemoryDev: + memdev := devInfo.(*MemoryDevice) + return memdev.SizeMB, nil + } + return nil, nil +} + +func (m *mock) HotplugRemoveDevice(devInfo interface{}, devType Device) (interface{}, error) { + switch devType { + case CPUDev: + return devInfo.(uint32), nil + case MemoryDev: + return 0, nil + } + return nil, nil +} + +func (m *mock) GetSandboxConsole(sandboxID string) (string, error) { + return "", nil +} + +func (m *mock) ResizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { + return 0, nil +} +func (m *mock) ResizeVCPUs(cpus uint32) (uint32, uint32, error) { + return 0, 0, nil +} + +func (m *mock) Disconnect() { +} + +func (m *mock) GetThreadIDs() (*ThreadIDs, error) { + vcpus := []int{os.Getpid()} + return &ThreadIDs{vcpus}, nil +} + +func (m *mock) Cleanup() error { + return nil +} diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/hypervisor/mock_test.go similarity index 52% rename from virtcontainers/mock_hypervisor_test.go rename to virtcontainers/hypervisor/mock_test.go index 34885c45f9..2d156c187c 100644 --- a/virtcontainers/mock_hypervisor_test.go +++ b/virtcontainers/hypervisor/mock_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" @@ -12,67 +12,63 @@ import ( ) func TestMockHypervisorCreateSandbox(t *testing.T) { - var m *mockHypervisor - - sandbox := &Sandbox{ - config: &SandboxConfig{ - ID: "mock_sandbox", - HypervisorConfig: HypervisorConfig{ - KernelPath: "", - ImagePath: "", - HypervisorPath: "", - }, - }, + var m *mock + sandboxID := "mock_sandbox" + + hypervisorConfig := Config{ + KernelPath: "", + ImagePath: "", + HypervisorPath: "", } ctx := context.Background() // wrong config - if err := m.createSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err == nil { + if err := m.CreateSandbox(ctx, sandboxID, &hypervisorConfig, nil); err == nil { t.Fatal() } - sandbox.config.HypervisorConfig = HypervisorConfig{ + validHypervisorConfig := Config{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - if err := m.createSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err != nil { + if err := m.CreateSandbox(ctx, sandboxID, &validHypervisorConfig, nil); err != nil { t.Fatal(err) } } func TestMockHypervisorStartSandbox(t *testing.T) { - var m *mockHypervisor + var m *mock - if err := m.startSandbox(vmStartTimeout); err != nil { + if err := m.StartSandbox(10); err != nil { t.Fatal(err) } } func TestMockHypervisorStopSandbox(t *testing.T) { - var m *mockHypervisor + var m *mock - if err := m.stopSandbox(); err != nil { + if err := m.StopSandbox(); err != nil { t.Fatal(err) } } func TestMockHypervisorAddDevice(t *testing.T) { - var m *mockHypervisor + var m *mock - if err := m.addDevice(nil, imgDev); err != nil { + if err := m.AddDevice(nil, ImgDev); err != nil { t.Fatal(err) } } func TestMockHypervisorGetSandboxConsole(t *testing.T) { - var m *mockHypervisor + var m *mock expected := "" - result, err := m.getSandboxConsole("testSandboxID") + result, err := m.GetSandboxConsole("testSandboxID") if err != nil { t.Fatal(err) } @@ -83,15 +79,15 @@ func TestMockHypervisorGetSandboxConsole(t *testing.T) { } func TestMockHypervisorSaveSandbox(t *testing.T) { - var m *mockHypervisor + var m *mock - if err := m.saveSandbox(); err != nil { + if err := m.SaveSandbox(); err != nil { t.Fatal(err) } } func TestMockHypervisorDisconnect(t *testing.T) { - var m *mockHypervisor + var m *mock - m.disconnect() + m.Disconnect() } diff --git a/virtcontainers/hypervisor/network.go b/virtcontainers/hypervisor/network.go new file mode 100644 index 0000000000..a60495615f --- /dev/null +++ b/virtcontainers/hypervisor/network.go @@ -0,0 +1,387 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + cryptoRand "crypto/rand" + "fmt" + "math/rand" + "net" + "os" + "runtime" + "time" + + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +func networkLogger() *logrus.Entry { + return logrus.WithField("source", "virtcontainers/hypervisor").WithField("subsystem", "network") +} + +// DoNetNS is free from any call to a go routine, and it calls +// into runtime.LockOSThread(), meaning it won't be executed in a +// different thread than the one expected by the caller. +func DoNetNS(netNSPath string, cb func(ns.NetNS) error) error { + // if netNSPath is empty, the callback function will be run in the current network namespace. + // So skip the whole function, just call cb(). cb() needs a NetNS as arg but ignored, give it a fake one. + if netNSPath == "" { + var netNs ns.NetNS + return cb(netNs) + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + currentNS, err := ns.GetCurrentNS() + if err != nil { + return err + } + defer currentNS.Close() + + targetNS, err := ns.GetNS(netNSPath) + if err != nil { + return err + } + + if err := targetNS.Set(); err != nil { + return err + } + defer currentNS.Set() + + return cb(targetNS) +} + +func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, queues int) (netlink.Link, []*os.File, error) { + var newLink netlink.Link + var fds []*os.File + + switch expectedLink.Type() { + case (&netlink.Bridge{}).Type(): + newLink = &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{Name: name}, + MulticastSnooping: expectedLink.(*netlink.Bridge).MulticastSnooping, + } + case (&netlink.Tuntap{}).Type(): + flags := netlink.TUNTAP_VNET_HDR + if queues > 0 { + flags |= netlink.TUNTAP_MULTI_QUEUE_DEFAULTS + } + newLink = &netlink.Tuntap{ + LinkAttrs: netlink.LinkAttrs{Name: name}, + Mode: netlink.TUNTAP_MODE_TAP, + Queues: queues, + Flags: flags, + } + case (&netlink.Macvtap{}).Type(): + qlen := expectedLink.Attrs().TxQLen + if qlen <= 0 { + qlen = types.DefaultQlen + } + newLink = &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + Mode: netlink.MACVLAN_MODE_BRIDGE, + LinkAttrs: netlink.LinkAttrs{ + Index: expectedLink.Attrs().Index, + Name: name, + TxQLen: qlen, + ParentIndex: expectedLink.Attrs().ParentIndex, + }, + }, + } + default: + return nil, fds, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) + } + + if err := netHandle.LinkAdd(newLink); err != nil { + return nil, fds, fmt.Errorf("LinkAdd() failed for %s name %s: %s", expectedLink.Type(), name, err) + } + + tuntapLink, ok := newLink.(*netlink.Tuntap) + if ok { + fds = tuntapLink.Fds + } + + newLink, err := getLinkByName(netHandle, name, expectedLink) + return newLink, fds, err +} + +const hostLinkOffset = 8192 // Host should not have more than 8k interfaces +const linkRange = 0xFFFF // This will allow upto 2^16 containers +const linkRetries = 128 // The numbers of time we try to find a non conflicting index +const macvtapWorkaround = true + +func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, queues int) (taplink netlink.Link, err error) { + if !macvtapWorkaround { + taplink, _, err = createLink(netHandle, name, link, queues) + return + } + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for i := 0; i < linkRetries; i++ { + index := hostLinkOffset + (r.Int() & linkRange) + link.Attrs().Index = index + taplink, _, err = createLink(netHandle, name, link, queues) + if err == nil { + break + } + } + + return +} + +func clearIPs(link netlink.Link, addrs []netlink.Addr) error { + for _, addr := range addrs { + if err := netlink.AddrDel(link, &addr); err != nil { + return err + } + } + return nil +} + +func setIPs(link netlink.Link, addrs []netlink.Addr) error { + for _, addr := range addrs { + if err := netlink.AddrAdd(link, &addr); err != nil { + return err + } + } + return nil +} + +func createFds(device string, numFds int) ([]*os.File, error) { + fds := make([]*os.File, numFds) + + for i := 0; i < numFds; i++ { + f, err := os.OpenFile(device, os.O_RDWR, types.DefaultFilePerms) + if err != nil { + utils.CleanupFds(fds, i) + return nil, err + } + fds[i] = f + } + return fds, nil +} + +func createMacvtapFds(linkIndex int, queues int) ([]*os.File, error) { + tapDev := fmt.Sprintf("/dev/tap%d", linkIndex) + return createFds(tapDev, queues) +} + +func createVhostFds(numFds int) ([]*os.File, error) { + vhostDev := "/dev/vhost-net" + return createFds(vhostDev, numFds) +} + +func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.Link) (netlink.Link, error) { + link, err := netHandle.LinkByName(name) + if err != nil { + return nil, fmt.Errorf("LinkByName() failed for %s name %s: %s", expectedLink.Type(), name, err) + } + + switch expectedLink.Type() { + case (&netlink.Bridge{}).Type(): + if l, ok := link.(*netlink.Bridge); ok { + return l, nil + } + case (&netlink.Tuntap{}).Type(): + if l, ok := link.(*netlink.GenericLink); ok { + return l, nil + } + case (&netlink.Veth{}).Type(): + if l, ok := link.(*netlink.Veth); ok { + return l, nil + } + case (&netlink.Macvtap{}).Type(): + if l, ok := link.(*netlink.Macvtap); ok { + return l, nil + } + case (&netlink.Macvlan{}).Type(): + if l, ok := link.(*netlink.Macvlan); ok { + return l, nil + } + case (&netlink.IPVlan{}).Type(): + if l, ok := link.(*netlink.IPVlan); ok { + return l, nil + } + default: + return nil, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) + } + + return nil, fmt.Errorf("Incorrect link type %s, expecting %s", link.Type(), expectedLink.Type()) +} + +// addQdiscIngress creates a new qdisc for nwtwork interface with the specified network index +// on "ingress". qdiscs normally don't work on ingress so this is really a special qdisc +// that you can consider an "alternate root" for inbound packets. +// Handle for ingress qdisc defaults to "ffff:" +// +// This is equivalent to calling `tc qdisc add dev eth0 ingress` +func addQdiscIngress(index int) error { + qdisc := &netlink.Ingress{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: index, + Parent: netlink.HANDLE_INGRESS, + }, + } + + err := netlink.QdiscAdd(qdisc) + if err != nil { + return fmt.Errorf("Failed to add qdisc for network index %d : %s", index, err) + } + + return nil +} + +// addRedirectTCFilter adds a tc filter for device with index "sourceIndex". +// All traffic for interface with index "sourceIndex" is redirected to interface with +// index "destIndex" +// +// This is equivalent to calling: +// `tc filter add dev source parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev dest` +func addRedirectTCFilter(sourceIndex, destIndex int) error { + filter := &netlink.U32{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: sourceIndex, + Parent: netlink.MakeHandle(0xffff, 0), + Protocol: unix.ETH_P_ALL, + }, + Actions: []netlink.Action{ + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: destIndex, + }, + }, + } + + if err := netlink.FilterAdd(filter); err != nil { + return fmt.Errorf("Failed to add filter for index %d : %s", sourceIndex, err) + } + + return nil +} + +// removeRedirectTCFilter removes all tc u32 filters created on ingress qdisc for "link". +func removeRedirectTCFilter(link netlink.Link) error { + if link == nil { + return nil + } + + // Handle 0xffff is used for ingress + filters, err := netlink.FilterList(link, netlink.MakeHandle(0xffff, 0)) + if err != nil { + return err + } + + for _, f := range filters { + u32, ok := f.(*netlink.U32) + + if !ok { + continue + } + + if err := netlink.FilterDel(u32); err != nil { + return err + } + } + return nil +} + +// removeQdiscIngress removes the ingress qdisc previously created on "link". +func removeQdiscIngress(link netlink.Link) error { + if link == nil { + return nil + } + + qdiscs, err := netlink.QdiscList(link) + if err != nil { + return err + } + + for _, qdisc := range qdiscs { + ingress, ok := qdisc.(*netlink.Ingress) + if !ok { + continue + } + + if err := netlink.QdiscDel(ingress); err != nil { + return err + } + } + return nil +} + +func generateRandomPrivateMacAddr() (string, error) { + buf := make([]byte, 6) + _, err := cryptoRand.Read(buf) + if err != nil { + return "", err + } + + // Set the local bit for local addresses + // Addresses in this range are local mac addresses: + // x2-xx-xx-xx-xx-xx , x6-xx-xx-xx-xx-xx , xA-xx-xx-xx-xx-xx , xE-xx-xx-xx-xx-xx + buf[0] = (buf[0] | 2) & 0xfe + + hardAddr := net.HardwareAddr(buf) + return hardAddr.String(), nil +} + +func createNetworkInterfacePair(idx int, ifName string, interworkingModel types.NetInterworkingModel) (types.NetworkInterfacePair, error) { + uniqueID := uuid.Generate().String() + + randomMacAddr, err := generateRandomPrivateMacAddr() + if err != nil { + return types.NetworkInterfacePair{}, fmt.Errorf("Could not generate random mac address: %s", err) + } + + netPair := types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ + ID: uniqueID, + Name: fmt.Sprintf("br%d_kata", idx), + TAPIface: types.NetworkInterface{ + Name: fmt.Sprintf("tap%d_kata", idx), + }, + }, + VirtIface: types.NetworkInterface{ + Name: fmt.Sprintf("eth%d", idx), + HardAddr: randomMacAddr, + }, + NetInterworkingModel: interworkingModel, + } + + return netPair, nil +} + +func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (types.NetworkInfo, error) { + addrs, err := handle.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return types.NetworkInfo{}, err + } + + routes, err := handle.RouteList(link, netlink.FAMILY_ALL) + if err != nil { + return types.NetworkInfo{}, err + } + + return types.NetworkInfo{ + Iface: types.NetlinkIface{ + LinkAttrs: *(link.Attrs()), + Type: link.Type(), + }, + Addrs: addrs, + Routes: routes, + }, nil +} diff --git a/virtcontainers/hypervisor/network_test.go b/virtcontainers/hypervisor/network_test.go new file mode 100644 index 0000000000..a1bc57af3b --- /dev/null +++ b/virtcontainers/hypervisor/network_test.go @@ -0,0 +1,166 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisor + +import ( + "net" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + + "github.com/kata-containers/runtime/virtcontainers/types" +) + +func TestGenerateRandomPrivateMacAdd(t *testing.T) { + assert := assert.New(t) + + addr1, err := generateRandomPrivateMacAddr() + assert.NoError(err) + + _, err = net.ParseMAC(addr1) + assert.NoError(err) + + addr2, err := generateRandomPrivateMacAddr() + assert.NoError(err) + + _, err = net.ParseMAC(addr2) + assert.NoError(err) + + assert.NotEqual(addr1, addr2) +} + +func TestCreateGetBridgeLink(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + brName := "testbr0" + brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) + assert.NoError(err) + assert.NotNil(brLink) + + brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) + assert.NoError(err) + + err = netHandle.LinkDel(brLink) + assert.NoError(err) +} + +func TestCreateGetTunTapLink(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + tapName := "testtap0" + tapLink, fds, err := createLink(netHandle, tapName, &netlink.Tuntap{}, 1) + assert.NoError(err) + assert.NotNil(tapLink) + assert.NotZero(len(fds)) + + tapLink, err = getLinkByName(netHandle, tapName, &netlink.Tuntap{}) + assert.NoError(err) + + err = netHandle.LinkDel(tapLink) + assert.NoError(err) +} + +func TestCreateMacVtap(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + brName := "testbr0" + brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) + assert.NoError(err) + + attrs := brLink.Attrs() + + mcLink := &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + LinkAttrs: netlink.LinkAttrs{ + TxQLen: attrs.TxQLen, + ParentIndex: attrs.Index, + }, + }, + } + + macvtapName := "testmc0" + _, err = createMacVtap(netHandle, macvtapName, mcLink, 1) + assert.NoError(err) + + macvtapLink, err := getLinkByName(netHandle, macvtapName, &netlink.Macvtap{}) + assert.NoError(err) + + err = netHandle.LinkDel(macvtapLink) + assert.NoError(err) + + brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) + assert.NoError(err) + + err = netHandle.LinkDel(brLink) + assert.NoError(err) +} + +func TestTcRedirectNetwork(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + assert.NoError(err) + defer netHandle.Delete() + + // Create a test veth interface. + vethName := "foo" + veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: vethName, TxQLen: 200, MTU: 1400}, PeerName: "bar"} + + err = netlink.LinkAdd(veth) + assert.NoError(err) + + endpoint, err := createVethNetworkEndpoint(1, vethName, types.NetXConnectTCFilterModel) + assert.NoError(err) + + link, err := netlink.LinkByName(vethName) + assert.NoError(err) + + err = netHandle.LinkSetUp(link) + assert.NoError(err) + + err = setupTCFiltering(endpoint, 1, true) + assert.NoError(err) + + err = removeTCFiltering(endpoint) + assert.NoError(err) + + // Remove the veth created for testing. + err = netHandle.LinkDel(link) + assert.NoError(err) +} diff --git a/virtcontainers/physical_endpoint.go b/virtcontainers/hypervisor/physical_endpoint.go similarity index 88% rename from virtcontainers/physical_endpoint.go rename to virtcontainers/hypervisor/physical_endpoint.go index 989233f158..478a5f0ae4 100644 --- a/virtcontainers/physical_endpoint.go +++ b/virtcontainers/hypervisor/physical_endpoint.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -14,6 +14,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/safchain/ethtool" ) @@ -21,7 +22,7 @@ import ( type PhysicalEndpoint struct { IfaceName string HardAddr string - EndpointProperties NetworkInfo + EndpointProperties types.NetworkInfo EndpointType EndpointType BDF string Driver string @@ -30,7 +31,7 @@ type PhysicalEndpoint struct { } // Properties returns the properties of the physical interface. -func (endpoint *PhysicalEndpoint) Properties() NetworkInfo { +func (endpoint *PhysicalEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -60,18 +61,18 @@ func (endpoint *PhysicalEndpoint) SetPciAddr(pciAddr string) { } // SetProperties sets the properties of the physical endpoint. -func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *PhysicalEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } // NetworkPair returns the network pair of the endpoint. -func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *PhysicalEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } // Attach for physical endpoint binds the physical network interface to // vfio-pci and adds device to the hypervisor with vfio-passthrough. -func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { +func (endpoint *PhysicalEndpoint) Attach(h Hypervisor) error { // Unbind physical interface from host driver and bind to vfio // so that it can be passed to qemu. if err := bindNICToVFIO(endpoint); err != nil { @@ -83,7 +84,7 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { BDF: endpoint.BDF, } - return h.addDevice(d, vfioDev) + return h.AddDevice(d, VfioDev) } // Detach for physical endpoint unbinds the physical network interface from vfio-pci @@ -99,12 +100,12 @@ func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) er } // HotAttach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error { +func (endpoint *PhysicalEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("PhysicalEndpoint does not support Hot attach") } // HotDetach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *PhysicalEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("PhysicalEndpoint does not support Hot detach") } @@ -137,7 +138,7 @@ func isPhysicalIface(ifaceName string) (bool, error) { var sysPCIDevicesPath = "/sys/bus/pci/devices" -func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { +func createPhysicalEndpoint(netInfo types.NetworkInfo) (*PhysicalEndpoint, error) { // Get ethtool handle to derive driver and bus ethHandle, err := ethtool.NewEthtool() if err != nil { diff --git a/virtcontainers/physical_endpoint_test.go b/virtcontainers/hypervisor/physical_endpoint_test.go similarity index 93% rename from virtcontainers/physical_endpoint_test.go rename to virtcontainers/hypervisor/physical_endpoint_test.go index f22a8f1520..c51a194d6b 100644 --- a/virtcontainers/physical_endpoint_test.go +++ b/virtcontainers/hypervisor/physical_endpoint_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" @@ -23,7 +23,7 @@ func TestPhysicalEndpoint_HotAttach(t *testing.T) { HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), } - h := &mockHypervisor{} + h := &mock{} err := v.HotAttach(h) assert.Error(err) @@ -36,7 +36,7 @@ func TestPhysicalEndpoint_HotDetach(t *testing.T) { HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), } - h := &mockHypervisor{} + h := &mock{} err := v.HotDetach(h, true, "") assert.Error(err) @@ -88,7 +88,7 @@ func TestIsPhysicalIface(t *testing.T) { } var isPhysical bool - err = doNetNS(n.Path(), func(_ ns.NetNS) error { + err = DoNetNS(n.Path(), func(_ ns.NetNS) error { var err error isPhysical, err = isPhysicalIface(testNetIface) return err diff --git a/virtcontainers/qemu.go b/virtcontainers/hypervisor/qemu.go similarity index 89% rename from virtcontainers/qemu.go rename to virtcontainers/hypervisor/qemu.go index 2220108385..f5cd2a76d1 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/hypervisor/qemu.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" @@ -57,13 +57,13 @@ type QemuState struct { HotplugVFIOOnRootBus bool } -// qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. +// qemu is an Hypervisor interface implementation for the Linux qemu type qemu struct { id string store *store.VCStore - config HypervisorConfig + config Config qmpMonitorCh qmpChannel @@ -80,6 +80,7 @@ type qemu struct { ctx context.Context nvdimmCount int + maxVCPUs uint32 } const ( @@ -103,18 +104,13 @@ var defaultKernelParameters = []Param{ {"panic", "1"}, } -const ( - addDevice operation = iota - removeDevice -) - type qmpLogger struct { logger *logrus.Entry } func newQMPLogger() qmpLogger { return qmpLogger{ - logger: virtLog.WithField("subsystem", "qmp"), + logger: logrus.WithField("source", "virtcontainers/hypervisor").WithField("subsystem", "qmp"), } } @@ -140,7 +136,7 @@ func (l qmpLogger) Errorf(format string, v ...interface{}) { // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemu) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu") + return logrus.WithField("source", "virtcontainers/hypervisor").WithField("subsystem", "qemu") } func (q *qemu) kernelParameters() string { @@ -151,7 +147,7 @@ func (q *qemu) kernelParameters() string { params = append(params, defaultKernelParameters...) // set the maximum number of vCPUs - params = append(params, Param{"nr_cpus", fmt.Sprintf("%d", q.config.DefaultMaxVCPUs)}) + params = append(params, Param{"nr_cpus", fmt.Sprintf("%d", q.maxVCPUs)}) // add the params specified by the provided config. As the kernel // honours the last parameter value set and since the config-provided @@ -163,15 +159,15 @@ func (q *qemu) kernelParameters() string { return strings.Join(paramsStr, " ") } -// Adds all capabilities supported by qemu implementation of hypervisor interface -func (q *qemu) capabilities() types.Capabilities { +// Capabilities add all supported capabilities by QEMU +func (q *qemu) Capabilities() types.Capabilities { span, _ := q.trace("capabilities") defer span.Finish() return q.arch.capabilities() } -func (q *qemu) hypervisorConfig() HypervisorConfig { +func (q *qemu) Config() Config { return q.config } @@ -211,12 +207,11 @@ func (q *qemu) trace(name string) (opentracing.Span, context.Context) { } // setup sets the Qemu structure up. -func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (q *qemu) setup(id string, hypervisorConfig *Config, vcStore *store.VCStore) error { span, _ := q.trace("setup") defer span.Finish() - err := hypervisorConfig.valid() - if err != nil { + if err := hypervisorConfig.Valid(); err != nil { return err } @@ -225,6 +220,12 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto q.config = *hypervisorConfig q.arch = newQemuArch(q.config) + if q.config.DefaultMaxVCPUs == 0 { + q.maxVCPUs = MaxQemuVCPUs() + } else { + q.maxVCPUs = q.config.DefaultMaxVCPUs + } + initrdPath, err := q.config.InitrdAssetPath() if err != nil { return err @@ -259,7 +260,7 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto } } - nested, err := RunningOnVMM(procCPUInfo) + nested, err := RunningOnVMM(ProcCPUInfo) if err != nil { return err } @@ -282,11 +283,11 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto } func (q *qemu) cpuTopology() govmmQemu.SMP { - return q.arch.cpuTopology(q.config.NumVCPUs, q.config.DefaultMaxVCPUs) + return q.arch.cpuTopology(q.config.NumVCPUs, q.maxVCPUs) } func (q *qemu) hostMemMB() (uint64, error) { - hostMemKb, err := getHostMemorySizeKb(procMemInfo) + hostMemKb, err := GetHostMemorySizeKb(ProcMemInfo) if err != nil { return 0, fmt.Errorf("Unable to read memory info: %s", err) } @@ -369,7 +370,7 @@ func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) { func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.IOThread, error) { var devices []govmmQemu.Device - console, err := q.getSandboxConsole(q.id) + console, err := q.GetSandboxConsole(q.id) if err != nil { return nil, nil, err } @@ -416,8 +417,8 @@ func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) g return incoming } -// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. -func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { +// CreateSandbox is the Hypervisor sandbox creation implementation for govmmQemu. +func (q *qemu) CreateSandbox(ctx context.Context, id string, hypervisorConfig *Config, store *store.VCStore) error { // Save the tracing context q.ctx = ctx @@ -535,8 +536,8 @@ func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *H return nil } -// startSandbox will start the Sandbox's VM. -func (q *qemu) startSandbox(timeout int) error { +// StartSandbox will start the Sandbox's VM. +func (q *qemu) StartSandbox(timeout int) error { span, _ := q.trace("startSandbox") defer span.Finish() @@ -638,7 +639,7 @@ func (q *qemu) waitSandbox(timeout int) error { } // stopSandbox will stop the Sandbox's VM. -func (q *qemu) stopSandbox() error { +func (q *qemu) StopSandbox() error { span, _ := q.trace("stopSandbox") defer span.Finish() @@ -768,7 +769,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error { return err } -func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) error { +func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op Operation, devID string) error { var err error if q.config.BlockDeviceDriver == config.Nvdimm { @@ -831,7 +832,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev return nil } -func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error { +func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op Operation) error { err := q.qmpSetup() if err != nil { return err @@ -839,7 +840,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error devID := "virtio-" + drive.ID - if op == addDevice { + if op == AddDevice { err = q.hotplugAddBlockDevice(drive, op, devID) } else { if q.config.BlockDeviceDriver == config.VirtioBlock { @@ -860,7 +861,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error return err } -func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { +func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op Operation) error { err := q.qmpSetup() if err != nil { return err @@ -868,7 +869,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { devID := device.ID - if op == addDevice { + if op == AddDevice { // In case HotplugVFIOOnRootBus is true, devices are hotplugged on the root bus // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. @@ -933,12 +934,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) } -func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error { +func (q *qemu) hotplugNetDevice(endpoint Endpoint, op Operation) error { err := q.qmpSetup() if err != nil { return err } - var tap TapInterface + var tap types.TapInterface devID := "virtio-" + tap.ID switch endpoint.Type() { @@ -952,7 +953,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error { return fmt.Errorf("this endpoint is not supported") } - if op == addDevice { + if op == AddDevice { if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { return err @@ -990,21 +991,21 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error { return nil } -func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) { +func (q *qemu) hotplugDevice(devInfo interface{}, devType Device, op Operation) (interface{}, error) { switch devType { - case blockDev: + case BlockDev: drive := devInfo.(*config.BlockDrive) return nil, q.hotplugBlockDevice(drive, op) - case cpuDev: + case CPUDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) - case vfioDev: + case VfioDev: device := devInfo.(*config.VFIODev) return nil, q.hotplugVFIODevice(device, op) - case memoryDev: - memdev := devInfo.(*memoryDevice) + case MemoryDev: + memdev := devInfo.(*MemoryDevice) return q.hotplugMemory(memdev, op) - case netDev: + case NetDev: device := devInfo.(Endpoint) return nil, q.hotplugNetDevice(device, op) default: @@ -1012,11 +1013,11 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati } } -func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (q *qemu) HotplugAddDevice(devInfo interface{}, devType Device) (interface{}, error) { span, _ := q.trace("hotplugAddDevice") defer span.Finish() - data, err := q.hotplugDevice(devInfo, devType, addDevice) + data, err := q.hotplugDevice(devInfo, devType, AddDevice) if err != nil { return data, err } @@ -1024,11 +1025,11 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf return data, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { +func (q *qemu) HotplugRemoveDevice(devInfo interface{}, devType Device) (interface{}, error) { span, _ := q.trace("hotplugRemoveDevice") defer span.Finish() - data, err := q.hotplugDevice(devInfo, devType, removeDevice) + data, err := q.hotplugDevice(devInfo, devType, RemoveDevice) if err != nil { return data, err } @@ -1036,7 +1037,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int return data, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) { +func (q *qemu) hotplugCPUs(vcpus uint32, op Operation) (uint32, error) { if vcpus == 0 { q.Logger().Warnf("cannot hotplug 0 vCPUs") return 0, nil @@ -1047,7 +1048,7 @@ func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) { return 0, err } - if op == addDevice { + if op == AddDevice { return q.hotplugAddCPUs(vcpus) } @@ -1060,15 +1061,15 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { // Don't fail if the number of max vCPUs is exceeded, log a warning and hot add the vCPUs needed // to reach out max vCPUs - if currentVCPUs+amount > q.config.DefaultMaxVCPUs { + if currentVCPUs+amount > q.maxVCPUs { q.Logger().Warnf("Cannot hotplug %d CPUs, currently this SB has %d CPUs and the maximum amount of CPUs is %d", - amount, currentVCPUs, q.config.DefaultMaxVCPUs) - amount = q.config.DefaultMaxVCPUs - currentVCPUs + amount, currentVCPUs, q.maxVCPUs) + amount = q.maxVCPUs - currentVCPUs } if amount == 0 { // Don't fail if no more vCPUs can be added, since cgroups still can be updated - q.Logger().Warnf("maximum number of vCPUs '%d' has been reached", q.config.DefaultMaxVCPUs) + q.Logger().Warnf("maximum number of vCPUs '%d' has been reached", q.maxVCPUs) return 0, nil } @@ -1149,17 +1150,17 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) { return amount, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) { +func (q *qemu) hotplugMemory(memDev *MemoryDevice, op Operation) (int, error) { if !q.arch.supportGuestMemoryHotplug() { return 0, fmt.Errorf("guest memory hotplug not supported") } - if memDev.sizeMB < 0 { - return 0, fmt.Errorf("cannot hotplug negative size (%d) memory", memDev.sizeMB) + if memDev.SizeMB < 0 { + return 0, fmt.Errorf("cannot hotplug negative size (%d) memory", memDev.SizeMB) } memLog := q.Logger().WithField("hotplug", "memory") - memLog.WithField("hotplug-memory-mb", memDev.sizeMB).Debug("requested memory hotplug") + memLog.WithField("hotplug-memory-mb", memDev.SizeMB).Debug("requested memory hotplug") err := q.qmpSetup() if err != nil { return 0, err @@ -1167,29 +1168,29 @@ func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) { currentMemory := int(q.config.MemorySize) + q.state.HotpluggedMemory - if memDev.sizeMB == 0 { + if memDev.SizeMB == 0 { memLog.Debug("hotplug is not required") return 0, nil } switch op { - case removeDevice: - memLog.WithField("operation", "remove").Debugf("Requested to remove memory: %d MB", memDev.sizeMB) + case RemoveDevice: + memLog.WithField("Operation", "remove").Debugf("Requested to remove memory: %d MB", memDev.SizeMB) // Dont fail but warn that this is not supported. memLog.Warn("hot-remove VM memory not supported") return 0, nil - case addDevice: - memLog.WithField("operation", "add").Debugf("Requested to add memory: %d MB", memDev.sizeMB) + case AddDevice: + memLog.WithField("Operation", "add").Debugf("Requested to add memory: %d MB", memDev.SizeMB) maxMem, err := q.hostMemMB() if err != nil { return 0, err } // Don't exceed the maximum amount of memory - if currentMemory+memDev.sizeMB > int(maxMem) { + if currentMemory+memDev.SizeMB > int(maxMem) { // Fixme: return a typed error return 0, fmt.Errorf("Unable to hotplug %d MiB memory, the SB has %d MiB and the maximum amount is %d MiB", - memDev.sizeMB, currentMemory, maxMem) + memDev.SizeMB, currentMemory, maxMem) } memoryAdded, err := q.hotplugAddMemory(memDev) if err != nil { @@ -1202,33 +1203,33 @@ func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) { } -func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { +func (q *qemu) hotplugAddMemory(memDev *MemoryDevice) (int, error) { memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx) if err != nil { return 0, fmt.Errorf("failed to query memory devices: %v", err) } if len(memoryDevices) != 0 { - memDev.slot = memoryDevices[len(memoryDevices)-1].Data.Slot + 1 + memDev.Slot = memoryDevices[len(memoryDevices)-1].Data.Slot + 1 } - err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, "memory-backend-ram", "mem"+strconv.Itoa(memDev.slot), "", memDev.sizeMB) + err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, "memory-backend-ram", "mem"+strconv.Itoa(memDev.Slot), "", memDev.SizeMB) if err != nil { q.Logger().WithError(err).Error("hotplug memory") return 0, err } - q.state.HotpluggedMemory += memDev.sizeMB - return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state) + q.state.HotpluggedMemory += memDev.SizeMB + return memDev.SizeMB, q.store.Store(store.Hypervisor, q.state) } -func (q *qemu) pauseSandbox() error { +func (q *qemu) PauseSandbox() error { span, _ := q.trace("pauseSandbox") defer span.Finish() return q.togglePauseSandbox(true) } -func (q *qemu) resumeSandbox() error { +func (q *qemu) ResumeSandbox() error { span, _ := q.trace("resumeSandbox") defer span.Finish() @@ -1236,7 +1237,7 @@ func (q *qemu) resumeSandbox() error { } // addDevice will add extra devices to Qemu command line. -func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { +func (q *qemu) AddDevice(devInfo interface{}, devType Device) error { var err error span, _ := q.trace("addDevice") defer span.Finish() @@ -1246,8 +1247,8 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { q.qemuConfig.Devices = q.arch.append9PVolume(q.qemuConfig.Devices, v) case types.Socket: q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v) - case kataVSOCK: - q.fds = append(q.fds, v.vhostFd) + case types.VSOCK: + q.fds = append(q.fds, v.VHostFd) q.qemuConfig.Devices = q.arch.appendVSockPCI(q.qemuConfig.Devices, v) case Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) @@ -1264,16 +1265,16 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { return err } -// getSandboxConsole builds the path of the console where we can read +// GetSandboxConsole builds the path of the console where we can read // logs coming from the sandbox. -func (q *qemu) getSandboxConsole(id string) (string, error) { +func (q *qemu) GetSandboxConsole(id string) (string, error) { span, _ := q.trace("getSandboxConsole") defer span.Finish() return utils.BuildSocketPath(store.RunVMStoragePath, id, consoleSocket) } -func (q *qemu) saveSandbox() error { +func (q *qemu) SaveSandbox() error { q.Logger().Info("save sandbox") err := q.qmpSetup() @@ -1328,14 +1329,14 @@ func (q *qemu) saveSandbox() error { return nil } -func (q *qemu) disconnect() { +func (q *qemu) Disconnect() { span, _ := q.trace("disconnect") defer span.Finish() q.qmpShutdown() } -// resizeMemory get a request to update the VM memory to reqMemMB +// ResizeMemory get a request to update the VM memory to reqMemMB // Memory update is managed with two approaches // Add memory to VM: // When memory is required to be added we hotplug memory @@ -1346,7 +1347,7 @@ func (q *qemu) disconnect() { // the memory to remove has to be at least the size of one slot. // To return memory back we are resizing the VM memory balloon. // A longer term solution is evaluate solutions like virtio-mem -func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { +func (q *qemu) ResizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory) err := q.qmpSetup() @@ -1362,10 +1363,10 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, return currentMemory, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotplugMB), + addMemDevice := &MemoryDevice{ + SizeMB: int(memHotplugMB), } - data, err := q.hotplugAddDevice(addMemDevice, memoryDev) + data, err := q.HotplugAddDevice(addMemDevice, MemoryDev) if err != nil { return currentMemory, err } @@ -1382,10 +1383,10 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, return currentMemory, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotunplugMB), + addMemDevice := &MemoryDevice{ + SizeMB: int(memHotunplugMB), } - data, err := q.hotplugRemoveDevice(addMemDevice, memoryDev) + data, err := q.HotplugRemoveDevice(addMemDevice, MemoryDev) if err != nil { return currentMemory, err } @@ -1488,7 +1489,7 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff return memory } -func (q *qemu) getThreadIDs() (*threadIDs, error) { +func (q *qemu) GetThreadIDs() (*ThreadIDs, error) { span, _ := q.trace("getThreadIDs") defer span.Finish() @@ -1503,10 +1504,10 @@ func (q *qemu) getThreadIDs() (*threadIDs, error) { return nil, err } - var tid threadIDs + var tid ThreadIDs for _, i := range cpuInfos { if i.ThreadID > 0 { - tid.vcpus = append(tid.vcpus, i.ThreadID) + tid.VCPUs = append(tid.VCPUs, i.ThreadID) } } return &tid, nil @@ -1521,7 +1522,7 @@ func calcHotplugMemMiBSize(mem uint32, memorySectionSizeMB uint32) (uint32, erro return uint32(math.Ceil(float64(mem)/float64(memorySectionSizeMB))) * memorySectionSizeMB, nil } -func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { +func (q *qemu) ResizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { currentVCPUs = q.config.NumVCPUs + uint32(len(q.state.HotpluggedVCPUs)) newVCPUs = currentVCPUs @@ -1529,7 +1530,7 @@ func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 case currentVCPUs < reqVCPUs: //hotplug addCPUs := reqVCPUs - currentVCPUs - data, err := q.hotplugAddDevice(addCPUs, cpuDev) + data, err := q.HotplugAddDevice(addCPUs, CPUDev) if err != nil { return currentVCPUs, newVCPUs, err } @@ -1541,7 +1542,7 @@ func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 case currentVCPUs > reqVCPUs: //hotunplug removeCPUs := currentVCPUs - reqVCPUs - data, err := q.hotplugRemoveDevice(removeCPUs, cpuDev) + data, err := q.HotplugRemoveDevice(removeCPUs, CPUDev) if err != nil { return currentVCPUs, newVCPUs, err } @@ -1554,7 +1555,7 @@ func (q *qemu) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint3 return currentVCPUs, newVCPUs, nil } -func (q *qemu) cleanup() error { +func (q *qemu) Cleanup() error { span, _ := q.trace("cleanup") defer span.Finish() diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/hypervisor/qemu_amd64.go similarity index 97% rename from virtcontainers/qemu_amd64.go rename to virtcontainers/hypervisor/qemu_amd64.go index 65d9838e44..6d49f0e656 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/hypervisor/qemu_amd64.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "os" @@ -80,7 +80,7 @@ func MaxQemuVCPUs() uint32 { return uint32(240) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_amd64_test.go b/virtcontainers/hypervisor/qemu_amd64_test.go similarity index 98% rename from virtcontainers/qemu_amd64_test.go rename to virtcontainers/hypervisor/qemu_amd64_test.go index 31e42341ad..390558be10 100644 --- a/virtcontainers/qemu_amd64_test.go +++ b/virtcontainers/hypervisor/qemu_amd64_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -17,7 +17,7 @@ import ( ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/hypervisor/qemu_arch_base.go similarity index 96% rename from virtcontainers/qemu_arch_base.go rename to virtcontainers/hypervisor/qemu_arch_base.go index 5e68bfd292..b9c12823ff 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/hypervisor/qemu_arch_base.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "encoding/hex" @@ -78,7 +78,7 @@ type qemuArch interface { appendSocket(devices []govmmQemu.Device, socket types.Socket) []govmmQemu.Device // appendVSockPCI appends a vsock PCI to devices - appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOCK) []govmmQemu.Device + appendVSockPCI(devices []govmmQemu.Device, vsock types.VSOCK) []govmmQemu.Device // appendNetwork appends a endpoint device to devices appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device @@ -96,7 +96,7 @@ type qemuArch interface { appendRNGDevice(devices []govmmQemu.Device, rngDevice config.RNGDev) []govmmQemu.Device // handleImagePath handles the Hypervisor Config image path - handleImagePath(config HypervisorConfig) + handleImagePath(config Config) // supportGuestMemoryHotplug returns if the guest supports memory hotplug supportGuestMemoryHotplug() bool @@ -121,7 +121,6 @@ const ( defaultCPUModel = "host" defaultBridgeBus = "pcie.0" maxDevIDSize = 31 - defaultMsize9p = 8192 ) // This is the PCI start address assigned to the first bridge that @@ -416,12 +415,12 @@ func (q *qemuArchBase) appendSocket(devices []govmmQemu.Device, socket types.Soc return devices } -func (q *qemuArchBase) appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOCK) []govmmQemu.Device { +func (q *qemuArchBase) appendVSockPCI(devices []govmmQemu.Device, vsock types.VSOCK) []govmmQemu.Device { devices = append(devices, govmmQemu.VSOCKDevice{ - ID: fmt.Sprintf("vsock-%d", vsock.contextID), - ContextID: vsock.contextID, - VHostFD: vsock.vhostFd, + ID: fmt.Sprintf("vsock-%d", vsock.ContextID), + ContextID: vsock.ContextID, + VHostFD: vsock.VHostFd, DisableModern: q.nestedRun, }, ) @@ -430,11 +429,11 @@ func (q *qemuArchBase) appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOC } -func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType { +func networkModelToQemuType(model types.NetInterworkingModel) govmmQemu.NetDeviceType { switch model { - case NetXConnectBridgedModel: + case types.NetXConnectBridgedModel: return govmmQemu.MACVTAP //TODO: We should rename MACVTAP to .NET_FD - case NetXConnectMacVtapModel: + case types.NetXConnectMacVtapModel: return govmmQemu.MACVTAP //case ModelEnlightened: // Here the Network plugin will create a VM native interface @@ -560,7 +559,7 @@ func (q *qemuArchBase) appendRNGDevice(devices []govmmQemu.Device, rngDev config return devices } -func (q *qemuArchBase) handleImagePath(config HypervisorConfig) { +func (q *qemuArchBase) handleImagePath(config Config) { if config.ImagePath != "" { q.kernelParams = append(q.kernelParams, kernelRootParams...) q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...) diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/hypervisor/qemu_arch_base_test.go similarity index 96% rename from virtcontainers/qemu_arch_base_test.go rename to virtcontainers/hypervisor/qemu_arch_base_test.go index 2a5f551afe..9bf5bd0c35 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/hypervisor/qemu_arch_base_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -168,10 +168,10 @@ func TestQemuArchBaseCPUTopology(t *testing.T) { Sockets: vcpus, Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: defaultMaxQemuVCPUs, + MaxCPUs: MaxQemuVCPUs(), } - smp := qemuArchBase.cpuTopology(vcpus, defaultMaxQemuVCPUs) + smp := qemuArchBase.cpuTopology(vcpus, MaxQemuVCPUs()) assert.Equal(expectedSMP, smp) } @@ -446,27 +446,27 @@ func TestQemuArchBaseAppendNetwork(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} macvlanEp := &BridgedMacvlanEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth4", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, EndpointType: BridgedMacvlanEndpointType, } macvtapEp := &MacvtapEndpoint{ EndpointType: MacvtapEndpointType, - EndpointProperties: NetworkInfo{ - Iface: NetlinkIface{ + EndpointProperties: types.NetworkInfo{ + Iface: types.NetlinkIface{ Type: "macvtap", }, }, diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/hypervisor/qemu_arm64.go similarity index 96% rename from virtcontainers/qemu_arm64.go rename to virtcontainers/hypervisor/qemu_arm64.go index bc72b0fa79..451e98a8f2 100644 --- a/virtcontainers/qemu_arm64.go +++ b/virtcontainers/hypervisor/qemu_arm64.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "io/ioutil" @@ -51,7 +51,7 @@ var supportedQemuMachines = []govmmQemu.Machine{ // Logger returns a logrus logger appropriate for logging qemu-aarch64 messages func qemuArmLogger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu-aarch64") + return logrus.WithField("subsystem", "hypervisor/qemu-aarch64") } // On ARM platform, we have different GIC interrupt controllers. Different @@ -127,7 +127,7 @@ func MaxQemuVCPUs() uint32 { return uint32(runtime.NumCPU()) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/hypervisor/qemu_arm64_test.go similarity index 97% rename from virtcontainers/qemu_arm64_test.go rename to virtcontainers/hypervisor/qemu_arm64_test.go index 30f3568010..c98bce596e 100644 --- a/virtcontainers/qemu_arm64_test.go +++ b/virtcontainers/hypervisor/qemu_arm64_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -18,7 +18,7 @@ import ( ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_ppc64le.go b/virtcontainers/hypervisor/qemu_ppc64le.go similarity index 96% rename from virtcontainers/qemu_ppc64le.go rename to virtcontainers/hypervisor/qemu_ppc64le.go index d986e87963..85296fe694 100644 --- a/virtcontainers/qemu_ppc64le.go +++ b/virtcontainers/hypervisor/qemu_ppc64le.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "encoding/hex" @@ -58,7 +58,7 @@ var supportedQemuMachines = []govmmQemu.Machine{ // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemuPPC64le) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu") + return logrus.WithField("subsystem", "hypervisor/qemu-ppc64le") } // MaxQemuVCPUs returns the maximum number of vCPUs supported @@ -66,7 +66,7 @@ func MaxQemuVCPUs() uint32 { return uint32(128) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_ppc64le_test.go b/virtcontainers/hypervisor/qemu_ppc64le_test.go similarity index 95% rename from virtcontainers/qemu_ppc64le_test.go rename to virtcontainers/hypervisor/qemu_ppc64le_test.go index ddb25fc006..a88983cb1e 100644 --- a/virtcontainers/qemu_ppc64le_test.go +++ b/virtcontainers/hypervisor/qemu_ppc64le_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -14,7 +14,7 @@ import ( ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_s390x.go b/virtcontainers/hypervisor/qemu_s390x.go similarity index 97% rename from virtcontainers/qemu_s390x.go rename to virtcontainers/hypervisor/qemu_s390x.go index 1ac865cbfa..56a103cbad 100644 --- a/virtcontainers/qemu_s390x.go +++ b/virtcontainers/hypervisor/qemu_s390x.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -55,7 +55,7 @@ func MaxQemuVCPUs() uint32 { return uint32(248) } -func newQemuArch(config HypervisorConfig) qemuArch { +func newQemuArch(config Config) qemuArch { machineType := config.HypervisorMachineType if machineType == "" { machineType = defaultQemuMachineType diff --git a/virtcontainers/qemu_s390x_test.go b/virtcontainers/hypervisor/qemu_s390x_test.go similarity index 96% rename from virtcontainers/qemu_s390x_test.go rename to virtcontainers/hypervisor/qemu_s390x_test.go index 9bb740ca31..73af6f7bef 100644 --- a/virtcontainers/qemu_s390x_test.go +++ b/virtcontainers/hypervisor/qemu_s390x_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -15,7 +15,7 @@ import ( ) func newTestQemu(machineType string) qemuArch { - config := HypervisorConfig{ + config := Config{ HypervisorMachineType: machineType, } return newQemuArch(config) diff --git a/virtcontainers/qemu_test.go b/virtcontainers/hypervisor/qemu_test.go similarity index 79% rename from virtcontainers/qemu_test.go rename to virtcontainers/hypervisor/qemu_test.go index 6c79a9297a..c631620414 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/hypervisor/qemu_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "context" @@ -20,18 +20,25 @@ import ( "github.com/stretchr/testify/assert" ) -func newQemuConfig() HypervisorConfig { - return HypervisorConfig{ +const sandboxID = "123456789" + +var testQemuKernelPath = "" +var testQemuInitrdPath = "" +var testQemuImagePath = "" +var testQemuPath = "" + +func newQemuConfig() Config { + return Config{ KernelPath: testQemuKernelPath, ImagePath: testQemuImagePath, InitrdPath: testQemuInitrdPath, HypervisorPath: testQemuPath, - NumVCPUs: defaultVCPUs, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, - Msize9p: defaultMsize9p, + NumVCPUs: DefaultVCPUs, + MemorySize: DefaultMemSzMiB, + DefaultBridges: DefaultBridges, + BlockDeviceDriver: DefaultBlockDriver, + DefaultMaxVCPUs: MaxQemuVCPUs(), + Msize9p: DefaultMsize9p, } } @@ -44,8 +51,9 @@ func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected strin } q := &qemu{ - config: qemuConfig, - arch: &qemuArchBase{}, + config: qemuConfig, + arch: &qemuArchBase{}, + maxVCPUs: MaxQemuVCPUs(), } params := q.kernelParameters() @@ -75,19 +83,12 @@ func TestQemuCreateSandbox(t *testing.T) { qemuConfig := newQemuConfig() q := &qemu{} - sandbox := &Sandbox{ - ctx: context.Background(), - id: "testSandbox", - config: &SandboxConfig{ - HypervisorConfig: qemuConfig, - }, - } + sandboxID := "testSandbox" - vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) + vcStore, err := store.NewVCSandboxStore(context.Background(), sandboxID) if err != nil { t.Fatal(err) } - sandbox.store = vcStore // Create the hypervisor fake binary testQemuPath := filepath.Join(testDir, testHypervisor) @@ -96,13 +97,13 @@ func TestQemuCreateSandbox(t *testing.T) { t.Fatalf("Could not create hypervisor file %s: %v", testQemuPath, err) } - // Create parent dir path for hypervisor.json - parentDir := store.SandboxConfigurationRootPath(sandbox.id) + // Create parent dir path for json + parentDir := store.SandboxConfigurationRootPath(sandboxID) if err := os.MkdirAll(parentDir, store.DirMode); err != nil { t.Fatalf("Could not create parent directory %s: %v", parentDir, err) } - if err := q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + if err := q.CreateSandbox(context.Background(), sandboxID, &qemuConfig, vcStore); err != nil { t.Fatal(err) } @@ -119,19 +120,12 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) { qemuConfig := newQemuConfig() q := &qemu{} - sandbox := &Sandbox{ - ctx: context.Background(), - id: "testSandbox", - config: &SandboxConfig{ - HypervisorConfig: qemuConfig, - }, - } + sandboxID := "testSandbox" - vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) + vcStore, err := store.NewVCSandboxStore(context.Background(), sandboxID) if err != nil { t.Fatal(err) } - sandbox.store = vcStore // Create the hypervisor fake binary testQemuPath := filepath.Join(testDir, testHypervisor) @@ -140,13 +134,13 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) { t.Fatalf("Could not create hypervisor file %s: %v", testQemuPath, err) } - // Ensure parent dir path for hypervisor.json does not exist. - parentDir := store.SandboxConfigurationRootPath(sandbox.id) + // Ensure parent dir path for json does not exist. + parentDir := store.SandboxConfigurationRootPath(sandboxID) if err := os.RemoveAll(parentDir); err != nil { t.Fatal(err) } - if err := q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + if err := q.CreateSandbox(context.Background(), sandboxID, &qemuConfig, vcStore); err != nil { t.Fatalf("Qemu createSandbox() is not expected to fail because of missing parent directory for storage: %v", err) } } @@ -156,10 +150,11 @@ func TestQemuCPUTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, - config: HypervisorConfig{ + config: Config{ NumVCPUs: uint32(vcpus), DefaultMaxVCPUs: uint32(vcpus), }, + maxVCPUs: uint32(vcpus), } expectedOut := govmmQemu.SMP{ @@ -183,13 +178,13 @@ func TestQemuMemoryTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, - config: HypervisorConfig{ + config: Config{ MemorySize: mem, MemSlots: slots, }, } - hostMemKb, err := getHostMemorySizeKb(procMemInfo) + hostMemKb, err := GetHostMemorySizeKb(ProcMemInfo) if err != nil { t.Fatal(err) } @@ -211,13 +206,13 @@ func TestQemuMemoryTopology(t *testing.T) { } } -func testQemuAddDevice(t *testing.T, devInfo interface{}, devType deviceType, expected []govmmQemu.Device) { +func testQemuAddDevice(t *testing.T, devInfo interface{}, devType Device, expected []govmmQemu.Device) { q := &qemu{ ctx: context.Background(), arch: &qemuArchBase{}, } - err := q.addDevice(devInfo, devType) + err := q.AddDevice(devInfo, devType) if err != nil { t.Fatal(err) } @@ -247,7 +242,7 @@ func TestQemuAddDeviceFsDev(t *testing.T) { HostPath: hostPath, } - testQemuAddDevice(t, volume, fsDev, expectedOut) + testQemuAddDevice(t, volume, FsDev, expectedOut) } func TestQemuAddDeviceSerialPortDev(t *testing.T) { @@ -274,7 +269,7 @@ func TestQemuAddDeviceSerialPortDev(t *testing.T) { Name: name, } - testQemuAddDevice(t, socket, serialPortDev, expectedOut) + testQemuAddDevice(t, socket, SerialPortDev, expectedOut) } func TestQemuAddDeviceKataVSOCK(t *testing.T) { @@ -290,13 +285,13 @@ func TestQemuAddDeviceKataVSOCK(t *testing.T) { }, } - vsock := kataVSOCK{ - contextID: contextID, - port: port, - vhostFd: vHostFD, + vsock := types.VSOCK{ + ContextID: contextID, + Port: port, + VHostFd: vHostFD, } - testQemuAddDevice(t, vsock, vSockPCIDev, expectedOut) + testQemuAddDevice(t, vsock, VSockPCIDev, expectedOut) } func TestQemuGetSandboxConsole(t *testing.T) { @@ -306,7 +301,7 @@ func TestQemuGetSandboxConsole(t *testing.T) { sandboxID := "testSandboxID" expected := filepath.Join(store.RunVMStoragePath, sandboxID, consoleSocket) - result, err := q.getSandboxConsole(sandboxID) + result, err := q.GetSandboxConsole(sandboxID) if err != nil { t.Fatal(err) } @@ -322,7 +317,7 @@ func TestQemuCapabilities(t *testing.T) { arch: &qemuArchBase{}, } - caps := q.capabilities() + caps := q.Capabilities() if !caps.IsBlockDeviceHotplugSupported() { t.Fatal("Block device hotplug should be supported") } @@ -392,9 +387,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) { } q.store = vcStore - _, err = q.hotplugAddDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.HotplugAddDevice(&MemoryDevice{0, 128}, FsDev) assert.Error(err) - _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.HotplugRemoveDevice(&MemoryDevice{0, 128}, FsDev) assert.Error(err) } @@ -421,6 +416,6 @@ func TestQemuCleanup(t *testing.T) { config: newQemuConfig(), } - err := q.cleanup() + err := q.Cleanup() assert.Nil(err) } diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/hypervisor/tap_endpoint.go similarity index 83% rename from virtcontainers/tap_endpoint.go rename to virtcontainers/hypervisor/tap_endpoint.go index c3cd3c34cc..61be0f2d6c 100644 --- a/virtcontainers/tap_endpoint.go +++ b/virtcontainers/hypervisor/tap_endpoint.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -12,18 +12,19 @@ import ( "github.com/vishvananda/netlink" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/types" ) // TapEndpoint represents just a tap endpoint type TapEndpoint struct { - TapInterface TapInterface - EndpointProperties NetworkInfo + TapInterface types.TapInterface + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } // Properties returns the properties of the tap interface. -func (endpoint *TapEndpoint) Properties() NetworkInfo { +func (endpoint *TapEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -53,17 +54,17 @@ func (endpoint *TapEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *TapEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *TapEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } // SetProperties sets the properties for the endpoint. -func (endpoint *TapEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *TapEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } // Attach for tap endpoint adds the tap interface to the hypervisor. -func (endpoint *TapEndpoint) Attach(h hypervisor) error { +func (endpoint *TapEndpoint) Attach(h Hypervisor) error { return fmt.Errorf("TapEndpoint does not support Attach, if you're using docker please use --net none") } @@ -74,20 +75,20 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error { } networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint") - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return unTapNetwork(endpoint.TapInterface.TAPIface.Name) }) } // HotAttach for the tap endpoint uses hot plug device -func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { +func (endpoint *TapEndpoint) HotAttach(h Hypervisor) error { networkLogger().Info("Hot attaching tap endpoint") - if err := tapNetwork(endpoint, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := tapNetwork(endpoint, h.Config().NumVCPUs, h.Config().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging tap ep") return err } - if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugAddDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error attach tap ep") return err } @@ -95,15 +96,15 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { } // HotDetach for the tap endpoint uses hot pull device -func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *TapEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { networkLogger().Info("Hot detaching tap endpoint") - if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + if err := DoNetNS(netNsPath, func(_ ns.NetNS) error { return unTapNetwork(endpoint.TapInterface.TAPIface.Name) }); err != nil { networkLogger().WithError(err).Warn("Error un-bridging tap ep") } - if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugRemoveDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach tap ep") return err } @@ -117,10 +118,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { uniqueID := uuid.Generate().String() endpoint := &TapEndpoint{ - TapInterface: TapInterface{ + TapInterface: types.TapInterface{ ID: uniqueID, Name: fmt.Sprintf("eth%d", idx), - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: fmt.Sprintf("tap%d_kata", idx), }, }, diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/hypervisor/veth_endpoint.go similarity index 78% rename from virtcontainers/veth_endpoint.go rename to virtcontainers/hypervisor/veth_endpoint.go index 66994a1497..ac9e4841a3 100644 --- a/virtcontainers/veth_endpoint.go +++ b/virtcontainers/hypervisor/veth_endpoint.go @@ -3,24 +3,25 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + "github.com/kata-containers/runtime/virtcontainers/types" ) // VethEndpoint gathers a network pair and its properties. type VethEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo + NetPair types.NetworkInterfacePair + EndpointProperties types.NetworkInfo Physical bool EndpointType EndpointType PCIAddr string } -func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) { +func createVethNetworkEndpoint(idx int, ifName string, interworkingModel types.NetInterworkingModel) (*VethEndpoint, error) { if idx < 0 { return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) } @@ -45,7 +46,7 @@ func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInte } // Properties returns properties for the veth interface in the network pair. -func (endpoint *VethEndpoint) Properties() NetworkInfo { +func (endpoint *VethEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -76,24 +77,24 @@ func (endpoint *VethEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *VethEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *VethEndpoint) NetworkPair() *types.NetworkInterfacePair { return &endpoint.NetPair } // SetProperties sets the properties for the endpoint. -func (endpoint *VethEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *VethEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } // Attach for veth endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -func (endpoint *VethEndpoint) Attach(h hypervisor) error { +func (endpoint *VethEndpoint) Attach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual endpoint") return err } - return h.addDevice(endpoint, netDev) + return h.AddDevice(endpoint, NetDev) } // Detach for the veth endpoint tears down the tap and bridge @@ -105,19 +106,19 @@ func (endpoint *VethEndpoint) Detach(netNsCreated bool, netNsPath string) error return nil } - return doNetNS(netNsPath, func(_ ns.NetNS) error { + return DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }) } // HotAttach for the veth endpoint uses hot plug device -func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { +func (endpoint *VethEndpoint) HotAttach(h Hypervisor) error { if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } - if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugAddDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error attach virtual ep") return err } @@ -125,18 +126,18 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { } // HotDetach for the veth endpoint uses hot pull device -func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VethEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { if !netNsCreated { return nil } - if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + if err := DoNetNS(netNsPath, func(_ ns.NetNS) error { return xDisconnectVMNetwork(endpoint) }); err != nil { networkLogger().WithError(err).Warn("Error un-bridging virtual ep") } - if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { + if _, err := h.HotplugRemoveDevice(endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach virtual ep") return err } diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/hypervisor/veth_endpoint_test.go similarity index 72% rename from virtcontainers/veth_endpoint_test.go rename to virtcontainers/hypervisor/veth_endpoint_test.go index ae62ddd525..0c16447aea 100644 --- a/virtcontainers/veth_endpoint_test.go +++ b/virtcontainers/hypervisor/veth_endpoint_test.go @@ -3,36 +3,38 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "net" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestCreateVethNetworkEndpoint(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &VethEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth4", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, EndpointType: VethEndpointType, } - result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel) + result, err := createVethNetworkEndpoint(4, "", types.DefaultNetInterworkingModel) if err != nil { t.Fatal(err) } @@ -52,24 +54,24 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) { macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} expected := &VethEndpoint{ - NetPair: NetworkInterfacePair{ - TapInterface: TapInterface{ + NetPair: types.NetworkInterfacePair{ + TapInterface: types.TapInterface{ ID: "uniqueTestID-4", Name: "br4_kata", - TAPIface: NetworkInterface{ + TAPIface: types.NetworkInterface{ Name: "tap4_kata", }, }, - VirtIface: NetworkInterface{ + VirtIface: types.NetworkInterface{ Name: "eth1", HardAddr: macAddr.String(), }, - NetInterworkingModel: DefaultNetInterworkingModel, + NetInterworkingModel: types.DefaultNetInterworkingModel, }, EndpointType: VethEndpointType, } - result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel) + result, err := createVethNetworkEndpoint(4, "eth1", types.DefaultNetInterworkingModel) if err != nil { t.Fatal(err) } @@ -98,7 +100,7 @@ func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) { } for _, d := range failingValues { - result, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel) + result, err := createVethNetworkEndpoint(d.idx, d.ifName, types.DefaultNetInterworkingModel) if err == nil { t.Fatalf("expected invalid endpoint for %v, got %v", d, result) } diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/hypervisor/vhostuser_endpoint.go similarity index 82% rename from virtcontainers/vhostuser_endpoint.go rename to virtcontainers/hypervisor/vhostuser_endpoint.go index 4960dba1be..fae735c3b3 100644 --- a/virtcontainers/vhostuser_endpoint.go +++ b/virtcontainers/hypervisor/vhostuser_endpoint.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "encoding/hex" @@ -11,6 +11,7 @@ import ( "os" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -27,13 +28,13 @@ type VhostUserEndpoint struct { // MAC address of the interface HardAddr string IfaceName string - EndpointProperties NetworkInfo + EndpointProperties types.NetworkInfo EndpointType EndpointType PCIAddr string } // Properties returns the properties of the interface. -func (endpoint *VhostUserEndpoint) Properties() NetworkInfo { +func (endpoint *VhostUserEndpoint) Properties() types.NetworkInfo { return endpoint.EndpointProperties } @@ -53,7 +54,7 @@ func (endpoint *VhostUserEndpoint) Type() EndpointType { } // SetProperties sets the properties of the endpoint. -func (endpoint *VhostUserEndpoint) SetProperties(properties NetworkInfo) { +func (endpoint *VhostUserEndpoint) SetProperties(properties types.NetworkInfo) { endpoint.EndpointProperties = properties } @@ -68,12 +69,12 @@ func (endpoint *VhostUserEndpoint) SetPciAddr(pciAddr string) { } // NetworkPair returns the network pair of the endpoint. -func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair { +func (endpoint *VhostUserEndpoint) NetworkPair() *types.NetworkInterfacePair { return nil } // Attach for vhostuser endpoint -func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { +func (endpoint *VhostUserEndpoint) Attach(h Hypervisor) error { // Generate a unique ID to be used for hypervisor commandline fields randBytes, err := utils.GenerateRandomBytes(8) if err != nil { @@ -88,7 +89,7 @@ func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { Type: config.VhostUserNet, } - return h.addDevice(d, vhostuserDev) + return h.AddDevice(d, VhostuserDev) } // Detach for vhostuser endpoint @@ -97,17 +98,17 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e } // HotAttach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error { +func (endpoint *VhostUserEndpoint) HotAttach(h Hypervisor) error { return fmt.Errorf("VhostUserEndpoint does not support Hot attach") } // HotDetach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VhostUserEndpoint) HotDetach(h Hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("VhostUserEndpoint does not support Hot detach") } // Create a vhostuser endpoint -func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { +func createVhostUserEndpoint(netInfo types.NetworkInfo, socket string) (*VhostUserEndpoint, error) { vhostUserEndpoint := &VhostUserEndpoint{ SocketPath: socket, @@ -120,7 +121,7 @@ func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndp // findVhostUserNetSocketPath checks if an interface is a dummy placeholder // for a vhost-user socket, and if it is it returns the path to the socket -func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { +func findVhostUserNetSocketPath(netInfo types.NetworkInfo) (string, error) { if netInfo.Iface.Name == "lo" { return "", nil } @@ -142,7 +143,7 @@ func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { func vhostUserSocketPath(info interface{}) (string, error) { switch v := info.(type) { - case NetworkInfo: + case types.NetworkInfo: return findVhostUserNetSocketPath(v) default: return "", nil diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/hypervisor/vhostuser_endpoint_test.go similarity index 92% rename from virtcontainers/vhostuser_endpoint_test.go rename to virtcontainers/hypervisor/vhostuser_endpoint_test.go index 6d1ad91e86..7270b5926f 100644 --- a/virtcontainers/vhostuser_endpoint_test.go +++ b/virtcontainers/hypervisor/vhostuser_endpoint_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package hypervisor import ( "fmt" @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" + + "github.com/kata-containers/runtime/virtcontainers/types" ) func TestVhostUserSocketPath(t *testing.T) { @@ -47,7 +49,7 @@ func TestVhostUserSocketPath(t *testing.T) { if err != nil { t.Fatal(err) } - netinfo := NetworkInfo{ + netinfo := types.NetworkInfo{ Addrs: addresses, } @@ -66,7 +68,7 @@ func TestVhostUserSocketPath(t *testing.T) { }, }, } - netinfoFail := NetworkInfo{ + netinfoFail := types.NetworkInfo{ Addrs: addressesFalse, } @@ -94,7 +96,7 @@ func TestVhostUserEndpointAttach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + h := &mock{} err := v.Attach(h) if err != nil { @@ -110,7 +112,7 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + h := &mock{} err := v.HotAttach(h) assert.Error(err) @@ -124,7 +126,7 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + h := &mock{} err := v.HotDetach(h, true, "") assert.Error(err) @@ -135,8 +137,8 @@ func TestCreateVhostUserEndpoint(t *testing.T) { ifcName := "vhost-deadbeef" socket := "/tmp/vhu_192.168.0.1" - netinfo := NetworkInfo{ - Iface: NetlinkIface{ + netinfo := types.NetworkInfo{ + Iface: types.NetlinkIface{ LinkAttrs: netlink.LinkAttrs{ HardwareAddr: macAddr, Name: ifcName, diff --git a/virtcontainers/iostream_test.go b/virtcontainers/iostream_test.go index 152f0a6118..c080db5acb 100644 --- a/virtcontainers/iostream_test.go +++ b/virtcontainers/iostream_test.go @@ -8,12 +8,14 @@ package virtcontainers import ( "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) func TestIOStream(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{}, nil) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index dcf4735d47..44a8d72f5f 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -23,6 +23,7 @@ import ( kataclient "github.com/kata-containers/agent/protocols/client" "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" @@ -56,7 +57,6 @@ var ( mountGuest9pTag = "kataShared" kataGuestSandboxDir = "/run/kata-containers/sandbox/" type9pFs = "9p" - vsockSocketScheme = "vsock" // port numbers below 1024 are called privileged ports. Only a process with // CAP_NET_BIND_SERVICE capability may bind to these port numbers. vSockPort = 1024 @@ -79,16 +79,6 @@ type KataAgentConfig struct { UseVSock bool } -type kataVSOCK struct { - contextID uint64 - port uint32 - vhostFd *os.File -} - -func (s *kataVSOCK) String() string { - return fmt.Sprintf("%s://%d:%d", vsockSocketScheme, s.contextID, s.port) -} - // KataAgentState is the structure describing the data stored from this // agent implementation. type KataAgentState struct { @@ -144,7 +134,7 @@ func (k *kataAgent) generateVMSocket(id string, c KataAgentConfig) error { // We want to go through VSOCK. The VM VSOCK endpoint will be our gRPC. k.Logger().Debug("agent: Using vsock VM socket endpoint") // We dont know yet the context ID - set empty vsock configuration - k.vmSocket = kataVSOCK{} + k.vmSocket = types.VSOCK{} } else { k.Logger().Debug("agent: Using unix socket form VM socket endpoint") // We need to generate a host UNIX socket path for the emulated serial port. @@ -205,7 +195,7 @@ func (k *kataAgent) agentURL() (string, error) { switch s := k.vmSocket.(type) { case types.Socket: return s.HostPath, nil - case kataVSOCK: + case types.VSOCK: return s.String(), nil default: return "", fmt.Errorf("Invalid socket type") @@ -221,7 +211,7 @@ func (k *kataAgent) capabilities() types.Capabilities { return caps } -func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error { +func (k *kataAgent) configure(h hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error { if config != nil { switch c := config.(type) { case KataAgentConfig: @@ -236,18 +226,18 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, switch s := k.vmSocket.(type) { case types.Socket: - err := h.addDevice(s, serialPortDev) + err := h.AddDevice(s, hypervisor.SerialPortDev) if err != nil { return err } - case kataVSOCK: + case types.VSOCK: var err error - s.vhostFd, s.contextID, err = utils.FindContextID() + s.VHostFd, s.ContextID, err = utils.FindContextID() if err != nil { return err } - s.port = uint32(vSockPort) - if err := h.addDevice(s, vSockPCIDev); err != nil { + s.Port = uint32(vSockPort) + if err := h.AddDevice(s, hypervisor.VSockPCIDev); err != nil { return err } k.vmSocket = s @@ -261,7 +251,7 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, // Neither create shared directory nor add 9p device if hypervisor // doesn't support filesystem sharing. - caps := h.capabilities() + caps := h.Capabilities() if !caps.IsFsSharingSupported() { return nil } @@ -277,7 +267,7 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, return err } - return h.addDevice(sharedVolume, fsDev) + return h.AddDevice(sharedVolume, hypervisor.FsDev) } func (k *kataAgent) createSandbox(sandbox *Sandbox) error { @@ -505,7 +495,7 @@ func (k *kataAgent) startProxy(sandbox *Sandbox) error { return err } - consoleURL, err := sandbox.hypervisor.getSandboxConsole(sandbox.id) + consoleURL, err := sandbox.hypervisor.GetSandboxConsole(sandbox.id) if err != nil { return err } @@ -615,7 +605,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { // // Setup network interfaces and routes // - interfaces, routes, err := generateInterfacesAndRoutes(sandbox.networkNS) + interfaces, routes, err := sandbox.networkNS.interfacesAndRoutes(sandbox.networkNS) if err != nil { return err } @@ -627,7 +617,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { } storages := []*grpc.Storage{} - caps := sandbox.hypervisor.capabilities() + caps := sandbox.hypervisor.Capabilities() // append 9p shared volume to storages only if filesystem sharing is supported if caps.IsFsSharingSupported() { diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 5a55cbd7e9..b0d69168ce 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -29,6 +29,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/mock" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" @@ -412,7 +413,7 @@ func TestAppendDevices(t *testing.T) { } sandboxConfig := &SandboxConfig{ - HypervisorConfig: HypervisorConfig{ + HypervisorConfig: hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, }, } @@ -648,7 +649,7 @@ func TestAgentPathAPI(t *testing.T) { c.UseVSock = true err = k2.generateVMSocket(id, c) assert.Nil(err) - _, ok = k2.vmSocket.(kataVSOCK) + _, ok = k2.vmSocket.(types.VSOCK) assert.True(ok) } @@ -659,7 +660,7 @@ func TestAgentConfigure(t *testing.T) { assert.Nil(err) k := &kataAgent{} - h := &mockHypervisor{} + h := hypervisor.NewMock() c := KataAgentConfig{} id := "foobar" @@ -730,13 +731,13 @@ func TestAgentCreateContainer(t *testing.T) { id: "foobar", config: &SandboxConfig{ ID: "foobar", - HypervisorType: MockHypervisor, - HypervisorConfig: HypervisorConfig{ + HypervisorType: hypervisor.Mock, + HypervisorConfig: hypervisor.Config{ KernelPath: "foo", ImagePath: "bar", }, }, - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), } vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) @@ -787,7 +788,7 @@ func TestAgentCreateContainer(t *testing.T) { dir, err := ioutil.TempDir("", "kata-agent-test") assert.Nil(err) - err = k.configure(&mockHypervisor{}, sandbox.id, dir, true, KataAgentConfig{}) + err = k.configure(hypervisor.NewMock(), sandbox.id, dir, true, KataAgentConfig{}) assert.Nil(err) // We'll fail on container metadata file creation, but it helps increasing coverage... diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go deleted file mode 100644 index 37dccaf901..0000000000 --- a/virtcontainers/mock_hypervisor.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2016 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "context" - "os" - - "github.com/kata-containers/runtime/virtcontainers/store" - "github.com/kata-containers/runtime/virtcontainers/types" -) - -type mockHypervisor struct { -} - -func (m *mockHypervisor) capabilities() types.Capabilities { - return types.Capabilities{} -} - -func (m *mockHypervisor) hypervisorConfig() HypervisorConfig { - return HypervisorConfig{} -} - -func (m *mockHypervisor) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { - err := hypervisorConfig.valid() - if err != nil { - return err - } - - return nil -} - -func (m *mockHypervisor) startSandbox(timeout int) error { - return nil -} - -func (m *mockHypervisor) stopSandbox() error { - return nil -} - -func (m *mockHypervisor) pauseSandbox() error { - return nil -} - -func (m *mockHypervisor) resumeSandbox() error { - return nil -} - -func (m *mockHypervisor) saveSandbox() error { - return nil -} - -func (m *mockHypervisor) addDevice(devInfo interface{}, devType deviceType) error { - return nil -} - -func (m *mockHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { - switch devType { - case cpuDev: - return devInfo.(uint32), nil - case memoryDev: - memdev := devInfo.(*memoryDevice) - return memdev.sizeMB, nil - } - return nil, nil -} - -func (m *mockHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { - switch devType { - case cpuDev: - return devInfo.(uint32), nil - case memoryDev: - return 0, nil - } - return nil, nil -} - -func (m *mockHypervisor) getSandboxConsole(sandboxID string) (string, error) { - return "", nil -} - -func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { - return 0, nil -} -func (m *mockHypervisor) resizeVCPUs(cpus uint32) (uint32, uint32, error) { - return 0, 0, nil -} - -func (m *mockHypervisor) disconnect() { -} - -func (m *mockHypervisor) getThreadIDs() (*threadIDs, error) { - vcpus := []int{os.Getpid()} - return &threadIDs{vcpus}, nil -} - -func (m *mockHypervisor) cleanup() error { - return nil -} diff --git a/virtcontainers/monitor_test.go b/virtcontainers/monitor_test.go index 2d786acf37..f7c9853520 100644 --- a/virtcontainers/monitor_test.go +++ b/virtcontainers/monitor_test.go @@ -9,6 +9,8 @@ import ( "errors" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) @@ -18,7 +20,7 @@ func TestMonitorSuccess(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } @@ -43,7 +45,7 @@ func TestMonitorClosedChannel(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 01a666c8a0..d50e2476ae 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -7,1158 +7,136 @@ package virtcontainers import ( "context" - cryptoRand "crypto/rand" "encoding/json" "fmt" - "math/rand" - "net" "os" - "runtime" - "sort" - "time" "github.com/containernetworking/plugins/pkg/ns" opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" "golang.org/x/sys/unix" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" - "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" - "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/kata-containers/runtime/virtcontainers/types" ) -// NetInterworkingModel defines the network model connecting -// the network interface to the virtual machine. -type NetInterworkingModel int - -const ( - // NetXConnectDefaultModel Ask to use DefaultNetInterworkingModel - NetXConnectDefaultModel NetInterworkingModel = iota - - // NetXConnectBridgedModel uses a linux bridge to interconnect - // the container interface to the VM. This is the - // safe default that works for most cases except - // macvlan and ipvlan - NetXConnectBridgedModel - - // NetXConnectMacVtapModel can be used when the Container network - // interface can be bridged using macvtap - NetXConnectMacVtapModel - - // NetXConnectEnlightenedModel can be used when the Network plugins - // are enlightened to create VM native interfaces - // when requested by the runtime - // This will be used for vethtap, macvtap, ipvtap - NetXConnectEnlightenedModel - - // NetXConnectTCFilterModel redirects traffic from the network interface - // provided by the network plugin to a tap interface. - // This works for ipvlan and macvlan as well. - NetXConnectTCFilterModel - - // NetXConnectNoneModel can be used when the VM is in the host network namespace - NetXConnectNoneModel - - // NetXConnectInvalidModel is the last item to check valid values by IsValid() - NetXConnectInvalidModel -) - -//IsValid checks if a model is valid -func (n NetInterworkingModel) IsValid() bool { - return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel) -} - -const ( - defaultNetModelStr = "default" - - bridgedNetModelStr = "bridged" - - macvtapNetModelStr = "macvtap" - - enlightenedNetModelStr = "enlightened" - - tcFilterNetModelStr = "tcfilter" - - noneNetModelStr = "none" -) - -//SetModel change the model string value -func (n *NetInterworkingModel) SetModel(modelName string) error { - switch modelName { - case defaultNetModelStr: - *n = DefaultNetInterworkingModel - return nil - case bridgedNetModelStr: - *n = NetXConnectBridgedModel - return nil - case macvtapNetModelStr: - *n = NetXConnectMacVtapModel - return nil - case enlightenedNetModelStr: - *n = NetXConnectEnlightenedModel - return nil - case tcFilterNetModelStr: - *n = NetXConnectTCFilterModel - return nil - case noneNetModelStr: - *n = NetXConnectNoneModel - return nil - } - return fmt.Errorf("Unknown type %s", modelName) -} - -// DefaultNetInterworkingModel is a package level default -// that determines how the VM should be connected to the -// the container network interface -var DefaultNetInterworkingModel = NetXConnectMacVtapModel - -// Introduces constants related to networking -const ( - defaultRouteDest = "0.0.0.0/0" - defaultRouteLabel = "default" - defaultFilePerms = 0600 - defaultQlen = 1500 -) - -// DNSInfo describes the DNS setup related to a network interface. -type DNSInfo struct { - Servers []string - Domain string - Searches []string - Options []string -} - -// NetlinkIface describes fully a network interface. -type NetlinkIface struct { - netlink.LinkAttrs - Type string -} - -// NetworkInfo gathers all information related to a network interface. -// It can be used to store the description of the underlying network. -type NetworkInfo struct { - Iface NetlinkIface - Addrs []netlink.Addr - Routes []netlink.Route - DNS DNSInfo -} - -// NetworkInterface defines a network interface. -type NetworkInterface struct { - Name string - HardAddr string - Addrs []netlink.Addr -} - -// TapInterface defines a tap interface -type TapInterface struct { - ID string - Name string - TAPIface NetworkInterface - VMFds []*os.File - VhostFds []*os.File -} - -// NetworkInterfacePair defines a pair between VM and virtual network interfaces. -type NetworkInterfacePair struct { - TapInterface - VirtIface NetworkInterface - NetInterworkingModel -} - -// NetworkConfig is the network configuration related to a network. -type NetworkConfig struct { - NetNSPath string - NetNsCreated bool - DisableNewNetNs bool - NetmonConfig NetmonConfig - InterworkingModel NetInterworkingModel -} - -func networkLogger() *logrus.Entry { - return virtLog.WithField("subsystem", "network") -} - -// NetworkNamespace contains all data related to its network namespace. -type NetworkNamespace struct { - NetNsPath string - NetNsCreated bool - Endpoints []Endpoint - NetmonPID int -} - -// TypedJSONEndpoint is used as an intermediate representation for -// marshalling and unmarshalling Endpoint objects. -type TypedJSONEndpoint struct { - Type EndpointType - Data json.RawMessage -} - -// MarshalJSON is the custom NetworkNamespace JSON marshalling routine. -// This is needed to properly marshall Endpoints array. -func (n NetworkNamespace) MarshalJSON() ([]byte, error) { - // We need a shadow structure in order to prevent json from - // entering a recursive loop when only calling json.Marshal(). - type shadow struct { - NetNsPath string - NetNsCreated bool - Endpoints []TypedJSONEndpoint - } - - s := &shadow{ - NetNsPath: n.NetNsPath, - NetNsCreated: n.NetNsCreated, - } - - var typedEndpoints []TypedJSONEndpoint - for _, endpoint := range n.Endpoints { - tempJSON, _ := json.Marshal(endpoint) - - t := TypedJSONEndpoint{ - Type: endpoint.Type(), - Data: tempJSON, - } - - typedEndpoints = append(typedEndpoints, t) - } - - s.Endpoints = typedEndpoints - - b, err := json.Marshal(s) - return b, err -} - -func generateEndpoints(typedEndpoints []TypedJSONEndpoint) ([]Endpoint, error) { - var endpoints []Endpoint - - for _, e := range typedEndpoints { - switch e.Type { - case PhysicalEndpointType: - var endpoint PhysicalEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "physical", - }).Info("endpoint unmarshalled") - - case VethEndpointType: - var endpoint VethEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "virtual", - }).Info("endpoint unmarshalled") - - case VhostUserEndpointType: - var endpoint VhostUserEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "vhostuser", - }).Info("endpoint unmarshalled") - - case BridgedMacvlanEndpointType: - var endpoint BridgedMacvlanEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "macvlan", - }).Info("endpoint unmarshalled") - - case MacvtapEndpointType: - var endpoint MacvtapEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "macvtap", - }).Info("endpoint unmarshalled") - - case TapEndpointType: - var endpoint TapEndpoint - err := json.Unmarshal(e.Data, &endpoint) - if err != nil { - return nil, err - } - - endpoints = append(endpoints, &endpoint) - networkLogger().WithFields(logrus.Fields{ - "endpoint": endpoint, - "endpoint-type": "tap", - }).Info("endpoint unmarshalled") - - default: - networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type") - } - } - return endpoints, nil -} - -// UnmarshalJSON is the custom NetworkNamespace unmarshalling routine. -// This is needed for unmarshalling the Endpoints interfaces array. -func (n *NetworkNamespace) UnmarshalJSON(b []byte) error { - var s struct { - NetNsPath string - NetNsCreated bool - Endpoints json.RawMessage - } - - if err := json.Unmarshal(b, &s); err != nil { - return err - } - - (*n).NetNsPath = s.NetNsPath - (*n).NetNsCreated = s.NetNsCreated - - var typedEndpoints []TypedJSONEndpoint - if err := json.Unmarshal([]byte(string(s.Endpoints)), &typedEndpoints); err != nil { - return err - } - endpoints, err := generateEndpoints(typedEndpoints) - if err != nil { - return err - } - - (*n).Endpoints = endpoints - return nil -} - -func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, queues int) (netlink.Link, []*os.File, error) { - var newLink netlink.Link - var fds []*os.File - - switch expectedLink.Type() { - case (&netlink.Bridge{}).Type(): - newLink = &netlink.Bridge{ - LinkAttrs: netlink.LinkAttrs{Name: name}, - MulticastSnooping: expectedLink.(*netlink.Bridge).MulticastSnooping, - } - case (&netlink.Tuntap{}).Type(): - flags := netlink.TUNTAP_VNET_HDR - if queues > 0 { - flags |= netlink.TUNTAP_MULTI_QUEUE_DEFAULTS - } - newLink = &netlink.Tuntap{ - LinkAttrs: netlink.LinkAttrs{Name: name}, - Mode: netlink.TUNTAP_MODE_TAP, - Queues: queues, - Flags: flags, - } - case (&netlink.Macvtap{}).Type(): - qlen := expectedLink.Attrs().TxQLen - if qlen <= 0 { - qlen = defaultQlen - } - newLink = &netlink.Macvtap{ - Macvlan: netlink.Macvlan{ - Mode: netlink.MACVLAN_MODE_BRIDGE, - LinkAttrs: netlink.LinkAttrs{ - Index: expectedLink.Attrs().Index, - Name: name, - TxQLen: qlen, - ParentIndex: expectedLink.Attrs().ParentIndex, - }, - }, - } - default: - return nil, fds, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) - } - - if err := netHandle.LinkAdd(newLink); err != nil { - return nil, fds, fmt.Errorf("LinkAdd() failed for %s name %s: %s", expectedLink.Type(), name, err) - } - - tuntapLink, ok := newLink.(*netlink.Tuntap) - if ok { - fds = tuntapLink.Fds - } - - newLink, err := getLinkByName(netHandle, name, expectedLink) - return newLink, fds, err -} - -func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.Link, error) { - var link netlink.Link - - switch ep := endpoint.(type) { - case *VethEndpoint: - link = &netlink.Veth{} - case *BridgedMacvlanEndpoint: - link = &netlink.Macvlan{} - case *IPVlanEndpoint: - link = &netlink.IPVlan{} - default: - return nil, fmt.Errorf("Unexpected endpointType %s", ep.Type()) - } - - return getLinkByName(netHandle, endpoint.NetworkPair().VirtIface.Name, link) -} - -func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.Link) (netlink.Link, error) { - link, err := netHandle.LinkByName(name) - if err != nil { - return nil, fmt.Errorf("LinkByName() failed for %s name %s: %s", expectedLink.Type(), name, err) - } - - switch expectedLink.Type() { - case (&netlink.Bridge{}).Type(): - if l, ok := link.(*netlink.Bridge); ok { - return l, nil - } - case (&netlink.Tuntap{}).Type(): - if l, ok := link.(*netlink.GenericLink); ok { - return l, nil - } - case (&netlink.Veth{}).Type(): - if l, ok := link.(*netlink.Veth); ok { - return l, nil - } - case (&netlink.Macvtap{}).Type(): - if l, ok := link.(*netlink.Macvtap); ok { - return l, nil - } - case (&netlink.Macvlan{}).Type(): - if l, ok := link.(*netlink.Macvlan); ok { - return l, nil - } - case (&netlink.IPVlan{}).Type(): - if l, ok := link.(*netlink.IPVlan); ok { - return l, nil - } - default: - return nil, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) - } - - return nil, fmt.Errorf("Incorrect link type %s, expecting %s", link.Type(), expectedLink.Type()) -} - -// The endpoint type should dictate how the connection needs to happen. -func xConnectVMNetwork(endpoint Endpoint, h hypervisor) error { - netPair := endpoint.NetworkPair() - - queues := 0 - caps := h.capabilities() - if caps.IsMultiQueueSupported() { - queues = int(h.hypervisorConfig().NumVCPUs) - } - - disableVhostNet := h.hypervisorConfig().DisableVhostNet - - if netPair.NetInterworkingModel == NetXConnectDefaultModel { - netPair.NetInterworkingModel = DefaultNetInterworkingModel - } - - switch netPair.NetInterworkingModel { - case NetXConnectBridgedModel: - return bridgeNetworkPair(endpoint, queues, disableVhostNet) - case NetXConnectMacVtapModel: - return tapNetworkPair(endpoint, queues, disableVhostNet) - case NetXConnectTCFilterModel: - return setupTCFiltering(endpoint, queues, disableVhostNet) - case NetXConnectEnlightenedModel: - return fmt.Errorf("Unsupported networking model") - default: - return fmt.Errorf("Invalid internetworking model") - } -} - -// The endpoint type should dictate how the disconnection needs to happen. -func xDisconnectVMNetwork(endpoint Endpoint) error { - netPair := endpoint.NetworkPair() - - if netPair.NetInterworkingModel == NetXConnectDefaultModel { - netPair.NetInterworkingModel = DefaultNetInterworkingModel - } - - switch netPair.NetInterworkingModel { - case NetXConnectBridgedModel: - return unBridgeNetworkPair(endpoint) - case NetXConnectMacVtapModel: - return untapNetworkPair(endpoint) - case NetXConnectTCFilterModel: - return removeTCFiltering(endpoint) - case NetXConnectEnlightenedModel: - return fmt.Errorf("Unsupported networking model") - default: - return fmt.Errorf("Invalid internetworking model") - } -} - -func createMacvtapFds(linkIndex int, queues int) ([]*os.File, error) { - tapDev := fmt.Sprintf("/dev/tap%d", linkIndex) - return createFds(tapDev, queues) -} - -func createVhostFds(numFds int) ([]*os.File, error) { - vhostDev := "/dev/vhost-net" - return createFds(vhostDev, numFds) -} - -func createFds(device string, numFds int) ([]*os.File, error) { - fds := make([]*os.File, numFds) - - for i := 0; i < numFds; i++ { - f, err := os.OpenFile(device, os.O_RDWR, defaultFilePerms) - if err != nil { - utils.CleanupFds(fds, i) - return nil, err - } - fds[i] = f - } - return fds, nil -} - -// There is a limitation in the linux kernel that prevents a macvtap/macvlan link -// from getting the correct link index when created in a network namespace -// https://github.com/clearcontainers/runtime/issues/708 -// -// Till that bug is fixed we need to pick a random non conflicting index and try to -// create a link. If that fails, we need to try with another. -// All the kernel does not check if the link id conflicts with a link id on the host -// hence we need to offset the link id to prevent any overlaps with the host index -// -// Here the kernel will ensure that there is no race condition - -const hostLinkOffset = 8192 // Host should not have more than 8k interfaces -const linkRange = 0xFFFF // This will allow upto 2^16 containers -const linkRetries = 128 // The numbers of time we try to find a non conflicting index -const macvtapWorkaround = true - -func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, queues int) (taplink netlink.Link, err error) { - - if !macvtapWorkaround { - taplink, _, err = createLink(netHandle, name, link, queues) - return - } - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - - for i := 0; i < linkRetries; i++ { - index := hostLinkOffset + (r.Int() & linkRange) - link.Attrs().Index = index - taplink, _, err = createLink(netHandle, name, link, queues) - if err == nil { - break - } - } - - return -} - -func clearIPs(link netlink.Link, addrs []netlink.Addr) error { - for _, addr := range addrs { - if err := netlink.AddrDel(link, &addr); err != nil { - return err - } - } - return nil -} - -func setIPs(link netlink.Link, addrs []netlink.Addr) error { - for _, addr := range addrs { - if err := netlink.AddrAdd(link, &addr); err != nil { - return err - } - } - return nil -} - -func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - link, err := getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - attrs := link.Attrs() - - // Attach the macvtap interface to the underlying container - // interface. Also picks relevant attributes from the parent - tapLink, err := createMacVtap(netHandle, netPair.TAPIface.Name, - &netlink.Macvtap{ - Macvlan: netlink.Macvlan{ - LinkAttrs: netlink.LinkAttrs{ - TxQLen: attrs.TxQLen, - ParentIndex: attrs.Index, - }, - }, - }, queues) - - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } - - // Save the veth MAC address to the TAP so that it can later be used - // to build the hypervisor command line. This MAC address has to be - // the one inside the VM in order to avoid any firewall issues. The - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - tapHardAddr := attrs.HardwareAddr - netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - - if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) - } - - hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) - if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetHardwareAddr(tapLink, tapHardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) - } - - // Clear the IP addresses from the veth interface to prevent ARP conflict - netPair.VirtIface.Addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - return fmt.Errorf("Unable to obtain veth IP addresses: %s", err) - } - - if err := clearIPs(link, netPair.VirtIface.Addrs); err != nil { - return fmt.Errorf("Unable to clear veth IP addresses: %s", err) - } - - if err := netHandle.LinkSetUp(link); err != nil { - return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) - } - - // Note: The underlying interfaces need to be up prior to fd creation. - - netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, queues) - if err != nil { - return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err) - } - - if !disableVhostNet { - vhostFds, err := createVhostFds(queues) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) - } - netPair.VhostFds = vhostFds - } - - return nil -} - -func bridgeNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } - netPair.VMFds = fds - - if !disableVhostNet { - vhostFds, err := createVhostFds(queues) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) - } - netPair.VhostFds = vhostFds - } - - var attrs *netlink.LinkAttrs - var link netlink.Link - - link, err = getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - attrs = link.Attrs() - - // Save the veth MAC address to the TAP so that it can later be used - // to build the hypervisor command line. This MAC address has to be - // the one inside the VM in order to avoid any firewall issues. The - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - - if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) - } - - hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) - if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - mcastSnoop := false - bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, queues) - if err != nil { - return fmt.Errorf("Could not create bridge: %s", err) - } - - if err := netHandle.LinkSetMaster(tapLink, bridgeLink.(*netlink.Bridge)); err != nil { - return fmt.Errorf("Could not attach TAP %s to the bridge %s: %s", - netPair.TAPIface.Name, netPair.Name, err) - } - - if err := netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkSetMaster(link, bridgeLink.(*netlink.Bridge)); err != nil { - return fmt.Errorf("Could not attach veth %s to the bridge %s: %s", - netPair.VirtIface.Name, netPair.Name, err) - } - - if err := netHandle.LinkSetUp(link); err != nil { - return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetUp(bridgeLink); err != nil { - return fmt.Errorf("Could not enable bridge %s: %s", netPair.Name, err) - } - - return nil -} - -func setupTCFiltering(endpoint Endpoint, queues int, disableVhostNet bool) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } - netPair.VMFds = fds - - if !disableVhostNet { - vhostFds, err := createVhostFds(queues) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) - } - netPair.VhostFds = vhostFds - } - - var attrs *netlink.LinkAttrs - var link netlink.Link - - link, err = getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - attrs = link.Attrs() - - // Save the veth MAC address to the TAP so that it can later be used - // to build the hypervisor command line. This MAC address has to be - // the one inside the VM in order to avoid any firewall issues. The - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - - if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) - } - - if err := netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) - } - - tapAttrs := tapLink.Attrs() - - if err := addQdiscIngress(tapAttrs.Index); err != nil { - return err - } - - if err := addQdiscIngress(attrs.Index); err != nil { - return err - } - - if err := addRedirectTCFilter(attrs.Index, tapAttrs.Index); err != nil { - return err - } - - if err := addRedirectTCFilter(tapAttrs.Index, attrs.Index); err != nil { - return err - } - - return nil -} - -// addQdiscIngress creates a new qdisc for nwtwork interface with the specified network index -// on "ingress". qdiscs normally don't work on ingress so this is really a special qdisc -// that you can consider an "alternate root" for inbound packets. -// Handle for ingress qdisc defaults to "ffff:" -// -// This is equivalent to calling `tc qdisc add dev eth0 ingress` -func addQdiscIngress(index int) error { - qdisc := &netlink.Ingress{ - QdiscAttrs: netlink.QdiscAttrs{ - LinkIndex: index, - Parent: netlink.HANDLE_INGRESS, - }, - } - - err := netlink.QdiscAdd(qdisc) - if err != nil { - return fmt.Errorf("Failed to add qdisc for network index %d : %s", index, err) - } - - return nil -} - -// addRedirectTCFilter adds a tc filter for device with index "sourceIndex". -// All traffic for interface with index "sourceIndex" is redirected to interface with -// index "destIndex" -// -// This is equivalent to calling: -// `tc filter add dev source parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev dest` -func addRedirectTCFilter(sourceIndex, destIndex int) error { - filter := &netlink.U32{ - FilterAttrs: netlink.FilterAttrs{ - LinkIndex: sourceIndex, - Parent: netlink.MakeHandle(0xffff, 0), - Protocol: unix.ETH_P_ALL, - }, - Actions: []netlink.Action{ - &netlink.MirredAction{ - ActionAttrs: netlink.ActionAttrs{ - Action: netlink.TC_ACT_STOLEN, - }, - MirredAction: netlink.TCA_EGRESS_REDIR, - Ifindex: destIndex, - }, - }, - } - - if err := netlink.FilterAdd(filter); err != nil { - return fmt.Errorf("Failed to add filter for index %d : %s", sourceIndex, err) - } - - return nil -} - -// removeRedirectTCFilter removes all tc u32 filters created on ingress qdisc for "link". -func removeRedirectTCFilter(link netlink.Link) error { - if link == nil { - return nil - } - - // Handle 0xffff is used for ingress - filters, err := netlink.FilterList(link, netlink.MakeHandle(0xffff, 0)) - if err != nil { - return err - } - - for _, f := range filters { - u32, ok := f.(*netlink.U32) - - if !ok { - continue - } - - if err := netlink.FilterDel(u32); err != nil { - return err - } - } - return nil -} - -// removeQdiscIngress removes the ingress qdisc previously created on "link". -func removeQdiscIngress(link netlink.Link) error { - if link == nil { - return nil - } - - qdiscs, err := netlink.QdiscList(link) - if err != nil { - return err - } - - for _, qdisc := range qdiscs { - ingress, ok := qdisc.(*netlink.Ingress) - if !ok { - continue - } - - if err := netlink.QdiscDel(ingress); err != nil { - return err - } - } - return nil +func networkLogger() *logrus.Entry { + return virtLog.WithField("subsystem", "network") } -func untapNetworkPair(endpoint Endpoint) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Macvtap{}) - if err != nil { - return fmt.Errorf("Could not get TAP interface %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkDel(tapLink); err != nil { - return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) - } - - link, err := getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) +func createNetNS() (string, error) { + n, err := ns.NewNS() if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetDown(link); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + return "", err } - // Restore the IPs that were cleared - err = setIPs(link, netPair.VirtIface.Addrs) - return err + return n.Path(), nil } -func unBridgeNetworkPair(endpoint Endpoint) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) - if err != nil { - return fmt.Errorf("Could not get TAP interface: %s", err) - } - - bridgeLink, err := getLinkByName(netHandle, netPair.Name, &netlink.Bridge{}) - if err != nil { - return fmt.Errorf("Could not get bridge interface: %s", err) - } - - if err := netHandle.LinkSetDown(bridgeLink); err != nil { - return fmt.Errorf("Could not disable bridge %s: %s", netPair.Name, err) - } - - if err := netHandle.LinkSetDown(tapLink); err != nil { - return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkSetNoMaster(tapLink); err != nil { - return fmt.Errorf("Could not detach TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkDel(bridgeLink); err != nil { - return fmt.Errorf("Could not remove bridge %s: %s", netPair.Name, err) - } - - if err := netHandle.LinkDel(tapLink); err != nil { - return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) - } - - link, err := getLinkForEndpoint(endpoint, netHandle) +func deleteNetNS(netNSPath string) error { + n, err := ns.GetNS(netNSPath) if err != nil { return err } - hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) + err = n.Close() if err != nil { return err } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - if err := netHandle.LinkSetDown(link); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + if err = unix.Unmount(netNSPath, unix.MNT_DETACH); err != nil { + return fmt.Errorf("Failed to unmount namespace %s: %v", netNSPath, err) } - - if err := netHandle.LinkSetNoMaster(link); err != nil { - return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) + if err := os.RemoveAll(netNSPath); err != nil { + return fmt.Errorf("Failed to clean up namespace %s: %v", netNSPath, err) } return nil } -func removeTCFiltering(endpoint Endpoint) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) - if err != nil { - return fmt.Errorf("Could not get TAP interface: %s", err) - } - - if err := netHandle.LinkSetDown(tapLink); err != nil { - return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) - } +// NetworkNamespace contains all data related to its network namespace. +type NetworkNamespace struct { + NetNsPath string + NetNsCreated bool + Endpoints []hypervisor.Endpoint + NetmonPID int +} - if err := netHandle.LinkDel(tapLink); err != nil { - return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) +// MarshalJSON is the custom NetworkNamespace JSON marshalling routine. +// This is needed to properly marshall Endpoints array. +func (n NetworkNamespace) MarshalJSON() ([]byte, error) { + // We need a shadow structure in order to prevent json from + // entering a recursive loop when only calling json.Marshal(). + type shadow struct { + NetNsPath string + NetNsCreated bool + Endpoints []hypervisor.TypedJSONEndpoint } - link, err := getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err + s := &shadow{ + NetNsPath: n.NetNsPath, + NetNsCreated: n.NetNsCreated, } - if err := removeRedirectTCFilter(link); err != nil { - return err - } + var typedEndpoints []hypervisor.TypedJSONEndpoint + for _, ep := range n.Endpoints { + tempJSON, _ := json.Marshal(ep) - if err := removeQdiscIngress(link); err != nil { - return err - } + t := hypervisor.TypedJSONEndpoint{ + Type: ep.Type(), + Data: tempJSON, + } - if err := netHandle.LinkSetDown(link); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + typedEndpoints = append(typedEndpoints, t) } - return nil -} - -func createNetNS() (string, error) { - n, err := ns.NewNS() - if err != nil { - return "", err - } + s.Endpoints = typedEndpoints - return n.Path(), nil + b, err := json.Marshal(s) + return b, err } -// doNetNS is free from any call to a go routine, and it calls -// into runtime.LockOSThread(), meaning it won't be executed in a -// different thread than the one expected by the caller. -func doNetNS(netNSPath string, cb func(ns.NetNS) error) error { - // if netNSPath is empty, the callback function will be run in the current network namespace. - // So skip the whole function, just call cb(). cb() needs a NetNS as arg but ignored, give it a fake one. - if netNSPath == "" { - var netNs ns.NetNS - return cb(netNs) - } - - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - currentNS, err := ns.GetCurrentNS() - if err != nil { - return err - } - defer currentNS.Close() - - targetNS, err := ns.GetNS(netNSPath) - if err != nil { - return err +// UnmarshalJSON is the custom NetworkNamespace unmarshalling routine. +// This is needed for unmarshalling the Endpoints interfaces array. +func (n *NetworkNamespace) UnmarshalJSON(b []byte) error { + var s struct { + NetNsPath string + NetNsCreated bool + Endpoints json.RawMessage } - if err := targetNS.Set(); err != nil { + if err := json.Unmarshal(b, &s); err != nil { return err } - defer currentNS.Set() - return cb(targetNS) -} + (*n).NetNsPath = s.NetNsPath + (*n).NetNsCreated = s.NetNsCreated -func deleteNetNS(netNSPath string) error { - n, err := ns.GetNS(netNSPath) - if err != nil { + var typedEndpoints []hypervisor.TypedJSONEndpoint + if err := json.Unmarshal([]byte(string(s.Endpoints)), &typedEndpoints); err != nil { return err } - - err = n.Close() + endpoints, err := hypervisor.UnmarshalEndpoints(typedEndpoints) if err != nil { return err } - if err = unix.Unmount(netNSPath, unix.MNT_DETACH); err != nil { - return fmt.Errorf("Failed to unmount namespace %s: %v", netNSPath, err) - } - if err := os.RemoveAll(netNSPath); err != nil { - return fmt.Errorf("Failed to clean up namespace %s: %v", netNSPath, err) - } - + (*n).Endpoints = endpoints return nil } -func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) { +func (n NetworkNamespace) interfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) { - if networkNS.NetNsPath == "" { + if n.NetNsPath == "" { return nil, nil, nil } var routes []*vcTypes.Route var ifaces []*vcTypes.Interface - for _, endpoint := range networkNS.Endpoints { + for _, endpoint := range n.Endpoints { var ipAddresses []*vcTypes.IPAddress for _, addr := range endpoint.Properties().Addrs { @@ -1237,186 +215,6 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa return ifaces, routes, nil } -func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { - uniqueID := uuid.Generate().String() - - randomMacAddr, err := generateRandomPrivateMacAddr() - if err != nil { - return NetworkInterfacePair{}, fmt.Errorf("Could not generate random mac address: %s", err) - } - - netPair := NetworkInterfacePair{ - TapInterface: TapInterface{ - ID: uniqueID, - Name: fmt.Sprintf("br%d_kata", idx), - TAPIface: NetworkInterface{ - Name: fmt.Sprintf("tap%d_kata", idx), - }, - }, - VirtIface: NetworkInterface{ - Name: fmt.Sprintf("eth%d", idx), - HardAddr: randomMacAddr, - }, - NetInterworkingModel: interworkingModel, - } - - return netPair, nil -} - -func generateRandomPrivateMacAddr() (string, error) { - buf := make([]byte, 6) - _, err := cryptoRand.Read(buf) - if err != nil { - return "", err - } - - // Set the local bit for local addresses - // Addresses in this range are local mac addresses: - // x2-xx-xx-xx-xx-xx , x6-xx-xx-xx-xx-xx , xA-xx-xx-xx-xx-xx , xE-xx-xx-xx-xx-xx - buf[0] = (buf[0] | 2) & 0xfe - - hardAddr := net.HardwareAddr(buf) - return hardAddr.String(), nil -} - -func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo, error) { - addrs, err := handle.AddrList(link, netlink.FAMILY_ALL) - if err != nil { - return NetworkInfo{}, err - } - - routes, err := handle.RouteList(link, netlink.FAMILY_ALL) - if err != nil { - return NetworkInfo{}, err - } - - return NetworkInfo{ - Iface: NetlinkIface{ - LinkAttrs: *(link.Attrs()), - Type: link.Type(), - }, - Addrs: addrs, - Routes: routes, - }, nil -} - -func createEndpointsFromScan(networkNSPath string, config *NetworkConfig) ([]Endpoint, error) { - var endpoints []Endpoint - - netnsHandle, err := netns.GetFromPath(networkNSPath) - if err != nil { - return []Endpoint{}, err - } - defer netnsHandle.Close() - - netlinkHandle, err := netlink.NewHandleAt(netnsHandle) - if err != nil { - return []Endpoint{}, err - } - defer netlinkHandle.Delete() - - linkList, err := netlinkHandle.LinkList() - if err != nil { - return []Endpoint{}, err - } - - idx := 0 - for _, link := range linkList { - var ( - endpoint Endpoint - errCreate error - ) - - netInfo, err := networkInfoFromLink(netlinkHandle, link) - if err != nil { - return []Endpoint{}, err - } - - // Ignore unconfigured network interfaces. These are - // either base tunnel devices that are not namespaced - // like gre0, gretap0, sit0, ipip0, tunl0 or incorrectly - // setup interfaces. - if len(netInfo.Addrs) == 0 { - continue - } - - // Skip any loopback interfaces: - if (netInfo.Iface.Flags & net.FlagLoopback) != 0 { - continue - } - - if err := doNetNS(networkNSPath, func(_ ns.NetNS) error { - endpoint, errCreate = createEndpoint(netInfo, idx, config.InterworkingModel) - return errCreate - }); err != nil { - return []Endpoint{}, err - } - - endpoint.SetProperties(netInfo) - endpoints = append(endpoints, endpoint) - - idx++ - } - - sort.Slice(endpoints, func(i, j int) bool { - return endpoints[i].Name() < endpoints[j].Name() - }) - - networkLogger().WithField("endpoints", endpoints).Info("Endpoints found after scan") - - return endpoints, nil -} - -func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (Endpoint, error) { - var endpoint Endpoint - // TODO: This is the incoming interface - // based on the incoming interface we should create - // an appropriate EndPoint based on interface type - // This should be a switch - - // Check if interface is a physical interface. Do not create - // tap interface/bridge if it is. - isPhysical, err := isPhysicalIface(netInfo.Iface.Name) - if err != nil { - return nil, err - } - - if isPhysical { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") - endpoint, err = createPhysicalEndpoint(netInfo) - } else { - var socketPath string - - // Check if this is a dummy interface which has a vhost-user socket associated with it - socketPath, err = vhostUserSocketPath(netInfo) - if err != nil { - return nil, err - } - - if socketPath != "" { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") - endpoint, err = createVhostUserEndpoint(netInfo, socketPath) - } else if netInfo.Iface.Type == "macvlan" { - networkLogger().Infof("macvlan interface found") - endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) - } else if netInfo.Iface.Type == "macvtap" { - networkLogger().Infof("macvtap interface found") - endpoint, err = createMacvtapNetworkEndpoint(netInfo) - } else if netInfo.Iface.Type == "tap" { - networkLogger().Info("tap interface found") - endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) - } else if netInfo.Iface.Type == "veth" { - endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) - } else if netInfo.Iface.Type == "ipvlan" { - endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) - } else { - return nil, fmt.Errorf("Unsupported network interface") - } - } - - return endpoint, err -} - // Network is the virtcontainer network structure type Network struct { } @@ -1435,30 +233,30 @@ func (n *Network) Run(networkNSPath string, cb func() error) error { span, _ := n.trace(context.Background(), "run") defer span.Finish() - return doNetNS(networkNSPath, func(_ ns.NetNS) error { + return hypervisor.DoNetNS(networkNSPath, func(_ ns.NetNS) error { return cb() }) } // Add adds all needed interfaces inside the network namespace. -func (n *Network) Add(ctx context.Context, config *NetworkConfig, hypervisor hypervisor, hotplug bool) ([]Endpoint, error) { +func (n *Network) Add(ctx context.Context, config *types.NetworkConfig, h hypervisor.Hypervisor, hotplug bool) ([]hypervisor.Endpoint, error) { span, _ := n.trace(ctx, "add") defer span.Finish() - endpoints, err := createEndpointsFromScan(config.NetNSPath, config) + endpoints, err := hypervisor.CreateEndpointsFromScan(config.NetNSPath, config) if err != nil { return endpoints, err } - err = doNetNS(config.NetNSPath, func(_ ns.NetNS) error { + err = hypervisor.DoNetNS(config.NetNSPath, func(_ ns.NetNS) error { for _, endpoint := range endpoints { networkLogger().WithField("endpoint-type", endpoint.Type()).WithField("hotplug", hotplug).Info("Attaching endpoint") if hotplug { - if err := endpoint.HotAttach(hypervisor); err != nil { + if err := endpoint.HotAttach(h); err != nil { return err } } else { - if err := endpoint.Attach(hypervisor); err != nil { + if err := endpoint.Attach(h); err != nil { return err } } @@ -1467,7 +265,7 @@ func (n *Network) Add(ctx context.Context, config *NetworkConfig, hypervisor hyp return nil }) if err != nil { - return []Endpoint{}, err + return []hypervisor.Endpoint{}, err } networkLogger().Debug("Network added") @@ -1477,7 +275,7 @@ func (n *Network) Add(ctx context.Context, config *NetworkConfig, hypervisor hyp // Remove network endpoints in the network namespace. It also deletes the network // namespace in case the namespace has been created by us. -func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor hypervisor, hotunplug bool) error { +func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, h hypervisor.Hypervisor, hotunplug bool) error { span, _ := n.trace(ctx, "remove") defer span.Finish() @@ -1486,7 +284,7 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h // if required. networkLogger().WithField("endpoint-type", endpoint.Type()).WithField("hotunplug", hotunplug).Info("Detaching endpoint") if hotunplug { - if err := endpoint.HotDetach(hypervisor, ns.NetNsCreated, ns.NetNsPath); err != nil { + if err := endpoint.HotDetach(h, ns.NetNsCreated, ns.NetNsPath); err != nil { return err } } else { diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go index c1d4804f45..3f6bb5f2d7 100644 --- a/virtcontainers/network_test.go +++ b/virtcontainers/network_test.go @@ -11,7 +11,9 @@ import ( "reflect" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" + "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" ) @@ -63,8 +65,8 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {LinkIndex: 329, Dst: dst2, Src: src2, Gw: gw2}, } - networkInfo := NetworkInfo{ - Iface: NetlinkIface{ + networkInfo := types.NetworkInfo{ + Iface: types.NetlinkIface{ LinkAttrs: netlink.LinkAttrs{MTU: 1500}, Type: "", }, @@ -72,17 +74,17 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { Routes: routes, } - ep0 := &PhysicalEndpoint{ + ep0 := &hypervisor.PhysicalEndpoint{ IfaceName: "eth0", HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), EndpointProperties: networkInfo, } - endpoints := []Endpoint{ep0} + endpoints := []hypervisor.Endpoint{ep0} nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints} - resInterfaces, resRoutes, err := generateInterfacesAndRoutes(nns) + resInterfaces, resRoutes, err := nns.interfacesAndRoutes(nns) // // Build expected results: @@ -108,199 +110,3 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { "Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes) } - -func TestNetInterworkingModelIsValid(t *testing.T) { - tests := []struct { - name string - n NetInterworkingModel - want bool - }{ - {"Invalid Model", NetXConnectInvalidModel, false}, - {"Default Model", NetXConnectDefaultModel, true}, - {"Bridged Model", NetXConnectBridgedModel, true}, - {"TC Filter Model", NetXConnectTCFilterModel, true}, - {"Macvtap Model", NetXConnectMacVtapModel, true}, - {"Enlightened Model", NetXConnectEnlightenedModel, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.n.IsValid(); got != tt.want { - t.Errorf("NetInterworkingModel.IsValid() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestNetInterworkingModelSetModel(t *testing.T) { - var n NetInterworkingModel - tests := []struct { - name string - modelName string - wantErr bool - }{ - {"Invalid Model", "Invalid", true}, - {"default Model", defaultNetModelStr, false}, - {"bridged Model", bridgedNetModelStr, false}, - {"macvtap Model", macvtapNetModelStr, false}, - {"enlightened Model", enlightenedNetModelStr, false}, - {"tcfilter Model", tcFilterNetModelStr, false}, - {"none Model", noneNetModelStr, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := n.SetModel(tt.modelName); (err != nil) != tt.wantErr { - t.Errorf("NetInterworkingModel.SetModel() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestGenerateRandomPrivateMacAdd(t *testing.T) { - assert := assert.New(t) - - addr1, err := generateRandomPrivateMacAddr() - assert.NoError(err) - - _, err = net.ParseMAC(addr1) - assert.NoError(err) - - addr2, err := generateRandomPrivateMacAddr() - assert.NoError(err) - - _, err = net.ParseMAC(addr2) - assert.NoError(err) - - assert.NotEqual(addr1, addr2) -} - -func TestCreateGetBridgeLink(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - defer netHandle.Delete() - - assert.NoError(err) - - brName := "testbr0" - brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) - assert.NoError(err) - assert.NotNil(brLink) - - brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) - assert.NoError(err) - - err = netHandle.LinkDel(brLink) - assert.NoError(err) -} - -func TestCreateGetTunTapLink(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - defer netHandle.Delete() - - assert.NoError(err) - - tapName := "testtap0" - tapLink, fds, err := createLink(netHandle, tapName, &netlink.Tuntap{}, 1) - assert.NoError(err) - assert.NotNil(tapLink) - assert.NotZero(len(fds)) - - tapLink, err = getLinkByName(netHandle, tapName, &netlink.Tuntap{}) - assert.NoError(err) - - err = netHandle.LinkDel(tapLink) - assert.NoError(err) -} - -func TestCreateMacVtap(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - defer netHandle.Delete() - - assert.NoError(err) - - brName := "testbr0" - brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) - assert.NoError(err) - - attrs := brLink.Attrs() - - mcLink := &netlink.Macvtap{ - Macvlan: netlink.Macvlan{ - LinkAttrs: netlink.LinkAttrs{ - TxQLen: attrs.TxQLen, - ParentIndex: attrs.Index, - }, - }, - } - - macvtapName := "testmc0" - _, err = createMacVtap(netHandle, macvtapName, mcLink, 1) - assert.NoError(err) - - macvtapLink, err := getLinkByName(netHandle, macvtapName, &netlink.Macvtap{}) - assert.NoError(err) - - err = netHandle.LinkDel(macvtapLink) - assert.NoError(err) - - brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) - assert.NoError(err) - - err = netHandle.LinkDel(brLink) - assert.NoError(err) -} - -func TestTcRedirectNetwork(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - assert.NoError(err) - defer netHandle.Delete() - - // Create a test veth interface. - vethName := "foo" - veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: vethName, TxQLen: 200, MTU: 1400}, PeerName: "bar"} - - err = netlink.LinkAdd(veth) - assert.NoError(err) - - endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel) - assert.NoError(err) - - link, err := netlink.LinkByName(vethName) - assert.NoError(err) - - err = netHandle.LinkSetUp(link) - assert.NoError(err) - - err = setupTCFiltering(endpoint, 1, true) - assert.NoError(err) - - err = removeTCFiltering(endpoint) - assert.NoError(err) - - // Remove the veth created for testing. - err = netHandle.LinkDel(link) - assert.NoError(err) -} diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go index 6ba15d4b33..c8ca3b1b97 100644 --- a/virtcontainers/noop_agent.go +++ b/virtcontainers/noop_agent.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/types" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -167,7 +168,7 @@ func (n *noopAgent) resumeContainer(sandbox *Sandbox, c Container) error { } // configHypervisor is the Noop agent hypervisor configuration implementation. It does nothing. -func (n *noopAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error { +func (n *noopAgent) configure(h hypervisor.Hypervisor, id, sharePath string, builtin bool, config interface{}) error { return nil } diff --git a/virtcontainers/noop_agent_test.go b/virtcontainers/noop_agent_test.go index ad471f4647..6a0aae6cbf 100644 --- a/virtcontainers/noop_agent_test.go +++ b/virtcontainers/noop_agent_test.go @@ -10,6 +10,7 @@ import ( "context" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) @@ -166,7 +167,7 @@ func TestNoopAgentResumeContainer(t *testing.T) { func TestNoopAgentConfigure(t *testing.T) { n := &noopAgent{} - h := &mockHypervisor{} + h := hypervisor.NewMock() id := "foobar" sharePath := "foobarDir" err := n.configure(h, id, sharePath, true, nil) diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 4ffb4ec957..06557ded1d 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -23,6 +23,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" "github.com/kata-containers/runtime/virtcontainers/types" @@ -100,8 +101,8 @@ type FactoryConfig struct { // RuntimeConfig aggregates all runtime specific settings type RuntimeConfig struct { - HypervisorType vc.HypervisorType - HypervisorConfig vc.HypervisorConfig + HypervisorType hypervisor.Type + HypervisorConfig hypervisor.Config NetmonConfig vc.NetmonConfig @@ -118,7 +119,7 @@ type RuntimeConfig struct { //Determines how the VM should be connected to the //the container network interface - InterNetworkModel vc.NetInterworkingModel + InterNetworkModel types.NetInterworkingModel FactoryConfig FactoryConfig Debug bool Trace bool @@ -132,7 +133,7 @@ type RuntimeConfig struct { // AddKernelParam allows the addition of new kernel parameters to an existing // hypervisor configuration stored inside the current runtime configuration. -func (config *RuntimeConfig) AddKernelParam(p vc.Param) error { +func (config *RuntimeConfig) AddKernelParam(p hypervisor.Param) error { return config.HypervisorConfig.AddKernelParam(p) } @@ -315,13 +316,13 @@ func ContainerCapabilities(s CompatOCISpec) (types.LinuxCapabilities, error) { return containerCapabilities(s) } -func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.NetworkConfig, error) { +func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (types.NetworkConfig, error) { linux := ocispec.Linux if linux == nil { - return vc.NetworkConfig{}, ErrNoLinux + return types.NetworkConfig{}, ErrNoLinux } - var netConf vc.NetworkConfig + var netConf types.NetworkConfig for _, n := range linux.Namespaces { if n.Type != spec.NetworkNamespace { @@ -335,7 +336,7 @@ func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.NetworkConfi netConf.InterworkingModel = config.InterNetworkModel netConf.DisableNewNetNs = config.DisableNewNetNs - netConf.NetmonConfig = vc.NetmonConfig{ + netConf.NetmonConfig = types.NetmonConfig{ Path: config.NetmonConfig.Path, Debug: config.NetmonConfig.Debug, Enable: config.NetmonConfig.Enable, diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index ff1daa9b64..3f367d0816 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -23,6 +23,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -106,7 +107,7 @@ func TestMinimalSandboxConfig(t *testing.T) { }() runtimeConfig := RuntimeConfig{ - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, AgentType: vc.HyperstartAgent, ProxyType: vc.CCProxyType, ShimType: vc.CCShimType, @@ -215,13 +216,13 @@ func TestMinimalSandboxConfig(t *testing.T) { }}, } - expectedNetworkConfig := vc.NetworkConfig{} + expectedNetworkConfig := types.NetworkConfig{} expectedSandboxConfig := vc.SandboxConfig{ ID: containerID, Hostname: "testHostname", - HypervisorType: vc.QemuHypervisor, + HypervisorType: hypervisor.Qemu, AgentType: vc.HyperstartAgent, ProxyType: vc.CCProxyType, ShimType: vc.CCShimType, @@ -623,7 +624,7 @@ func TestSandboxIDFailure(t *testing.T) { func TestAddKernelParamValid(t *testing.T) { var config RuntimeConfig - expected := []vc.Param{ + expected := []hypervisor.Param{ { Key: "foo", Value: "bar", @@ -639,7 +640,7 @@ func TestAddKernelParamValid(t *testing.T) { func TestAddKernelParamInvalid(t *testing.T) { var config RuntimeConfig - invalid := []vc.Param{ + invalid := []hypervisor.Param{ { Key: "", Value: "bar", diff --git a/virtcontainers/pkg/vcmock/mock_test.go b/virtcontainers/pkg/vcmock/mock_test.go index d369e6574a..84f77e9f2b 100644 --- a/virtcontainers/pkg/vcmock/mock_test.go +++ b/virtcontainers/pkg/vcmock/mock_test.go @@ -13,6 +13,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/sirupsen/logrus" @@ -711,12 +712,12 @@ func TestVCMockSetVMFactory(t *testing.T) { m := &VCMock{} assert.Nil(m.SetFactoryFunc) - hyperConfig := vc.HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: "foobar", ImagePath: "foobar", } vmConfig := vc.VMConfig{ - HypervisorType: vc.MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: vc.NoopAgentType, HypervisorConfig: hyperConfig, } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 1e1e9fc6bf..c3a3a11187 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -24,6 +24,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -41,8 +42,8 @@ const ( type SandboxStatus struct { ID string State types.State - Hypervisor HypervisorType - HypervisorConfig HypervisorConfig + Hypervisor hypervisor.Type + HypervisorConfig hypervisor.Config Agent AgentType ContainersStatus []ContainerStatus @@ -58,8 +59,8 @@ type SandboxConfig struct { Hostname string - HypervisorType HypervisorType - HypervisorConfig HypervisorConfig + HypervisorType hypervisor.Type + HypervisorConfig hypervisor.Config AgentType AgentType AgentConfig interface{} @@ -70,7 +71,7 @@ type SandboxConfig struct { ShimType ShimType ShimConfig interface{} - NetworkConfig NetworkConfig + NetworkConfig types.NetworkConfig // Volumes is a list of shared volumes between the host and the Sandbox. Volumes []types.Volume @@ -130,8 +131,8 @@ func (sandboxConfig *SandboxConfig) valid() bool { return false } - if _, err := newHypervisor(sandboxConfig.HypervisorType); err != nil { - sandboxConfig.HypervisorType = QemuHypervisor + if _, err := hypervisor.New(sandboxConfig.HypervisorType); err != nil { + sandboxConfig.HypervisorType = hypervisor.Qemu } return true @@ -144,7 +145,7 @@ type Sandbox struct { sync.Mutex factory Factory - hypervisor hypervisor + hypervisor hypervisor.Hypervisor agent agent store *store.VCStore network Network @@ -257,7 +258,7 @@ func (s *Sandbox) Release() error { if s.monitor != nil { s.monitor.stop() } - s.hypervisor.disconnect() + s.hypervisor.Disconnect() return s.agent.disconnect() } @@ -391,7 +392,7 @@ func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error { } for _, a := range []*types.Asset{kernel, image, initrd} { - if err := sandboxConfig.HypervisorConfig.addCustomAsset(a); err != nil { + if err := sandboxConfig.HypervisorConfig.AddCustomAsset(a); err != nil { return err } } @@ -484,7 +485,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor agent := newAgent(sandboxConfig.AgentType) - hypervisor, err := newHypervisor(sandboxConfig.HypervisorType) + hypervisor, err := hypervisor.New(sandboxConfig.HypervisorType) if err != nil { return nil, err } @@ -532,7 +533,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor } }() - if err = s.hypervisor.createSandbox(ctx, s.id, &sandboxConfig.HypervisorConfig, s.store); err != nil { + if err = s.hypervisor.CreateSandbox(ctx, s.id, &sandboxConfig.HypervisorConfig, s.store); err != nil { return nil, err } @@ -714,7 +715,7 @@ func (s *Sandbox) Delete() error { s.monitor.stop() } - if err := s.hypervisor.cleanup(); err != nil { + if err := s.hypervisor.Cleanup(); err != nil { s.Logger().WithError(err).Error("failed to cleanup hypervisor") } @@ -806,10 +807,10 @@ func (s *Sandbox) removeNetwork() error { return s.network.Remove(s.ctx, &s.networkNS, s.hypervisor, s.factory != nil) } -func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { +func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (types.NetworkInfo, error) { hw, err := net.ParseMAC(inf.HwAddr) if err != nil { - return NetworkInfo{}, err + return types.NetworkInfo{}, err } var addrs []netlink.Addr @@ -817,14 +818,14 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask) netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr) if err != nil { - return NetworkInfo{}, fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err) + return types.NetworkInfo{}, fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err) } addrs = append(addrs, *netlinkAddr) } - return NetworkInfo{ - Iface: NetlinkIface{ + return types.NetworkInfo{ + Iface: types.NetlinkIface{ LinkAttrs: netlink.LinkAttrs{ Name: inf.Name, HardwareAddr: hw, @@ -843,13 +844,13 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro return nil, err } - endpoint, err := createEndpoint(netInfo, len(s.networkNS.Endpoints), s.config.NetworkConfig.InterworkingModel) + endpoint, err := hypervisor.CreateEndpoint(netInfo, len(s.networkNS.Endpoints), s.config.NetworkConfig.InterworkingModel) if err != nil { return nil, err } endpoint.SetProperties(netInfo) - if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { + if err := hypervisor.DoNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint") return endpoint.HotAttach(s.hypervisor) }); err != nil { @@ -927,7 +928,7 @@ func (s *Sandbox) startVM() error { return nil } - return s.hypervisor.startSandbox(vmStartTimeout) + return s.hypervisor.StartSandbox(vmStartTimeout) }); err != nil { return err } @@ -978,7 +979,7 @@ func (s *Sandbox) stopVM() error { } s.Logger().Info("Stopping VM") - return s.hypervisor.stopSandbox() + return s.hypervisor.StopSandbox() } func (s *Sandbox) addContainer(c *Container) error { @@ -1323,13 +1324,13 @@ func (s *Sandbox) Stop() error { return err } - // Remove the network. + // Remove the hypervisor. return s.removeNetwork() } // Pause pauses the sandbox func (s *Sandbox) Pause() error { - if err := s.hypervisor.pauseSandbox(); err != nil { + if err := s.hypervisor.PauseSandbox(); err != nil { return err } @@ -1346,7 +1347,7 @@ func (s *Sandbox) Pause() error { // Resume resumes the sandbox func (s *Sandbox) Resume() error { - if err := s.hypervisor.resumeSandbox(); err != nil { + if err := s.hypervisor.ResumeSandbox(); err != nil { return err } @@ -1500,7 +1501,7 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) // adding a group of VFIO devices for _, dev := range vfioDevices { - if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil { + if _, err := s.hypervisor.HotplugAddDevice(dev, hypervisor.VfioDev); err != nil { s.Logger(). WithFields(logrus.Fields{ "sandbox": s.id, @@ -1516,7 +1517,7 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) if !ok { return fmt.Errorf("device type mismatch, expect device type to be %s", devType) } - _, err := s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev) + _, err := s.hypervisor.HotplugAddDevice(blockDevice.BlockDrive, hypervisor.BlockDev) return err case config.DeviceGeneric: // TODO: what? @@ -1537,7 +1538,7 @@ func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceTy // remove a group of VFIO devices for _, dev := range vfioDevices { - if _, err := s.hypervisor.hotplugRemoveDevice(dev, vfioDev); err != nil { + if _, err := s.hypervisor.HotplugRemoveDevice(dev, hypervisor.VfioDev); err != nil { s.Logger().WithError(err). WithFields(logrus.Fields{ "sandbox": s.id, @@ -1553,7 +1554,7 @@ func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceTy if !ok { return fmt.Errorf("device type mismatch, expect device type to be %s", devType) } - _, err := s.hypervisor.hotplugRemoveDevice(blockDrive, blockDev) + _, err := s.hypervisor.HotplugRemoveDevice(blockDrive, hypervisor.BlockDev) return err case config.DeviceGeneric: // TODO: what? @@ -1580,7 +1581,7 @@ func (s *Sandbox) DecrementSandboxBlockIndex() error { func (s *Sandbox) AppendDevice(device api.Device) error { switch device.DeviceType() { case config.VhostUserSCSI, config.VhostUserNet, config.VhostUserBlk: - return s.hypervisor.addDevice(device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), vhostuserDev) + return s.hypervisor.AddDevice(device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), hypervisor.VhostuserDev) } return fmt.Errorf("unsupported device type") } @@ -1634,14 +1635,14 @@ func (s *Sandbox) updateResources() error { } sandboxVCPUs := uint32(utils.ConstraintsToVCPUs(*sumResources.CPU.Quota, *sumResources.CPU.Period)) - sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs + sandboxVCPUs += s.hypervisor.Config().NumVCPUs - sandboxMemoryByte := int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift + sandboxMemoryByte := int64(s.hypervisor.Config().MemorySize) << utils.MibToBytesShift sandboxMemoryByte += *sumResources.Memory.Limit // Update VCPUs s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs") - oldCPUs, newCPUs, err := s.hypervisor.resizeVCPUs(sandboxVCPUs) + oldCPUs, newCPUs, err := s.hypervisor.ResizeVCPUs(sandboxVCPUs) if err != nil { return err } @@ -1656,7 +1657,7 @@ func (s *Sandbox) updateResources() error { // Update Memory s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") - newMemory, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB) + newMemory, err := s.hypervisor.ResizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB) if err != nil { return err } diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index e5b361b035..bd2f58a94d 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -23,14 +23,15 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "golang.org/x/sys/unix" ) -func newHypervisorConfig(kernelParams []Param, hParams []Param) HypervisorConfig { - return HypervisorConfig{ +func newHypervisorConfig(kernelParams []hypervisor.Param, hParams []hypervisor.Param) hypervisor.Config { + return hypervisor.Config{ KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), @@ -41,8 +42,8 @@ func newHypervisorConfig(kernelParams []Param, hParams []Param) HypervisorConfig } func testCreateSandbox(t *testing.T, id string, - htype HypervisorType, hconfig HypervisorConfig, atype AgentType, - nconfig NetworkConfig, containers []ContainerConfig, + htype hypervisor.Type, hconfig hypervisor.Config, atype AgentType, + nconfig types.NetworkConfig, containers []ContainerConfig, volumes []types.Volume) (*Sandbox, error) { sconfig := SandboxConfig{ @@ -80,7 +81,7 @@ func testCreateSandbox(t *testing.T, id string, } func TestCreateEmptySandbox(t *testing.T) { - _, err := testCreateSandbox(t, testSandboxID, MockHypervisor, HypervisorConfig{}, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hypervisor.Config{}, NoopAgentType, types.NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("VirtContainers should not allow empty sandboxes") } @@ -88,7 +89,7 @@ func TestCreateEmptySandbox(t *testing.T) { } func TestCreateEmptyHypervisorSandbox(t *testing.T) { - _, err := testCreateSandbox(t, testSandboxID, QemuHypervisor, HypervisorConfig{}, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Qemu, hypervisor.Config{}, NoopAgentType, types.NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("VirtContainers should not allow sandboxes with empty hypervisors") } @@ -98,7 +99,7 @@ func TestCreateEmptyHypervisorSandbox(t *testing.T) { func TestCreateMockSandbox(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - _, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + _, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err != nil { t.Fatal(err) } @@ -108,7 +109,7 @@ func TestCreateMockSandbox(t *testing.T) { func TestCreateSandboxEmptyID(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, "", MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, "", hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err == nil { t.Fatalf("Expected sandbox with empty ID to fail, but got sandbox %v", p) } @@ -118,7 +119,7 @@ func TestCreateSandboxEmptyID(t *testing.T) { func testSandboxStateTransition(t *testing.T, state types.StateString, newState types.StateString) error { hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err != nil { return err } @@ -450,7 +451,7 @@ func TestSandboxSetSandboxAndContainerState(t *testing.T) { hConfig := newHypervisorConfig(nil, nil) // create a sandbox - p, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, []ContainerConfig{contConfig}, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, []ContainerConfig{contConfig}, nil) if err != nil { t.Fatal(err) } @@ -648,7 +649,7 @@ func TestSandboxGetContainer(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - p, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + p, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, nil, nil) if err != nil { t.Fatal(err) } @@ -694,7 +695,7 @@ func TestContainerSetStateBlockIndex(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - sandbox, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) + sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, containers, nil) if err != nil { t.Fatal(err) } @@ -789,7 +790,7 @@ func TestContainerStateSetFstype(t *testing.T) { } hConfig := newHypervisorConfig(nil, nil) - sandbox, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, containers, nil) + sandbox, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, hConfig, NoopAgentType, types.NetworkConfig{}, containers, nil) if err != nil { t.Fatal(err) } @@ -927,7 +928,7 @@ func TestSandboxAttachDevicesVFIO(t *testing.T) { sandbox := Sandbox{ id: "100", containers: containers, - hypervisor: &mockHypervisor{}, + hypervisor: hypervisor.NewMock(), devManager: dm, ctx: context.Background(), } @@ -967,7 +968,7 @@ func TestSandboxCreateAssets(t *testing.T) { originalKernelPath := filepath.Join(testDir, testKernel) - hc := HypervisorConfig{ + hc := hypervisor.Config{ KernelPath: originalKernelPath, ImagePath: filepath.Join(testDir, testImage), } @@ -984,9 +985,9 @@ func TestSandboxCreateAssets(t *testing.T) { err = createAssets(context.Background(), p) assert.Nil(err) - a, ok := p.HypervisorConfig.customAssets[types.KernelAsset] - assert.True(ok) - assert.Equal(a.Path(), tmpfile.Name()) + path, err := p.HypervisorConfig.KernelAssetPath() + assert.Nil(err) + assert.Equal(path, tmpfile.Name()) p = &SandboxConfig{ Annotations: map[string]string{ @@ -1061,7 +1062,7 @@ func TestRemoveContainerSuccess(t *testing.T) { } func TestCreateContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1072,7 +1073,7 @@ func TestCreateContainer(t *testing.T) { } func TestDeleteContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1089,7 +1090,7 @@ func TestDeleteContainer(t *testing.T) { } func TestStartContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1109,7 +1110,7 @@ func TestStartContainer(t *testing.T) { } func TestStatusContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1129,7 +1130,7 @@ func TestStatusContainer(t *testing.T) { } func TestStatusSandbox(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1137,7 +1138,7 @@ func TestStatusSandbox(t *testing.T) { } func TestEnterContainer(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1161,7 +1162,7 @@ func TestEnterContainer(t *testing.T) { } func TestMonitor(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1181,7 +1182,7 @@ func TestMonitor(t *testing.T) { } func TestWaitProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1211,7 +1212,7 @@ func TestWaitProcess(t *testing.T) { } func TestSignalProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1241,7 +1242,7 @@ func TestSignalProcess(t *testing.T) { } func TestWinsizeProcess(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1271,7 +1272,7 @@ func TestWinsizeProcess(t *testing.T) { } func TestContainerProcessIOStream(t *testing.T) { - s, err := testCreateSandbox(t, testSandboxID, MockHypervisor, newHypervisorConfig(nil, nil), NoopAgentType, NetworkConfig{}, nil, nil) + s, err := testCreateSandbox(t, testSandboxID, hypervisor.Mock, newHypervisorConfig(nil, nil), NoopAgentType, types.NetworkConfig{}, nil, nil) assert.Nil(t, err, "VirtContainers should not allow empty sandboxes") defer cleanUp() @@ -1301,9 +1302,9 @@ func TestContainerProcessIOStream(t *testing.T) { } func TestAttachBlockDevice(t *testing.T) { - hypervisor := &mockHypervisor{} + h := hypervisor.NewMock() - hConfig := HypervisorConfig{ + hConfig := hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, } @@ -1313,7 +1314,7 @@ func TestAttachBlockDevice(t *testing.T) { sandbox := &Sandbox{ id: testSandboxID, - hypervisor: hypervisor, + hypervisor: h, config: sconfig, ctx: context.Background(), } @@ -1389,9 +1390,9 @@ func TestAttachBlockDevice(t *testing.T) { } func TestPreAddDevice(t *testing.T) { - hypervisor := &mockHypervisor{} + h := hypervisor.NewMock() - hConfig := HypervisorConfig{ + hConfig := hypervisor.Config{ BlockDeviceDriver: config.VirtioBlock, } @@ -1403,7 +1404,7 @@ func TestPreAddDevice(t *testing.T) { // create a sandbox first sandbox := &Sandbox{ id: testSandboxID, - hypervisor: hypervisor, + hypervisor: h, config: sconfig, devManager: dm, ctx: context.Background(), @@ -1497,8 +1498,8 @@ func TestStartNetworkMonitor(t *testing.T) { s := &Sandbox{ id: testSandboxID, config: &SandboxConfig{ - NetworkConfig: NetworkConfig{ - NetmonConfig: NetmonConfig{ + NetworkConfig: types.NetworkConfig{ + NetmonConfig: types.NetmonConfig{ Path: trueBinPath, }, }, diff --git a/virtcontainers/types/network.go b/virtcontainers/types/network.go new file mode 100644 index 0000000000..4bcc4d1f33 --- /dev/null +++ b/virtcontainers/types/network.go @@ -0,0 +1,195 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package types + +import ( + "fmt" + "os" + + "github.com/vishvananda/netlink" +) + +// NetInterworkingModel defines the network model connecting +// the network interface to the virtual machine. +type NetInterworkingModel int + +const ( + // NetXConnectDefaultModel Ask to use DefaultNetInterworkingModel + NetXConnectDefaultModel NetInterworkingModel = iota + + // NetXConnectBridgedModel uses a linux bridge to interconnect + // the container interface to the VM. This is the + // safe default that works for most cases except + // macvlan and ipvlan + NetXConnectBridgedModel + + // NetXConnectMacVtapModel can be used when the Container network + // interface can be bridged using macvtap + NetXConnectMacVtapModel + + // NetXConnectEnlightenedModel can be used when the Network plugins + // are enlightened to create VM native interfaces + // when requested by the runtime + // This will be used for vethtap, macvtap, ipvtap + NetXConnectEnlightenedModel + + // NetXConnectTCFilterModel redirects traffic from the network interface + // provided by the network plugin to a tap interface. + // This works for ipvlan and macvlan as well. + NetXConnectTCFilterModel + + // NetXConnectNoneModel can be used when the VM is in the host network namespace + NetXConnectNoneModel + + // NetXConnectInvalidModel is the last item to check valid values by IsValid() + NetXConnectInvalidModel +) + +// DefaultNetInterworkingModel is a package level default +// that determines how the VM should be connected to the +// the container network interface +var DefaultNetInterworkingModel = NetXConnectMacVtapModel + +//IsValid checks if a model is valid +func (n NetInterworkingModel) IsValid() bool { + return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel) +} + +const ( + // DefaultNetModelStr is the default networking model. + DefaultNetModelStr = "default" + + // BridgedNetModelStr is the bridged networking model. + BridgedNetModelStr = "bridged" + + // MacvtapNetModelStr is the MacVTap networking model. + MacvtapNetModelStr = "macvtap" + + // EnlightenedNetModelStr is the VM enlightened networking model. + EnlightenedNetModelStr = "enlightened" + + // TcFilterNetModelStr is the Traffic Control filtering networking model. + TcFilterNetModelStr = "tcfilter" + + // NoneNetModelStr is the no network networking model. + NoneNetModelStr = "none" +) + +//SetModel change the model string value +func (n *NetInterworkingModel) SetModel(modelName string) error { + switch modelName { + case DefaultNetModelStr: + *n = DefaultNetInterworkingModel + return nil + case BridgedNetModelStr: + *n = NetXConnectBridgedModel + return nil + case MacvtapNetModelStr: + *n = NetXConnectMacVtapModel + return nil + case EnlightenedNetModelStr: + *n = NetXConnectEnlightenedModel + return nil + case TcFilterNetModelStr: + *n = NetXConnectTCFilterModel + return nil + case NoneNetModelStr: + *n = NetXConnectNoneModel + return nil + } + return fmt.Errorf("Unknown type %s", modelName) +} + +// Introduces constants related to networking +const ( + DefaultRouteDest = "0.0.0.0/0" + DefaultRouteLabel = "default" + DefaultFilePerms = 0600 + DefaultQlen = 1500 +) + +// DNSInfo describes the DNS setup related to a network interface. +type DNSInfo struct { + Servers []string + Domain string + Searches []string + Options []string +} + +// NetlinkIface describes fully a network interface. +type NetlinkIface struct { + netlink.LinkAttrs + Type string +} + +// NetworkInfo gathers all information related to a network interface. +// It can be used to store the description of the underlying network. +type NetworkInfo struct { + Iface NetlinkIface + Addrs []netlink.Addr + Routes []netlink.Route + DNS DNSInfo +} + +// NetworkInterface defines a network interface. +type NetworkInterface struct { + Name string + HardAddr string + Addrs []netlink.Addr +} + +// TapInterface defines a tap interface +type TapInterface struct { + ID string + Name string + TAPIface NetworkInterface + VMFds []*os.File + VhostFds []*os.File +} + +// NetworkInterfacePair defines a pair between VM and virtual network interfaces. +type NetworkInterfacePair struct { + TapInterface + VirtIface NetworkInterface + NetInterworkingModel +} + +// NetworkConfig is the network configuration related to a network. +type NetworkConfig struct { + NetNSPath string + NetNsCreated bool + DisableNewNetNs bool + NetmonConfig NetmonConfig + InterworkingModel NetInterworkingModel +} + +// NetmonConfig is the structure providing specific configuration +// for the network monitor. +type NetmonConfig struct { + Path string + Debug bool + Enable bool +} + +// VSOCKSocketScheme is the URL scheme for a vsock URL (vsock://). +var VSOCKSocketScheme = "vsock" + +// VSOCK represents a Linux virtual Socket (AF_VSOCK). +type VSOCK struct { + // ContextID is the VSOCK context identifier. + ContextID uint64 + + // Port is the VSOCK port. + Port uint32 + + // VHostFd is the vhost file descriptor backing the VSOCK. + VHostFd *os.File +} + +// String formats a VSOCK into its URL. +func (s *VSOCK) String() string { + return fmt.Sprintf("%s://%d:%d", VSOCKSocketScheme, s.ContextID, s.Port) +} diff --git a/virtcontainers/types/network_test.go b/virtcontainers/types/network_test.go new file mode 100644 index 0000000000..9af0de0da8 --- /dev/null +++ b/virtcontainers/types/network_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package types + +import ( + "testing" +) + +func TestNetInterworkingModelIsValid(t *testing.T) { + tests := []struct { + name string + n NetInterworkingModel + want bool + }{ + {"Invalid Model", NetXConnectInvalidModel, false}, + {"Default Model", NetXConnectDefaultModel, true}, + {"Bridged Model", NetXConnectBridgedModel, true}, + {"TC Filter Model", NetXConnectTCFilterModel, true}, + {"Macvtap Model", NetXConnectMacVtapModel, true}, + {"Enlightened Model", NetXConnectEnlightenedModel, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.n.IsValid(); got != tt.want { + t.Errorf("NetInterworkingModel.IsValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNetInterworkingModelSetModel(t *testing.T) { + var n NetInterworkingModel + tests := []struct { + name string + modelName string + wantErr bool + }{ + {"Invalid Model", "Invalid", true}, + {"default Model", DefaultNetModelStr, false}, + {"bridged Model", BridgedNetModelStr, false}, + {"macvtap Model", MacvtapNetModelStr, false}, + {"enlightened Model", EnlightenedNetModelStr, false}, + {"tcfilter Model", TcFilterNetModelStr, false}, + {"none Model", NoneNetModelStr, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := n.SetModel(tt.modelName); (err != nil) != tt.wantErr { + t.Errorf("NetInterworkingModel.SetModel() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index e04f521727..a023c6f3d5 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -11,6 +11,7 @@ import ( "path/filepath" "time" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/sirupsen/logrus" @@ -20,7 +21,7 @@ import ( type VM struct { id string - hypervisor hypervisor + hypervisor hypervisor.Hypervisor agent agent proxy proxy @@ -37,8 +38,8 @@ type VM struct { // VMConfig is a collection of all info that a new blackbox VM needs. type VMConfig struct { - HypervisorType HypervisorType - HypervisorConfig HypervisorConfig + HypervisorType hypervisor.Type + HypervisorConfig hypervisor.Config AgentType AgentType AgentConfig interface{} @@ -49,11 +50,11 @@ type VMConfig struct { // Valid check VMConfig validity. func (c *VMConfig) Valid() error { - return c.HypervisorConfig.valid() + return c.HypervisorConfig.Valid() } -func setupProxy(h hypervisor, agent agent, config VMConfig, id string) (int, string, proxy, error) { - consoleURL, err := h.getSandboxConsole(id) +func setupProxy(h hypervisor.Hypervisor, agent agent, config VMConfig, id string) (int, string, proxy, error) { + consoleURL, err := h.GetSandboxConsole(id) if err != nil { return -1, "", nil, err } @@ -102,7 +103,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { ) // 1. setup hypervisor - hypervisor, err := newHypervisor(config.HypervisorType) + hypervisor, err := hypervisor.New(config.HypervisorType) if err != nil { return nil, err } @@ -130,7 +131,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { } }() - if err = hypervisor.createSandbox(ctx, id, &config.HypervisorConfig, vcStore); err != nil { + if err = hypervisor.CreateSandbox(ctx, id, &config.HypervisorConfig, vcStore); err != nil { return nil, err } @@ -143,14 +144,14 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { } // 3. boot up guest vm - if err = hypervisor.startSandbox(vmStartTimeout); err != nil { + if err = hypervisor.StartSandbox(vmStartTimeout); err != nil { return nil, err } defer func() { if err != nil { virtLog.WithField("vm", id).WithError(err).Info("clean up vm") - hypervisor.stopSandbox() + hypervisor.StopSandbox() } }() @@ -203,25 +204,25 @@ func (v *VM) logger() logrus.FieldLogger { // Pause pauses a VM. func (v *VM) Pause() error { v.logger().Info("pause vm") - return v.hypervisor.pauseSandbox() + return v.hypervisor.PauseSandbox() } // Save saves a VM to persistent disk. func (v *VM) Save() error { v.logger().Info("save vm") - return v.hypervisor.saveSandbox() + return v.hypervisor.SaveSandbox() } // Resume resumes a paused VM. func (v *VM) Resume() error { v.logger().Info("resume vm") - return v.hypervisor.resumeSandbox() + return v.hypervisor.ResumeSandbox() } // Start kicks off a configured VM. func (v *VM) Start() error { v.logger().Info("start vm") - return v.hypervisor.startSandbox(vmStartTimeout) + return v.hypervisor.StartSandbox(vmStartTimeout) } // Disconnect agent and proxy connections to a VM @@ -242,7 +243,7 @@ func (v *VM) Disconnect() error { func (v *VM) Stop() error { v.logger().Info("stop vm") - if err := v.hypervisor.stopSandbox(); err != nil { + if err := v.hypervisor.StopSandbox(); err != nil { return err } @@ -253,7 +254,7 @@ func (v *VM) Stop() error { func (v *VM) AddCPUs(num uint32) error { if num > 0 { v.logger().Infof("hot adding %d vCPUs", num) - if _, err := v.hypervisor.hotplugAddDevice(num, cpuDev); err != nil { + if _, err := v.hypervisor.HotplugAddDevice(num, hypervisor.CPUDev); err != nil { return err } v.cpuDelta += num @@ -267,8 +268,11 @@ func (v *VM) AddCPUs(num uint32) error { func (v *VM) AddMemory(numMB uint32) error { if numMB > 0 { v.logger().Infof("hot adding %d MB memory", numMB) - dev := &memoryDevice{1, int(numMB)} - if _, err := v.hypervisor.hotplugAddDevice(dev, memoryDev); err != nil { + dev := &hypervisor.MemoryDevice{ + Slot: 1, + SizeMB: int(numMB), + } + if _, err := v.hypervisor.HotplugAddDevice(dev, hypervisor.MemoryDev); err != nil { return err } } diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go index 40a9729d6b..a92bb807bc 100644 --- a/virtcontainers/vm_test.go +++ b/virtcontainers/vm_test.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "testing" + "github.com/kata-containers/runtime/virtcontainers/hypervisor" "github.com/stretchr/testify/assert" ) @@ -18,11 +19,11 @@ func TestNewVM(t *testing.T) { testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") config := VMConfig{ - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: NoopAgentType, ProxyType: NoopProxyType, } - hyperConfig := HypervisorConfig{ + hyperConfig := hypervisor.Config{ KernelPath: testDir, ImagePath: testDir, } @@ -82,7 +83,7 @@ func TestVMConfigValid(t *testing.T) { assert.Error(err) testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") - config.HypervisorConfig = HypervisorConfig{ + config.HypervisorConfig = hypervisor.Config{ KernelPath: testDir, InitrdPath: testDir, } @@ -94,11 +95,11 @@ func TestSetupProxy(t *testing.T) { assert := assert.New(t) config := VMConfig{ - HypervisorType: MockHypervisor, + HypervisorType: hypervisor.Mock, AgentType: NoopAgentType, } - hypervisor := &mockHypervisor{} + hypervisor := hypervisor.NewMock() agent := &noopAgent{} // wrong proxy type