diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a1b4d7b647..0d1d4d34eea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,30 +115,51 @@ jobs: - ubuntu: 20.04 containerd: v1.6.24 target: test-integration-rootless + rootlesskit: v1.1.1 - ubuntu: 20.04 containerd: v1.7.7 target: test-integration-rootless + rootlesskit: v1.1.1 - ubuntu: 22.04 containerd: v1.7.7 target: test-integration-rootless + rootlesskit: v1.1.1 - ubuntu: 22.04 containerd: main target: test-integration-rootless + rootlesskit: v1.1.1 + - ubuntu: 20.04 + containerd: v1.7.7 + target: test-integration-rootless + rootlesskit: v2.0.0-alpha.0 + - ubuntu: 22.04 + containerd: v1.7.7 + target: test-integration-rootless + rootlesskit: v2.0.0-alpha.0 + - ubuntu: 22.04 + containerd: main + target: test-integration-rootless + rootlesskit: v2.0.0-alpha.0 - ubuntu: 20.04 containerd: v1.6.24 target: test-integration-rootless-port-slirp4netns + rootlesskit: v1.1.1 - ubuntu: 20.04 containerd: v1.7.7 target: test-integration-rootless-port-slirp4netns + rootlesskit: v1.1.1 - ubuntu: 22.04 containerd: v1.7.7 target: test-integration-rootless-port-slirp4netns + rootlesskit: v1.1.1 - ubuntu: 22.04 containerd: main target: test-integration-rootless-port-slirp4netns + rootlesskit: v1.1.1 env: UBUNTU_VERSION: "${{ matrix.ubuntu }}" CONTAINERD_VERSION: "${{ matrix.containerd }}" + ROOTLESSKIT_VERSION: "${{ matrix.rootlesskit }}" TEST_TARGET: "${{ matrix.target }}" steps: - uses: actions/checkout@v4.1.1 @@ -147,7 +168,7 @@ jobs: - name: "Register QEMU (tonistiigi/binfmt)" run: docker run --privileged --rm tonistiigi/binfmt --install all - name: "Prepare (network driver=slirp4netns, port driver=builtin)" - run: DOCKER_BUILDKIT=1 docker build -t ${TEST_TARGET} --target ${TEST_TARGET} --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} . + run: DOCKER_BUILDKIT=1 docker build -t ${TEST_TARGET} --target ${TEST_TARGET} --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} --build-arg ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION} . - name: "Test (network driver=slirp4netns, port driver=builtin)" run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=1 ${TEST_TARGET} diff --git a/Dockerfile.d/SHA256SUMS.d/rootlesskit-v2.0.0-alpha.0 b/Dockerfile.d/SHA256SUMS.d/rootlesskit-v2.0.0-alpha.0 new file mode 100644 index 00000000000..3bffac5b003 --- /dev/null +++ b/Dockerfile.d/SHA256SUMS.d/rootlesskit-v2.0.0-alpha.0 @@ -0,0 +1,6 @@ +74baab4363ff68d060c788605f5bc5979d3c662f0351b033ae874a3a72e6ebce rootlesskit-aarch64.tar.gz +630dce1a26263d6a7a9461656f3e004c63386b7d4ca71fdaaa37cd075e0f677c rootlesskit-armv7l.tar.gz +4377eb874bb202b7a00354b0924898de81d818753ac730cee8d16262eadd5617 rootlesskit-ppc64le.tar.gz +92861409fa4db5e8344a1b5409ea4e5cb47fa7db706b4647ff627e15bc806ffc rootlesskit-riscv64.tar.gz +5ea02fba90e5656660aa7eca66aece2b5c3207e01d147495da2f55cfb4726663 rootlesskit-s390x.tar.gz +3db2ac3022efc7d030f48fb60a0d568e9dcf8700bb3e0c926e02a4b080caa629 rootlesskit-x86_64.tar.gz \ No newline at end of file diff --git a/extras/rootless/containerd-rootless.sh b/extras/rootless/containerd-rootless.sh index d394eeabe56..ad5cad90430 100755 --- a/extras/rootless/containerd-rootless.sh +++ b/extras/rootless/containerd-rootless.sh @@ -107,6 +107,17 @@ if [ -z $_CONTAINERD_ROOTLESS_CHILD ]; then export _CONTAINERD_ROOTLESS_SELINUX fi fi + + detachNetns= + if command -v rootlesskit >/dev/null 2>&1; then + # If --detach-netns is present in --help, rootlesskit is >= v2.0.0. + if rootlesskit --help | grep -qw -- --detach-netns; then + detachNetns="--detach-netns" + else + echo "rootlesskit found but seems older than v2.0.0. Network namespace will kept attached." + fi + fi + # Re-exec the script via RootlessKit, so as to create unprivileged {user,mount,network} namespaces. # # --copy-up allows removing/creating files in the directories by creating tmpfs and symlinks @@ -116,6 +127,7 @@ if [ -z $_CONTAINERD_ROOTLESS_CHILD ]; then # * /run: copy-up is required so that we can create /run/containerd (hardcoded) in our namespace # * /var/lib: copy-up is required so that we can create /var/lib/containerd in our namespace exec rootlesskit \ + $detachNetns \ --state-dir=$CONTAINERD_ROOTLESS_ROOTLESSKIT_STATE_DIR \ --net=$net --mtu=$mtu \ --slirp4netns-sandbox=$CONTAINERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SANDBOX \ diff --git a/go.mod b/go.mod index 576068f9f38..24e14668318 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/pelletier/go-toml/v2 v2.1.0 github.com/rootless-containers/bypass4netns v0.3.0 github.com/rootless-containers/rootlesskit v1.1.1 + github.com/rootless-containers/rootlesskit/v2 v2.0.0-alpha.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/tidwall/gjson v1.17.0 @@ -84,7 +85,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.17.2 diff --git a/go.sum b/go.sum index 4295e70de7f..f60c0784896 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -263,6 +263,8 @@ github.com/rootless-containers/bypass4netns v0.3.0 h1:UwI55zWDZz7OGyN4YWgfCKdsI5 github.com/rootless-containers/bypass4netns v0.3.0/go.mod h1:IXHPjkQlJRygNBCN0hSSR3ITX6kDKr3aAaGHx6APd+g= github.com/rootless-containers/rootlesskit v1.1.1 h1:F5psKWoWY9/VjZ3ifVcaosjvFZJOagX85U22M0/EQZE= github.com/rootless-containers/rootlesskit v1.1.1/go.mod h1:UD5GoA3dqKCJrnvnhVgQQnweMF2qZnf9KLw8EewcMZI= +github.com/rootless-containers/rootlesskit/v2 v2.0.0-alpha.1 h1:EUh0kAOAmbKw2wlrYDvMgqrOix+XmPP6S8ouAxBb1fM= +github.com/rootless-containers/rootlesskit/v2 v2.0.0-alpha.1/go.mod h1:TOmphx2+hH4/98eGg0/ZXVcU8KWcvfymtQnt7Y2XSp0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/pkg/cmd/container/create.go b/pkg/cmd/container/create.go index e2409877288..faa0640e823 100644 --- a/pkg/cmd/container/create.go +++ b/pkg/cmd/container/create.go @@ -50,6 +50,7 @@ import ( "github.com/containerd/nerdctl/pkg/namestore" "github.com/containerd/nerdctl/pkg/platformutil" "github.com/containerd/nerdctl/pkg/referenceutil" + "github.com/containerd/nerdctl/pkg/rootlessutil" "github.com/containerd/nerdctl/pkg/strutil" dockercliopts "github.com/docker/cli/opts" dockeropts "github.com/docker/docker/opts" @@ -282,6 +283,20 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa opts = append(opts, propagateContainerdLabelsToOCIAnnotations()) + detachNetNs, err := rootlessutil.DetectRootlesskitFeature("--detach-netns") + if err != nil { + return nil, nil, err + } + if rootlessutil.IsRootlessChild() && detachNetNs { + stateDir, err := rootlessutil.RootlessKitStateDir() + if err != nil { + return nil, nil, err + } + if err := newContainerDetachNetNs(stateDir, id, &opts); err != nil { + return nil, nil, err + } + } + var s specs.Spec spec := containerd.WithSpec(&s, opts...) @@ -418,7 +433,7 @@ func withNerdctlOCIHook(cmd string, args []string) (oci.SpecOpts, error) { args = append([]string{cmd}, append(args, "internal", "oci-hook")...) return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { if s.Hooks == nil { - s.Hooks = &specs.Hooks{} + s.Hooks = new(specs.Hooks) } crArgs := append(args, "createRuntime") s.Hooks.CreateRuntime = append(s.Hooks.CreateRuntime, specs.Hook{ diff --git a/pkg/cmd/container/create_linux.go b/pkg/cmd/container/create_linux.go new file mode 100644 index 00000000000..3a3002e34c4 --- /dev/null +++ b/pkg/cmd/container/create_linux.go @@ -0,0 +1,41 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package container + +import ( + "fmt" + "path/filepath" + + "github.com/containerd/containerd/oci" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/rootless-containers/rootlesskit/v2/pkg/child" +) + +func newContainerDetachNetNs(stateDir, id string, opts *[]oci.SpecOpts) error { + return ns.WithNetNSPath(filepath.Join(stateDir, "netns"), func(_ ns.NetNS) error { + containerDetachNetNs := filepath.Join(stateDir, fmt.Sprintf("netns-%s", id)) + if err := child.NewNetNsWithPathWithoutEnter(containerDetachNetNs); err != nil { + return err + } + *opts = append(*opts, oci.WithLinuxNamespace(specs.LinuxNamespace{ + Type: specs.NetworkNamespace, + Path: containerDetachNetNs, + })) + return nil + }) +} diff --git a/pkg/cmd/container/create_other.go b/pkg/cmd/container/create_other.go new file mode 100644 index 00000000000..d8ad746e1b8 --- /dev/null +++ b/pkg/cmd/container/create_other.go @@ -0,0 +1,26 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package container + +import "github.com/containerd/containerd/oci" + +func newContainerDetachNetNs(_, _ string, _ *[]oci.SpecOpts) error { + //no op for !linux + return nil +} diff --git a/pkg/ocihook/ocihook.go b/pkg/ocihook/ocihook.go index 2b5a6aa956e..a467c5f8ca9 100644 --- a/pkg/ocihook/ocihook.go +++ b/pkg/ocihook/ocihook.go @@ -27,6 +27,8 @@ import ( "path/filepath" "strings" + "runtime" + gocni "github.com/containerd/go-cni" "github.com/containerd/log" "github.com/containerd/nerdctl/pkg/bypass4netnsutil" @@ -38,9 +40,9 @@ import ( "github.com/containerd/nerdctl/pkg/rootlessutil" types100 "github.com/containernetworking/cni/pkg/types/100" "github.com/opencontainers/runtime-spec/specs-go" - b4nndclient "github.com/rootless-containers/bypass4netns/pkg/api/daemon/client" rlkclient "github.com/rootless-containers/rootlesskit/pkg/api/client" + "github.com/vishvananda/netns" ) const ( @@ -86,6 +88,24 @@ func Run(stdin io.Reader, stderr io.Writer, event, dataStore, cniPath, cniNetcon return err } + detachNetNs, err := rootlessutil.DetectRootlesskitFeature("--detach-netns") + if err != nil { + return err + } + if rootlessutil.IsRootlessChild() && detachNetNs { + stateDir, err := rootlessutil.RootlessKitStateDir() + if err != nil { + return err + } + ns, err := netns.GetFromPath(filepath.Join(stateDir, "netns")) + if err != nil { + return err + } + if err = netns.Set(ns); err != nil { + return fmt.Errorf("switch to detached netns: %w", err) + } + } + switch event { case "createRuntime": return onCreateRuntime(opts) @@ -268,8 +288,7 @@ func getExtraHosts(state *specs.State) (map[string]string, error) { func getNetNSPath(state *specs.State) (string, error) { // If we have a network-namespace annotation we use it over the passed Pid. - netNsPath, netNsFound := state.Annotations[NetworkNamespace] - if netNsFound { + if netNsPath, netNsFound := state.Annotations[NetworkNamespace]; netNsFound { if _, err := os.Stat(netNsPath); err != nil { return "", err } @@ -277,7 +296,7 @@ func getNetNSPath(state *specs.State) (string, error) { return netNsPath, nil } - if state.Pid == 0 && !netNsFound { + if state.Pid == 0 { return "", errors.New("both state.Pid and the netNs annotation are unset") } @@ -403,10 +422,15 @@ func onCreateRuntime(opts *handlerOpts) error { ExtraHosts: opts.extraHosts, Name: opts.state.Annotations[labels.Name], } + runtime.LockOSThread() + // nsents verified here we are in detached netwoprk ns + // nsPath verified is pointing to the nested detached ns + // user ns is the detch user ns cniRes, err := opts.cni.Setup(ctx, opts.fullID, nsPath, namespaceOpts...) if err != nil { return fmt.Errorf("failed to call cni.Setup: %w", err) } + runtime.UnlockOSThread() cniResRaw := cniRes.Raw() for i, cniName := range opts.cniNames { hsMeta.Networks[cniName] = cniResRaw[i] diff --git a/pkg/rootlessutil/parent_linux.go b/pkg/rootlessutil/parent_linux.go index 42064671984..d35f15d82d1 100644 --- a/pkg/rootlessutil/parent_linux.go +++ b/pkg/rootlessutil/parent_linux.go @@ -87,15 +87,25 @@ func ParentMain(hostGatewayIP string) error { if err != nil { return err } + // args are compatible with both util-linux nsenter and busybox nsenter args := []string{ "-r/", // root dir (busybox nsenter wants this to be explicitly specified), "-w" + wd, // work dir "--preserve-credentials", - "-m", "-n", "-U", + "-m", "-U", "-t", strconv.Itoa(childPid), "-F", // no fork } + + detachNetNs, err := DetectRootlesskitFeature("--detach-netns") + if err != nil { + return err + } + if !detachNetNs { + args = append(args, "-n") + } + args = append(args, os.Args...) log.L.Debugf("rootless parent main: executing %q with %v", arg0, args) diff --git a/pkg/rootlessutil/parent_other.go b/pkg/rootlessutil/parent_other.go new file mode 100644 index 00000000000..a811d978e57 --- /dev/null +++ b/pkg/rootlessutil/parent_other.go @@ -0,0 +1,24 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package rootlessutil + +func RootlessKitStateDir() (string, error) { + // no op for !linux + return "", nil +} diff --git a/pkg/rootlessutil/rootlessutil_linux.go b/pkg/rootlessutil/rootlessutil_linux.go index 795df9cf750..6387809e91e 100644 --- a/pkg/rootlessutil/rootlessutil_linux.go +++ b/pkg/rootlessutil/rootlessutil_linux.go @@ -19,8 +19,10 @@ package rootlessutil import ( "fmt" "os" + "os/exec" "path/filepath" "strconv" + "strings" "github.com/rootless-containers/rootlesskit/pkg/api/client" ) @@ -68,6 +70,24 @@ func NewRootlessKitClient() (client.Client, error) { return client.New(apiSock) } +func DetectRootlesskitFeature(feature string) (bool, error) { + rootlesskit := "rootlesskit" + rootlesskitBinary, err := exec.LookPath(rootlesskit) + if err != nil { + return false, fmt.Errorf("%s binary is not installed: %w", rootlesskit, err) + } + cmd := exec.Command(rootlesskitBinary, "--help") + cmd.Env = os.Environ() + b, err := cmd.CombinedOutput() + if err != nil { + return false, fmt.Errorf("command \"%s --help\" failed, --help is not supported: %w", rootlesskitBinary, err) + } + if !strings.Contains(string(b), feature) { + return false, nil + } + return true, nil +} + // RootlessContainredSockAddress returns sock address of rootless containerd based on https://github.com/containerd/nerdctl/blob/main/docs/faq.md#containerd-socket-address func RootlessContainredSockAddress() (string, error) { stateDir, err := RootlessKitStateDir() diff --git a/pkg/rootlessutil/rootlessutil_other.go b/pkg/rootlessutil/rootlessutil_other.go index 9fa5d12f737..dea6187541c 100644 --- a/pkg/rootlessutil/rootlessutil_other.go +++ b/pkg/rootlessutil/rootlessutil_other.go @@ -58,6 +58,11 @@ func NewRootlessKitClient() (client.Client, error) { return nil, fmt.Errorf("cannot instantiate RootlessKit client on non-Linux hosts") } +// Always returns false on non-Linux platforms. +func DetectRootlesskitFeature(feature string) (bool, error) { + return false, nil +} + // Always errors out on non-Linux platforms. func ParentMain(hostGatewayIP string) error { return fmt.Errorf("cannot use RootlessKit on main entry point on non-Linux hosts")