From 990d7eca8408ef04f7d9a2ef28577bbc5abb3fd9 Mon Sep 17 00:00:00 2001 From: Vishwas Siravara Date: Wed, 13 Dec 2023 13:42:40 -0800 Subject: [PATCH] implement persistent logging for contaiers running in foreground Signed-off-by: Vishwas Siravara --- .github/workflows/test.yml | 2 +- Dockerfile | 2 +- cmd/nerdctl/container_logs_test.go | 63 ++++++++ cmd/nerdctl/container_run.go | 2 +- go.mod | 8 +- go.sum | 11 ++ pkg/cioutil/containe_io_windows.go | 111 +++++++++++++++ pkg/cioutil/container_io.go | 221 +++++++++++++++++++++++++++++ pkg/cioutil/container_io_unix.go | 136 ++++++++++++++++++ pkg/containerutil/containerutil.go | 4 +- pkg/taskutil/taskutil.go | 9 +- 11 files changed, 559 insertions(+), 10 deletions(-) create mode 100644 pkg/cioutil/containe_io_windows.go create mode 100644 pkg/cioutil/container_io.go create mode 100644 pkg/cioutil/container_io_unix.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e6ac21683b..a9a22221d01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,7 @@ jobs: test-integration: runs-on: "ubuntu-${{ matrix.ubuntu }}" - timeout-minutes: 40 + timeout-minutes: 60 strategy: fail-fast: false matrix: diff --git a/Dockerfile b/Dockerfile index 67992512f15..ccd1b23ff64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -313,7 +313,7 @@ RUN curl -L -o nydus-static.tgz "https://github.com/dragonflyoss/image-service/r mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \ rm nydus-static.tgz CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=github.com/containerd/nerdctl/cmd/nerdctl/...", \ - "--", "-timeout=30m", "-args", "-test.kill-daemon"] + "--", "-timeout=60m", "-args", "-test.kill-daemon"] FROM test-integration AS test-integration-rootless # Install SSH for creating systemd user session. diff --git a/cmd/nerdctl/container_logs_test.go b/cmd/nerdctl/container_logs_test.go index 3f78ba8e2ce..66b0dc2ee81 100644 --- a/cmd/nerdctl/container_logs_test.go +++ b/cmd/nerdctl/container_logs_test.go @@ -18,6 +18,7 @@ package main import ( "fmt" + "runtime" "strings" "testing" "time" @@ -143,3 +144,65 @@ func TestLogsWithFailingContainer(t *testing.T) { base.Cmd("logs", "-f", containerName).AssertNoOut("baz") base.Cmd("rm", "-f", containerName).AssertOK() } + +func TestLogsWithForegroundContainers(t *testing.T) { + t.Parallel() + if runtime.GOOS == "windows" { + t.Skip("dual logging is not supported on Windows") + } + base := testutil.NewBase(t) + tid := testutil.Identifier(t) + + // unbuffer(1) emulates tty, which is required by `nerdctl run -t`. + // unbuffer(1) can be installed with `apt-get install expect`. + unbuffer := []string{"unbuffer"} + + testCases := []struct { + name string + flags []string + tty bool + }{ + { + name: "foreground", + flags: nil, + tty: false, + }, + { + name: "interactive", + flags: []string{"-i"}, + tty: false, + }, + { + name: "PTY", + flags: []string{"-t"}, + tty: true, + }, + { + name: "interactivePTY", + flags: []string{"-i", "-t"}, + tty: true, + }, + } + + for _, tc := range testCases { + tc := tc + func(t *testing.T) { + containerName := tid + "-" + tc.name + var cmdArgs []string + defer base.Cmd("rm", "-f", containerName).Run() + cmdArgs = append(cmdArgs, "run", "--name", containerName) + cmdArgs = append(cmdArgs, tc.flags...) + cmdArgs = append(cmdArgs, testutil.CommonImage, "sh", "-euxc", "echo foo; echo bar") + + if tc.tty { + base.CmdWithHelper(unbuffer, cmdArgs...).AssertOK() + } else { + base.Cmd(cmdArgs...).AssertOK() + } + + base.Cmd("logs", containerName).AssertOutContains("foo") + base.Cmd("logs", containerName).AssertOutContains("bar") + base.Cmd("logs", containerName).AssertNoOut("baz") + }(t) + } +} diff --git a/cmd/nerdctl/container_run.go b/cmd/nerdctl/container_run.go index 44720eea3af..c1684e17e7a 100644 --- a/cmd/nerdctl/container_run.go +++ b/cmd/nerdctl/container_run.go @@ -373,7 +373,7 @@ func runAction(cmd *cobra.Command, args []string) error { logURI := lab[labels.LogURI] detachC := make(chan struct{}) task, err := taskutil.NewTask(ctx, client, c, false, createOpt.Interactive, createOpt.TTY, createOpt.Detach, - con, logURI, createOpt.DetachKeys, detachC) + con, logURI, createOpt.DetachKeys, createOpt.GOptions.Namespace, detachC) if err != nil { return err } diff --git a/go.mod b/go.mod index 4e462e3420a..48fac826961 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,12 @@ require ( gotest.tools/v3 v3.5.1 ) +require ( + github.com/containerd/go-runc v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect +) + require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect @@ -68,7 +74,7 @@ require ( github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cilium/ebpf v0.9.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect - github.com/containerd/fifo v1.1.0 // indirect + github.com/containerd/fifo v1.1.0 github.com/containerd/ttrpc v1.2.2 // indirect github.com/containerd/typeurl v1.0.3-0.20220422153119-7f6e6d160d67 // indirect github.com/containers/ocicrypt v1.1.9 // indirect diff --git a/go.sum b/go.sum index 66b956b5968..3a08c7f75e3 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,7 @@ github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaD github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= @@ -45,6 +46,8 @@ github.com/containerd/go-cni v1.1.9 h1:ORi7P1dYzCwVM6XPN4n3CbkuOx/NZ2DOqy+SHRdo9 github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM= github.com/containerd/imgcrypt v1.1.9 h1:AnXt0sMq1Z2uIdaLt/fIHcMgtfVlFx6XpuaZzoC2XV0= github.com/containerd/imgcrypt v1.1.9/go.mod h1:zEN6Nz5d5XIKgq06Tzk82YRlPZULKGSJ8fxhXhMwrYY= +github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nydus-snapshotter v0.13.4 h1:veTQCgpfRGdPD031dVNGlU+vK/W9vBhZNlMWR9oupiQ= @@ -159,6 +162,11 @@ github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBx github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -239,6 +247,7 @@ github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/ github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v1.1.7 h1:y2EZDS8sNng4Ksf0GUYNhKbTShZJPJg1FiXJNH/uoCk= github.com/opencontainers/runc v1.1.7/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= @@ -264,6 +273,7 @@ github.com/rootless-containers/bypass4netns v0.3.0/go.mod h1:IXHPjkQlJRygNBCN0hS github.com/rootless-containers/rootlesskit v1.1.1 h1:F5psKWoWY9/VjZ3ifVcaosjvFZJOagX85U22M0/EQZE= github.com/rootless-containers/rootlesskit v1.1.1/go.mod h1:UD5GoA3dqKCJrnvnhVgQQnweMF2qZnf9KLw8EewcMZI= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -379,6 +389,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/cioutil/containe_io_windows.go b/pkg/cioutil/containe_io_windows.go new file mode 100644 index 00000000000..ac1c09625bf --- /dev/null +++ b/pkg/cioutil/containe_io_windows.go @@ -0,0 +1,111 @@ +//go:build windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cioutil + +import ( + "fmt" + "io" + "os/exec" + + "github.com/Microsoft/go-winio" + "github.com/containerd/containerd/cio" + "github.com/containerd/log" +) + +// copyIO is from https://github.com/containerd/containerd/blob/148d21b1ae0718b75718a09ecb307bb874270f59/cio/io_windows.go#L44 +func copyIO(_ *exec.Cmd, fifos *cio.FIFOSet, ioset *cio.Streams) (_ *ncio, retErr error) { + ncios := &ncio{cmd: nil, config: fifos.Config} + + defer func() { + if retErr != nil { + _ = ncios.Close() + } + }() + + if fifos.Stdin != "" { + l, err := winio.ListenPipe(fifos.Stdin, nil) + if err != nil { + return nil, fmt.Errorf("failed to create stdin pipe %s: %w", fifos.Stdin, err) + } + ncios.closers = append(ncios.closers, l) + + go func() { + c, err := l.Accept() + if err != nil { + log.L.WithError(err).Errorf("failed to accept stdin connection on %s", fifos.Stdin) + return + } + + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + + io.CopyBuffer(c, ioset.Stdin, *p) + c.Close() + l.Close() + }() + } + + if fifos.Stdout != "" { + l, err := winio.ListenPipe(fifos.Stdout, nil) + if err != nil { + return nil, fmt.Errorf("failed to create stdout pipe %s: %w", fifos.Stdout, err) + } + ncios.closers = append(ncios.closers, l) + + go func() { + c, err := l.Accept() + if err != nil { + log.L.WithError(err).Errorf("failed to accept stdout connection on %s", fifos.Stdout) + return + } + + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + + io.CopyBuffer(ioset.Stdout, c, *p) + c.Close() + l.Close() + }() + } + + if fifos.Stderr != "" { + l, err := winio.ListenPipe(fifos.Stderr, nil) + if err != nil { + return nil, fmt.Errorf("failed to create stderr pipe %s: %w", fifos.Stderr, err) + } + ncios.closers = append(ncios.closers, l) + + go func() { + c, err := l.Accept() + if err != nil { + log.L.WithError(err).Errorf("failed to accept stderr connection on %s", fifos.Stderr) + return + } + + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + + io.CopyBuffer(ioset.Stderr, c, *p) + c.Close() + l.Close() + }() + } + + return ncios, nil +} diff --git a/pkg/cioutil/container_io.go b/pkg/cioutil/container_io.go new file mode 100644 index 00000000000..24fb89186ba --- /dev/null +++ b/pkg/cioutil/container_io.go @@ -0,0 +1,221 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cioutil + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "runtime" + "sync" + "syscall" + "time" + + "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/defaults" + "github.com/containerd/containerd/pkg/process" +) + +const binaryIOProcTermTimeout = 12 * time.Second // Give logger process 10 seconds for cleanup + +// ncio is a basic container IO implementation. +type ncio struct { + cmd *exec.Cmd + config cio.Config + wg *sync.WaitGroup + closers []io.Closer + cancel context.CancelFunc +} + +var bufPool = sync.Pool{ + New: func() interface{} { + buffer := make([]byte, 32<<10) + return &buffer + }, +} + +func (c *ncio) Config() cio.Config { + return c.config +} + +func (c *ncio) Wait() { + if c.wg != nil { + c.wg.Wait() + } +} + +func (c *ncio) Close() error { + + var lastErr error + + if c.cmd != nil && c.cmd.Process != nil { + + // Send SIGTERM first, so logger process has a chance to flush and exit properly + if err := c.cmd.Process.Signal(syscall.SIGTERM); err != nil { + lastErr = fmt.Errorf("failed to send SIGTERM: %w", err) + + if err := c.cmd.Process.Kill(); err != nil { + lastErr = errors.Join(lastErr, fmt.Errorf("failed to kill process after faulty SIGTERM: %w", err)) + } + + } + + done := make(chan error, 1) + go func() { + done <- c.cmd.Wait() + }() + + select { + case err := <-done: + return err + case <-time.After(binaryIOProcTermTimeout): + + err := c.cmd.Process.Kill() + if err != nil { + lastErr = fmt.Errorf("failed to kill shim logger process: %w", err) + } + + } + } + + for _, closer := range c.closers { + if closer == nil { + continue + } + if err := closer.Close(); err != nil { + lastErr = err + } + } + return lastErr +} + +func (c *ncio) Cancel() { + if c.cancel != nil { + c.cancel() + } +} + +func NewContainerIO(namespace string, logURI string, tty bool, stdin io.Reader, stdout, stderr io.Writer) cio.Creator { + return func(id string) (_ cio.IO, err error) { + var ( + cmd *exec.Cmd + closers []func() error + streams = &cio.Streams{ + Terminal: tty, + } + ) + + defer func() { + if err == nil { + return + } + result := []error{err} + for _, fn := range closers { + result = append(result, fn()) + } + err = errors.Join(result...) + }() + + if stdin != nil { + streams.Stdin = stdin + } + + var stdoutWriters []io.Writer + if stdout != nil { + stdoutWriters = append(stdoutWriters, stdout) + } + + var stderrWriters []io.Writer + if stderr != nil { + stderrWriters = append(stderrWriters, stderr) + } + + if runtime.GOOS != "windows" { + // starting logging binary logic is from https://github.com/containerd/containerd/blob/194a1fdd2cde35bc019ef138f30485e27fe0913e/cmd/containerd-shim-runc-v2/process/io.go#L247 + stdoutr, stdoutw, err := os.Pipe() + if err != nil { + return nil, err + } + closers = append(closers, stdoutr.Close, stdoutw.Close) + + stderrr, stderrw, err := os.Pipe() + if err != nil { + return nil, err + } + closers = append(closers, stderrr.Close, stderrw.Close) + + r, w, err := os.Pipe() + if err != nil { + return nil, err + } + closers = append(closers, r.Close, w.Close) + + u, err := url.Parse(logURI) + if err != nil { + return nil, err + } + cmd = process.NewBinaryCmd(u, id, namespace) + cmd.ExtraFiles = append(cmd.ExtraFiles, stdoutr, stderrr, w) + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start binary process with cmdArgs %v: %w", cmd.Args, err) + } + + closers = append(closers, func() error { return cmd.Process.Kill() }) + + // close our side of the pipe after start + if err := w.Close(); err != nil { + return nil, fmt.Errorf("failed to close write pipe after start: %w", err) + } + + // wait for the logging binary to be ready + b := make([]byte, 1) + if _, err := r.Read(b); err != nil && err != io.EOF { + return nil, fmt.Errorf("failed to read from logging binary: %w", err) + } + + stdoutWriters = append(stdoutWriters, stdoutw) + stderrWriters = append(stderrWriters, stderrw) + } + + streams.Stdout = io.MultiWriter(stdoutWriters...) + streams.Stderr = io.MultiWriter(stderrWriters...) + + if streams.FIFODir == "" { + streams.FIFODir = defaults.DefaultFIFODir + } + fifos, err := cio.NewFIFOSetInDir(streams.FIFODir, id, streams.Terminal) + if err != nil { + return nil, err + } + + if streams.Stdin == nil { + fifos.Stdin = "" + } + if streams.Stdout == nil { + fifos.Stdout = "" + } + if streams.Stderr == nil { + fifos.Stderr = "" + } + return copyIO(cmd, fifos, streams) + } +} diff --git a/pkg/cioutil/container_io_unix.go b/pkg/cioutil/container_io_unix.go new file mode 100644 index 00000000000..fd0632c3781 --- /dev/null +++ b/pkg/cioutil/container_io_unix.go @@ -0,0 +1,136 @@ +//go:build !windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cioutil + +import ( + "context" + "fmt" + "io" + "os/exec" + "sync" + "syscall" + + "github.com/containerd/containerd/cio" + "github.com/containerd/fifo" +) + +type pipes struct { + Stdin io.WriteCloser + Stdout io.ReadCloser + Stderr io.ReadCloser +} + +func (p *pipes) closers() []io.Closer { + return []io.Closer{p.Stdin, p.Stdout, p.Stderr} +} + +// copyIO is from https://github.com/containerd/containerd/blob/148d21b1ae0718b75718a09ecb307bb874270f59/cio/io_unix.go#L55 +func copyIO(cmd *exec.Cmd, fifos *cio.FIFOSet, ioset *cio.Streams) (*ncio, error) { + var ctx, cancel = context.WithCancel(context.Background()) + pipes, err := openFifos(ctx, fifos) + if err != nil { + cancel() + return nil, err + } + + if fifos.Stdin != "" { + go func() { + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + + io.CopyBuffer(pipes.Stdin, ioset.Stdin, *p) + pipes.Stdin.Close() + }() + } + + var wg = &sync.WaitGroup{} + if fifos.Stdout != "" { + wg.Add(1) + go func() { + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + + io.CopyBuffer(ioset.Stdout, pipes.Stdout, *p) + pipes.Stdout.Close() + wg.Done() + }() + } + + if !fifos.Terminal && fifos.Stderr != "" { + wg.Add(1) + go func() { + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + + io.CopyBuffer(ioset.Stderr, pipes.Stderr, *p) + pipes.Stderr.Close() + wg.Done() + }() + } + + return &ncio{ + cmd: cmd, + config: fifos.Config, + wg: wg, + closers: append(pipes.closers(), fifos), + cancel: func() { + cancel() + for _, c := range pipes.closers() { + if c != nil { + c.Close() + } + } + }, + }, nil +} + +func openFifos(ctx context.Context, fifos *cio.FIFOSet) (f pipes, retErr error) { + defer func() { + if retErr != nil { + fifos.Close() + } + }() + + if fifos.Stdin != "" { + if f.Stdin, retErr = fifo.OpenFifo(ctx, fifos.Stdin, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); retErr != nil { + return f, fmt.Errorf("failed to open stdin fifo: %w", retErr) + } + defer func() { + if retErr != nil && f.Stdin != nil { + f.Stdin.Close() + } + }() + } + if fifos.Stdout != "" { + if f.Stdout, retErr = fifo.OpenFifo(ctx, fifos.Stdout, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); retErr != nil { + return f, fmt.Errorf("failed to open stdout fifo: %w", retErr) + } + defer func() { + if retErr != nil && f.Stdout != nil { + f.Stdout.Close() + } + }() + } + if !fifos.Terminal && fifos.Stderr != "" { + if f.Stderr, retErr = fifo.OpenFifo(ctx, fifos.Stderr, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); retErr != nil { + return f, fmt.Errorf("failed to open stderr fifo: %w", retErr) + } + } + return f, nil +} diff --git a/pkg/containerutil/containerutil.go b/pkg/containerutil/containerutil.go index 79d328b3bf0..23d03545177 100644 --- a/pkg/containerutil/containerutil.go +++ b/pkg/containerutil/containerutil.go @@ -243,7 +243,7 @@ func Start(ctx context.Context, container containerd.Container, flagA bool, clie } logURI := lab[labels.LogURI] - + namespace := lab[labels.Namespace] cStatus := formatter.ContainerStatus(ctx, container) if cStatus == "Up" { log.G(ctx).Warnf("container %s is already running", container.ID()) @@ -266,7 +266,7 @@ func Start(ctx context.Context, container containerd.Container, flagA bool, clie } } detachC := make(chan struct{}) - task, err := taskutil.NewTask(ctx, client, container, flagA, false, flagT, true, con, logURI, detachKeys, detachC) + task, err := taskutil.NewTask(ctx, client, container, flagA, false, flagT, true, con, logURI, detachKeys, namespace, detachC) if err != nil { return err } diff --git a/pkg/taskutil/taskutil.go b/pkg/taskutil/taskutil.go index 62e622f716f..022da8c7df2 100644 --- a/pkg/taskutil/taskutil.go +++ b/pkg/taskutil/taskutil.go @@ -31,6 +31,7 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/cio" "github.com/containerd/log" + "github.com/containerd/nerdctl/pkg/cioutil" "github.com/containerd/nerdctl/pkg/consoleutil" "github.com/containerd/nerdctl/pkg/infoutil" "golang.org/x/term" @@ -38,7 +39,7 @@ import ( // NewTask is from https://github.com/containerd/containerd/blob/v1.4.3/cmd/ctr/commands/tasks/tasks_unix.go#L70-L108 func NewTask(ctx context.Context, client *containerd.Client, container containerd.Container, - flagA, flagI, flagT, flagD bool, con console.Console, logURI, detachKeys string, detachC chan<- struct{}) (containerd.Task, error) { + flagA, flagI, flagT, flagD bool, con console.Console, logURI, detachKeys, namespace string, detachC chan<- struct{}) (containerd.Task, error) { var t containerd.Task closer := func() { if detachC != nil { @@ -93,6 +94,7 @@ func NewTask(ctx context.Context, client *containerd.Client, container container args[0]: args[1], }) } else if flagT && !flagD { + if con == nil { return nil, errors.New("got nil con with flagT=true") } @@ -108,9 +110,8 @@ func NewTask(ctx context.Context, client *containerd.Client, container container return nil, err } } - ioCreator = cio.NewCreator(cio.WithStreams(in, os.Stdout, nil), cio.WithTerminal) + ioCreator = cioutil.NewContainerIO(namespace, logURI, true, in, os.Stdout, os.Stderr) } else if flagD && logURI != "" { - // TODO: support logURI for `nerdctl run -it` u, err := url.Parse(logURI) if err != nil { return nil, err @@ -136,7 +137,7 @@ func NewTask(ctx context.Context, client *containerd.Client, container container } in = stdinC } - ioCreator = cio.NewCreator(cio.WithStreams(in, os.Stdout, os.Stderr)) + ioCreator = cioutil.NewContainerIO(namespace, logURI, false, in, os.Stdout, os.Stderr) } t, err := container.NewTask(ctx, ioCreator) if err != nil {