From 573c1375f5d664c52a589f3b910e126fb54b7e4d Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Wed, 20 Oct 2021 06:20:09 -0700 Subject: [PATCH] Add reconnect logic for stdio pipes This change adds retry logic on the stdio relay if the server end of the named pipe disconnects. This is a common case if containerd restarts for example. The current approach is to make a io.Writer wrapper that handles the reconnection logic on a write failure if it can be determined that the error is from a disconnect. A new shim config option is exposed to tailor the retry timeout. This changes also adds cenkalti/backoff/v4 as a dependency to be used for handling exponential backoff logic for the stdio connection retry attempts. Retrying at a fixed interval is a bit naive as all of the shims would potentially be trying to reconnect to 3 pipes continuously all in bursts. This allows us to space out the connections, set an upper limit on timeout intervals and add an element of randomness to the retry attempts. Signed-off-by: Daniel Canter --- .github/workflows/ci.yml | 3 +- .golangci.yml | 2 +- .../options/runhcs.pb.go | 156 ++++++++++------- .../options/runhcs.proto | 5 + cmd/containerd-shim-runhcs-v1/task_hcs.go | 57 +++++-- go.mod | 1 + go.sum | 2 + internal/cmd/cmd.go | 18 -- internal/cmd/diag.go | 4 +- internal/cmd/io.go | 24 ++- internal/cmd/io_npipe.go | 124 +++++++++++++- internal/oci/uvm.go | 2 +- pkg/annotations/annotations.go | 1 + test/go.sum | 2 + .../Microsoft/hcsshim/.golangci.yml | 2 +- .../options/runhcs.pb.go | 156 ++++++++++------- .../options/runhcs.proto | 5 + .../github.com/Microsoft/hcsshim/go.mod | 1 + .../github.com/Microsoft/hcsshim/go.sum | 2 + .../Microsoft/hcsshim/internal/cmd/cmd.go | 18 -- .../Microsoft/hcsshim/internal/cmd/diag.go | 4 +- .../Microsoft/hcsshim/internal/cmd/io.go | 24 ++- .../hcsshim/internal/cmd/io_npipe.go | 124 +++++++++++++- .../Microsoft/hcsshim/internal/oci/uvm.go | 2 +- .../hcsshim/pkg/annotations/annotations.go | 1 + .../github.com/cenkalti/backoff/v4/.gitignore | 25 +++ .../cenkalti/backoff/v4/.travis.yml | 10 ++ .../github.com/cenkalti/backoff/v4/LICENSE | 20 +++ .../github.com/cenkalti/backoff/v4/README.md | 32 ++++ .../github.com/cenkalti/backoff/v4/backoff.go | 66 ++++++++ .../github.com/cenkalti/backoff/v4/context.go | 62 +++++++ .../cenkalti/backoff/v4/exponential.go | 158 ++++++++++++++++++ .../github.com/cenkalti/backoff/v4/go.mod | 3 + .../github.com/cenkalti/backoff/v4/retry.go | 112 +++++++++++++ .../github.com/cenkalti/backoff/v4/ticker.go | 97 +++++++++++ .../github.com/cenkalti/backoff/v4/timer.go | 35 ++++ .../github.com/cenkalti/backoff/v4/tries.go | 38 +++++ test/vendor/modules.txt | 2 + .../github.com/cenkalti/backoff/v4/.gitignore | 25 +++ .../cenkalti/backoff/v4/.travis.yml | 10 ++ vendor/github.com/cenkalti/backoff/v4/LICENSE | 20 +++ .../github.com/cenkalti/backoff/v4/README.md | 32 ++++ .../github.com/cenkalti/backoff/v4/backoff.go | 66 ++++++++ .../github.com/cenkalti/backoff/v4/context.go | 62 +++++++ .../cenkalti/backoff/v4/exponential.go | 158 ++++++++++++++++++ vendor/github.com/cenkalti/backoff/v4/go.mod | 3 + .../github.com/cenkalti/backoff/v4/retry.go | 112 +++++++++++++ .../github.com/cenkalti/backoff/v4/ticker.go | 97 +++++++++++ .../github.com/cenkalti/backoff/v4/timer.go | 35 ++++ .../github.com/cenkalti/backoff/v4/tries.go | 38 +++++ vendor/modules.txt | 2 + 51 files changed, 1862 insertions(+), 198 deletions(-) create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/.gitignore create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/.travis.yml create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/LICENSE create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/README.md create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/backoff.go create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/context.go create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/exponential.go create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/go.mod create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/retry.go create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/ticker.go create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/timer.go create mode 100644 test/vendor/github.com/cenkalti/backoff/v4/tries.go create mode 100644 vendor/github.com/cenkalti/backoff/v4/.gitignore create mode 100644 vendor/github.com/cenkalti/backoff/v4/.travis.yml create mode 100644 vendor/github.com/cenkalti/backoff/v4/LICENSE create mode 100644 vendor/github.com/cenkalti/backoff/v4/README.md create mode 100644 vendor/github.com/cenkalti/backoff/v4/backoff.go create mode 100644 vendor/github.com/cenkalti/backoff/v4/context.go create mode 100644 vendor/github.com/cenkalti/backoff/v4/exponential.go create mode 100644 vendor/github.com/cenkalti/backoff/v4/go.mod create mode 100644 vendor/github.com/cenkalti/backoff/v4/retry.go create mode 100644 vendor/github.com/cenkalti/backoff/v4/ticker.go create mode 100644 vendor/github.com/cenkalti/backoff/v4/timer.go create mode 100644 vendor/github.com/cenkalti/backoff/v4/tries.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 939f6bb38f..f0154e5961 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: with: version: v1.42.1 # Has fixes for stylecheck configuration https://github.com/golangci/golangci-lint/pull/2017/files args: --timeout=5m -v + only-new-issues: true verify-main-vendor: runs-on: 'windows-2019' @@ -40,7 +41,7 @@ jobs: Write-Error "Main modules are not up to date. Please validate your go version >= this job's and run `go mod vendor` followed by `go mod tidy` in the repo root path." } exit $process.ExitCode - + verify-test-vendor: runs-on: 'windows-2019' env: diff --git a/.golangci.yml b/.golangci.yml index 728178cff4..2830868fc4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,7 +24,7 @@ issues: linters: - stylecheck Text: "ST1003:" - + - path: cmd\\ncproxy\\nodenetsvc\\ linters: - stylecheck diff --git a/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go b/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go index fe11f8a2bb..89aff3723a 100644 --- a/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go +++ b/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go @@ -139,7 +139,11 @@ type Options struct { // log_level specifies the logrus log level for the shim. Supported values are a string representation of the // logrus log levels: "trace", "debug", "info", "warn", "error", "fatal", "panic". This setting will override // the `debug` field if both are specified, unless the level specified is also "debug", as these are equivalent. - LogLevel string `protobuf:"bytes,16,opt,name=log_level,json=logLevel,proto3" json:"log_level,omitempty"` + LogLevel string `protobuf:"bytes,16,opt,name=log_level,json=logLevel,proto3" json:"log_level,omitempty"` + // io_retry_timeout_in_sec is the timeout in seconds for how long to try and reconnect to an upstream IO provider if a connection is lost. + // The typical example is if Containerd has restarted but is expected to come back online. A 0 for this field is interpreted as an infinite + // timeout. + IoRetryTimeoutInSec int32 `protobuf:"varint,17,opt,name=io_retry_timeout_in_sec,json=ioRetryTimeoutInSec,proto3" json:"io_retry_timeout_in_sec,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -238,65 +242,67 @@ func init() { } var fileDescriptor_b643df6839c75082 = []byte{ - // 920 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x95, 0x5b, 0x6f, 0xdb, 0x36, - 0x14, 0xc7, 0xad, 0x26, 0x71, 0xac, 0x93, 0x9b, 0xc3, 0x05, 0x98, 0x90, 0xac, 0xb6, 0x91, 0x0e, - 0x68, 0x8a, 0x35, 0x52, 0xd2, 0x3d, 0x6e, 0xc0, 0xd0, 0xd8, 0x4e, 0xeb, 0x21, 0x17, 0x43, 0xce, - 0xd2, 0x5d, 0x1e, 0x08, 0x5d, 0x18, 0x59, 0xa8, 0x28, 0x0a, 0x24, 0xed, 0xc5, 0x7d, 0xda, 0x47, - 0xd8, 0xc7, 0xca, 0xe3, 0x1e, 0x07, 0x0c, 0xc8, 0x56, 0x7f, 0x88, 0x3d, 0x0f, 0xa4, 0xa8, 0xb4, - 0x0b, 0x82, 0xbd, 0xec, 0xc9, 0xe4, 0xff, 0xfc, 0xf8, 0xe7, 0xe1, 0x21, 0x8f, 0x05, 0xe7, 0x49, - 0x2a, 0xc7, 0x93, 0xd0, 0x8d, 0x18, 0xf5, 0x4e, 0xd3, 0x88, 0x33, 0xc1, 0xae, 0xa4, 0x37, 0x8e, - 0x84, 0x18, 0xa7, 0xd4, 0x8b, 0x68, 0xec, 0x45, 0x2c, 0x97, 0x41, 0x9a, 0x13, 0x1e, 0xef, 0x2b, - 0x6d, 0x9f, 0x4f, 0xf2, 0x71, 0x24, 0xf6, 0xa7, 0x87, 0x1e, 0x2b, 0x64, 0xca, 0x72, 0xe1, 0x95, - 0x8a, 0x5b, 0x70, 0x26, 0x19, 0xda, 0xfa, 0xc0, 0xbb, 0x26, 0x30, 0x3d, 0xdc, 0xde, 0x4a, 0x58, - 0xc2, 0x34, 0xe0, 0xa9, 0x51, 0xc9, 0x6e, 0xb7, 0x13, 0xc6, 0x92, 0x8c, 0x78, 0x7a, 0x16, 0x4e, - 0xae, 0x3c, 0x99, 0x52, 0x22, 0x64, 0x40, 0x8b, 0x12, 0xd8, 0xfd, 0xbb, 0x0e, 0xcb, 0xe7, 0xe5, - 0x2e, 0x68, 0x0b, 0x96, 0x62, 0x12, 0x4e, 0x12, 0xc7, 0xea, 0x58, 0x7b, 0x0d, 0xbf, 0x9c, 0xa0, - 0x63, 0x00, 0x3d, 0xc0, 0x72, 0x56, 0x10, 0xe7, 0x51, 0xc7, 0xda, 0x5b, 0x7f, 0xf1, 0xd4, 0x7d, - 0x28, 0x07, 0xd7, 0x18, 0xb9, 0x3d, 0xc5, 0x5f, 0xcc, 0x0a, 0xe2, 0xdb, 0x71, 0x35, 0x44, 0x4f, - 0x60, 0x8d, 0x93, 0x24, 0x15, 0x92, 0xcf, 0x30, 0x67, 0x4c, 0x3a, 0x0b, 0x1d, 0x6b, 0xcf, 0xf6, - 0x57, 0x2b, 0xd1, 0x67, 0x4c, 0x2a, 0x48, 0x04, 0x79, 0x1c, 0xb2, 0x6b, 0x9c, 0xd2, 0x20, 0x21, - 0xce, 0x62, 0x09, 0x19, 0x71, 0xa0, 0x34, 0xf4, 0x0c, 0x9a, 0x15, 0x54, 0x64, 0x81, 0xbc, 0x62, - 0x9c, 0x3a, 0x4b, 0x9a, 0xdb, 0x30, 0xfa, 0xd0, 0xc8, 0xe8, 0x27, 0xd8, 0xbc, 0xf3, 0x13, 0x2c, - 0x0b, 0x54, 0x7e, 0x4e, 0x5d, 0x9f, 0xc1, 0xfd, 0xef, 0x33, 0x8c, 0xcc, 0x8e, 0xd5, 0x2a, 0xbf, - 0xda, 0xf3, 0x4e, 0x41, 0x1e, 0x6c, 0x85, 0x8c, 0x49, 0x7c, 0x95, 0x66, 0x44, 0xe8, 0x33, 0xe1, - 0x22, 0x90, 0x63, 0x67, 0x59, 0xe7, 0xb2, 0xa9, 0x62, 0xc7, 0x2a, 0xa4, 0x4e, 0x36, 0x0c, 0xe4, - 0x18, 0x3d, 0x07, 0x34, 0xa5, 0xb8, 0xe0, 0x2c, 0x22, 0x42, 0x30, 0x8e, 0x23, 0x36, 0xc9, 0xa5, - 0xd3, 0xe8, 0x58, 0x7b, 0x4b, 0x7e, 0x73, 0x4a, 0x87, 0x55, 0xa0, 0xab, 0x74, 0xe4, 0xc2, 0xd6, - 0x94, 0x62, 0x4a, 0x28, 0xe3, 0x33, 0x2c, 0xd2, 0x77, 0x04, 0xa7, 0x39, 0xa6, 0xa1, 0x63, 0x57, - 0xfc, 0xa9, 0x0e, 0x8d, 0xd2, 0x77, 0x64, 0x90, 0x9f, 0x86, 0xa8, 0x05, 0xf0, 0x6a, 0xf8, 0xdd, - 0xe5, 0xeb, 0x9e, 0xda, 0xcb, 0x01, 0x9d, 0xc4, 0x47, 0x0a, 0xfa, 0x1a, 0x76, 0x44, 0x14, 0x64, - 0x04, 0x47, 0xc5, 0x04, 0x67, 0x29, 0x4d, 0xa5, 0xc0, 0x92, 0x61, 0x73, 0x2c, 0x67, 0x45, 0x5f, - 0xfa, 0xa7, 0x1a, 0xe9, 0x16, 0x93, 0x13, 0x0d, 0x5c, 0x30, 0x53, 0x07, 0x74, 0x0a, 0x9f, 0xc7, - 0xe4, 0x2a, 0x98, 0x64, 0x12, 0xdf, 0xd5, 0x0d, 0x8b, 0x88, 0x07, 0x32, 0x1a, 0xdf, 0x65, 0x97, - 0x84, 0xce, 0xaa, 0xce, 0xae, 0x6d, 0xd8, 0x6e, 0x85, 0x8e, 0x4a, 0xb2, 0x4c, 0xf6, 0x55, 0x88, - 0xbe, 0x81, 0xc7, 0x95, 0xdd, 0x94, 0x3e, 0xe4, 0xb3, 0xa6, 0x7d, 0x1c, 0x03, 0x5d, 0xd2, 0xfb, - 0x06, 0xea, 0xa5, 0x8c, 0x03, 0x4e, 0xaa, 0xb5, 0xce, 0xba, 0xce, 0x7f, 0x55, 0x8b, 0x06, 0x46, - 0x1d, 0x58, 0x39, 0xeb, 0x0e, 0x39, 0xbb, 0x9e, 0xbd, 0x8c, 0x63, 0xee, 0x6c, 0xe8, 0x9a, 0x7c, - 0x2c, 0xa1, 0x1d, 0xb0, 0x33, 0x96, 0xe0, 0x8c, 0x4c, 0x49, 0xe6, 0x34, 0x75, 0xbc, 0x91, 0xb1, - 0xe4, 0x44, 0xcd, 0x77, 0x9f, 0x81, 0x7d, 0xf7, 0x94, 0x91, 0x0d, 0x4b, 0x67, 0xc3, 0xc1, 0xb0, - 0xdf, 0xac, 0xa1, 0x06, 0x2c, 0x1e, 0x0f, 0x4e, 0xfa, 0x4d, 0x0b, 0x2d, 0xc3, 0x42, 0xff, 0xe2, - 0x4d, 0xf3, 0xd1, 0xae, 0x07, 0xcd, 0xfb, 0x2f, 0x06, 0xad, 0xc0, 0xf2, 0xd0, 0x3f, 0xef, 0xf6, - 0x47, 0xa3, 0x66, 0x0d, 0xad, 0x03, 0xbc, 0xfe, 0x61, 0xd8, 0xf7, 0x2f, 0x07, 0xa3, 0x73, 0xbf, - 0x69, 0xed, 0xfe, 0xb1, 0x00, 0xeb, 0xe6, 0xc2, 0x7b, 0x44, 0x06, 0x69, 0x26, 0xd0, 0x63, 0x00, - 0xfd, 0xe8, 0x71, 0x1e, 0x50, 0xa2, 0x9b, 0xd0, 0xf6, 0x6d, 0xad, 0x9c, 0x05, 0x94, 0xa0, 0x2e, - 0x40, 0xc4, 0x49, 0x20, 0x49, 0x8c, 0x03, 0xa9, 0x1b, 0x71, 0xe5, 0xc5, 0xb6, 0x5b, 0x36, 0xb8, - 0x5b, 0x35, 0xb8, 0x7b, 0x51, 0x35, 0xf8, 0x51, 0xe3, 0xe6, 0xb6, 0x5d, 0xfb, 0xf5, 0xcf, 0xb6, - 0xe5, 0xdb, 0x66, 0xdd, 0x4b, 0x89, 0xbe, 0x00, 0xf4, 0x96, 0xf0, 0x9c, 0x64, 0x58, 0xfd, 0x13, - 0xe0, 0xc3, 0x83, 0x03, 0x9c, 0x0b, 0xdd, 0x8a, 0x8b, 0xfe, 0x46, 0x19, 0x51, 0x0e, 0x87, 0x07, - 0x07, 0x67, 0x02, 0xb9, 0xf0, 0x89, 0x79, 0x7e, 0x11, 0xa3, 0x34, 0x95, 0x38, 0x9c, 0x49, 0x22, - 0x74, 0x4f, 0x2e, 0xfa, 0x9b, 0x65, 0xa8, 0xab, 0x23, 0x47, 0x2a, 0x80, 0x8e, 0xa1, 0x63, 0xf8, - 0x9f, 0x19, 0x7f, 0x9b, 0xe6, 0x09, 0x16, 0x44, 0xe2, 0x82, 0xa7, 0xd3, 0x40, 0x12, 0xb3, 0x78, - 0x49, 0x2f, 0xfe, 0xac, 0xe4, 0xde, 0x94, 0xd8, 0x88, 0xc8, 0x61, 0x09, 0x95, 0x3e, 0x3d, 0x68, - 0x3f, 0xe0, 0xa3, 0x6f, 0x36, 0x36, 0x36, 0x75, 0x6d, 0xb3, 0x73, 0xdf, 0x66, 0xa4, 0x99, 0xd2, - 0xe5, 0x39, 0x80, 0x69, 0x35, 0x9c, 0xc6, 0xba, 0x29, 0xd7, 0x8e, 0xd6, 0xe6, 0xb7, 0x6d, 0xdb, - 0x94, 0x7d, 0xd0, 0xf3, 0x6d, 0x03, 0x0c, 0x62, 0xf4, 0x14, 0x9a, 0x13, 0x41, 0xf8, 0xbf, 0xca, - 0xd2, 0xd0, 0x9b, 0xac, 0x29, 0xfd, 0x43, 0x51, 0x9e, 0xc0, 0x32, 0xb9, 0x26, 0x91, 0xf2, 0x54, - 0x9d, 0x68, 0x1f, 0xc1, 0xfc, 0xb6, 0x5d, 0xef, 0x5f, 0x93, 0x68, 0xd0, 0xf3, 0xeb, 0x2a, 0x34, - 0x88, 0x8f, 0xe2, 0x9b, 0xf7, 0xad, 0xda, 0xef, 0xef, 0x5b, 0xb5, 0x5f, 0xe6, 0x2d, 0xeb, 0x66, - 0xde, 0xb2, 0x7e, 0x9b, 0xb7, 0xac, 0xbf, 0xe6, 0x2d, 0xeb, 0xc7, 0x6f, 0xff, 0xff, 0xe7, 0xe0, - 0x2b, 0xf3, 0xfb, 0x7d, 0x2d, 0xac, 0xeb, 0x7b, 0xff, 0xf2, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x84, 0x30, 0x9b, 0x7c, 0x65, 0x06, 0x00, 0x00, + // 953 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x5d, 0x6f, 0xdb, 0x36, + 0x17, 0xb6, 0xda, 0x24, 0xb6, 0x4e, 0xbe, 0x1c, 0x36, 0x40, 0x85, 0xe4, 0xad, 0x6d, 0xa4, 0x2f, + 0xd0, 0x14, 0x6b, 0xa4, 0xa4, 0xdb, 0xdd, 0x06, 0x0c, 0x8d, 0xed, 0xb4, 0x1e, 0xf2, 0x61, 0xc8, + 0x59, 0xba, 0x8f, 0x0b, 0x42, 0x1f, 0x8c, 0x4c, 0x54, 0x12, 0x05, 0x92, 0xf6, 0xe2, 0x5e, 0xed, + 0x27, 0xec, 0x87, 0xec, 0x87, 0xe4, 0x72, 0x97, 0x03, 0x06, 0x64, 0xab, 0x7f, 0xc9, 0x40, 0x8a, + 0x4a, 0xbb, 0x20, 0xd8, 0xcd, 0xae, 0x4c, 0x3e, 0xcf, 0xc3, 0x87, 0xe7, 0x1c, 0x9d, 0x43, 0xc3, + 0x59, 0x42, 0xe5, 0x78, 0x12, 0xba, 0x11, 0xcb, 0xbc, 0x13, 0x1a, 0x71, 0x26, 0xd8, 0xa5, 0xf4, + 0xc6, 0x91, 0x10, 0x63, 0x9a, 0x79, 0x51, 0x16, 0x7b, 0x11, 0xcb, 0x65, 0x40, 0x73, 0xc2, 0xe3, + 0x3d, 0x85, 0xed, 0xf1, 0x49, 0x3e, 0x8e, 0xc4, 0xde, 0xf4, 0xc0, 0x63, 0x85, 0xa4, 0x2c, 0x17, + 0x5e, 0x89, 0xb8, 0x05, 0x67, 0x92, 0xa1, 0xcd, 0x8f, 0x7a, 0xd7, 0x10, 0xd3, 0x83, 0xad, 0xcd, + 0x84, 0x25, 0x4c, 0x0b, 0x3c, 0xb5, 0x2a, 0xb5, 0x5b, 0xed, 0x84, 0xb1, 0x24, 0x25, 0x9e, 0xde, + 0x85, 0x93, 0x4b, 0x4f, 0xd2, 0x8c, 0x08, 0x19, 0x64, 0x45, 0x29, 0xd8, 0xf9, 0xb5, 0x0e, 0xf5, + 0xb3, 0xf2, 0x16, 0xb4, 0x09, 0x8b, 0x31, 0x09, 0x27, 0x89, 0x63, 0x75, 0xac, 0xdd, 0x86, 0x5f, + 0x6e, 0xd0, 0x11, 0x80, 0x5e, 0x60, 0x39, 0x2b, 0x88, 0xf3, 0xa0, 0x63, 0xed, 0xae, 0xbd, 0x7c, + 0xe6, 0xde, 0x17, 0x83, 0x6b, 0x8c, 0xdc, 0x9e, 0xd2, 0x9f, 0xcf, 0x0a, 0xe2, 0xdb, 0x71, 0xb5, + 0x44, 0x4f, 0x61, 0x95, 0x93, 0x84, 0x0a, 0xc9, 0x67, 0x98, 0x33, 0x26, 0x9d, 0x87, 0x1d, 0x6b, + 0xd7, 0xf6, 0x57, 0x2a, 0xd0, 0x67, 0x4c, 0x2a, 0x91, 0x08, 0xf2, 0x38, 0x64, 0x57, 0x98, 0x66, + 0x41, 0x42, 0x9c, 0x85, 0x52, 0x64, 0xc0, 0x81, 0xc2, 0xd0, 0x73, 0x68, 0x56, 0xa2, 0x22, 0x0d, + 0xe4, 0x25, 0xe3, 0x99, 0xb3, 0xa8, 0x75, 0xeb, 0x06, 0x1f, 0x1a, 0x18, 0xfd, 0x08, 0x1b, 0xb7, + 0x7e, 0x82, 0xa5, 0x81, 0x8a, 0xcf, 0x59, 0xd2, 0x39, 0xb8, 0xff, 0x9e, 0xc3, 0xc8, 0xdc, 0x58, + 0x9d, 0xf2, 0xab, 0x3b, 0x6f, 0x11, 0xe4, 0xc1, 0x66, 0xc8, 0x98, 0xc4, 0x97, 0x34, 0x25, 0x42, + 0xe7, 0x84, 0x8b, 0x40, 0x8e, 0x9d, 0xba, 0x8e, 0x65, 0x43, 0x71, 0x47, 0x8a, 0x52, 0x99, 0x0d, + 0x03, 0x39, 0x46, 0x2f, 0x00, 0x4d, 0x33, 0x5c, 0x70, 0x16, 0x11, 0x21, 0x18, 0xc7, 0x11, 0x9b, + 0xe4, 0xd2, 0x69, 0x74, 0xac, 0xdd, 0x45, 0xbf, 0x39, 0xcd, 0x86, 0x15, 0xd1, 0x55, 0x38, 0x72, + 0x61, 0x73, 0x9a, 0xe1, 0x8c, 0x64, 0x8c, 0xcf, 0xb0, 0xa0, 0xef, 0x09, 0xa6, 0x39, 0xce, 0x42, + 0xc7, 0xae, 0xf4, 0x27, 0x9a, 0x1a, 0xd1, 0xf7, 0x64, 0x90, 0x9f, 0x84, 0xa8, 0x05, 0xf0, 0x7a, + 0xf8, 0xed, 0xc5, 0x9b, 0x9e, 0xba, 0xcb, 0x01, 0x1d, 0xc4, 0x27, 0x08, 0xfa, 0x0a, 0xb6, 0x45, + 0x14, 0xa4, 0x04, 0x47, 0xc5, 0x04, 0xa7, 0x34, 0xa3, 0x52, 0x60, 0xc9, 0xb0, 0x49, 0xcb, 0x59, + 0xd6, 0x1f, 0xfd, 0xb1, 0x96, 0x74, 0x8b, 0xc9, 0xb1, 0x16, 0x9c, 0x33, 0x53, 0x07, 0x74, 0x02, + 0xff, 0x8f, 0xc9, 0x65, 0x30, 0x49, 0x25, 0xbe, 0xad, 0x1b, 0x16, 0x11, 0x0f, 0x64, 0x34, 0xbe, + 0x8d, 0x2e, 0x09, 0x9d, 0x15, 0x1d, 0x5d, 0xdb, 0x68, 0xbb, 0x95, 0x74, 0x54, 0x2a, 0xcb, 0x60, + 0x5f, 0x87, 0xe8, 0x6b, 0x78, 0x52, 0xd9, 0x4d, 0xb3, 0xfb, 0x7c, 0x56, 0xb5, 0x8f, 0x63, 0x44, + 0x17, 0xd9, 0x5d, 0x03, 0xd5, 0x29, 0xe3, 0x80, 0x93, 0xea, 0xac, 0xb3, 0xa6, 0xe3, 0x5f, 0xd1, + 0xa0, 0x11, 0xa3, 0x0e, 0x2c, 0x9f, 0x76, 0x87, 0x9c, 0x5d, 0xcd, 0x5e, 0xc5, 0x31, 0x77, 0xd6, + 0x75, 0x4d, 0x3e, 0x85, 0xd0, 0x36, 0xd8, 0x29, 0x4b, 0x70, 0x4a, 0xa6, 0x24, 0x75, 0x9a, 0x9a, + 0x6f, 0xa4, 0x2c, 0x39, 0x56, 0x7b, 0xf4, 0x05, 0x3c, 0xa6, 0x0c, 0x73, 0xa2, 0x5a, 0x56, 0x0d, + 0x0e, 0x9b, 0x48, 0x15, 0x9d, 0x20, 0x91, 0xb3, 0xa1, 0xc3, 0x7b, 0x44, 0x99, 0xaf, 0xd8, 0xf3, + 0x92, 0x1c, 0xe4, 0x23, 0x12, 0xed, 0x3c, 0x07, 0xfb, 0x76, 0x00, 0x90, 0x0d, 0x8b, 0xa7, 0xc3, + 0xc1, 0xb0, 0xdf, 0xac, 0xa1, 0x06, 0x2c, 0x1c, 0x0d, 0x8e, 0xfb, 0x4d, 0x0b, 0xd5, 0xe1, 0x61, + 0xff, 0xfc, 0x6d, 0xf3, 0xc1, 0x8e, 0x07, 0xcd, 0xbb, 0x7d, 0x86, 0x96, 0xa1, 0x3e, 0xf4, 0xcf, + 0xba, 0xfd, 0xd1, 0xa8, 0x59, 0x43, 0x6b, 0x00, 0x6f, 0xbe, 0x1f, 0xf6, 0xfd, 0x8b, 0xc1, 0xe8, + 0xcc, 0x6f, 0x5a, 0x3b, 0x7f, 0x3c, 0x84, 0x35, 0xd3, 0x26, 0x3d, 0x22, 0x03, 0x9a, 0x0a, 0xf4, + 0x04, 0x40, 0x8f, 0x0a, 0xce, 0x83, 0x8c, 0xe8, 0xd1, 0xb5, 0x7d, 0x5b, 0x23, 0xa7, 0x41, 0x46, + 0x50, 0x17, 0x20, 0xe2, 0x24, 0x90, 0x24, 0xc6, 0x81, 0xd4, 0xe3, 0xbb, 0xfc, 0x72, 0xcb, 0x2d, + 0x9f, 0x05, 0xb7, 0x7a, 0x16, 0xdc, 0xf3, 0xea, 0x59, 0x38, 0x6c, 0x5c, 0xdf, 0xb4, 0x6b, 0xbf, + 0xfc, 0xd9, 0xb6, 0x7c, 0xdb, 0x9c, 0x7b, 0x25, 0xd1, 0x67, 0x80, 0xde, 0x11, 0x9e, 0x93, 0x54, + 0x97, 0x01, 0x1f, 0xec, 0xef, 0xe3, 0x5c, 0xe8, 0x01, 0x5e, 0xf0, 0xd7, 0x4b, 0x46, 0x39, 0x1c, + 0xec, 0xef, 0x9f, 0x0a, 0xe4, 0xc2, 0x23, 0xd3, 0xb4, 0x11, 0xcb, 0x32, 0x2a, 0x71, 0x38, 0x93, + 0x44, 0xe8, 0x49, 0x5e, 0xf0, 0x37, 0x4a, 0xaa, 0xab, 0x99, 0x43, 0x45, 0xa0, 0x23, 0xe8, 0x18, + 0xfd, 0x4f, 0x8c, 0xbf, 0xa3, 0x79, 0x82, 0x05, 0x91, 0xb8, 0xe0, 0x74, 0x1a, 0x48, 0x62, 0x0e, + 0x2f, 0xea, 0xc3, 0xff, 0x2b, 0x75, 0x6f, 0x4b, 0xd9, 0x88, 0xc8, 0x61, 0x29, 0x2a, 0x7d, 0x7a, + 0xd0, 0xbe, 0xc7, 0x47, 0xf7, 0x43, 0x6c, 0x6c, 0x96, 0xb4, 0xcd, 0xf6, 0x5d, 0x9b, 0x91, 0xd6, + 0x94, 0x2e, 0x2f, 0x00, 0xcc, 0x80, 0x62, 0x1a, 0xeb, 0x51, 0x5e, 0x3d, 0x5c, 0x9d, 0xdf, 0xb4, + 0x6d, 0x53, 0xf6, 0x41, 0xcf, 0xb7, 0x8d, 0x60, 0x10, 0xa3, 0x67, 0xd0, 0x9c, 0x08, 0xc2, 0xff, + 0x51, 0x96, 0x86, 0xbe, 0x64, 0x55, 0xe1, 0x1f, 0x8b, 0xf2, 0x14, 0xea, 0xe4, 0x8a, 0x44, 0xca, + 0x53, 0xcd, 0xaf, 0x7d, 0x08, 0xf3, 0x9b, 0xf6, 0x52, 0xff, 0x8a, 0x44, 0x83, 0x9e, 0xbf, 0xa4, + 0xa8, 0x41, 0x7c, 0x18, 0x5f, 0x7f, 0x68, 0xd5, 0x7e, 0xff, 0xd0, 0xaa, 0xfd, 0x3c, 0x6f, 0x59, + 0xd7, 0xf3, 0x96, 0xf5, 0xdb, 0xbc, 0x65, 0xfd, 0x35, 0x6f, 0x59, 0x3f, 0x7c, 0xf3, 0xdf, 0xff, + 0x44, 0xbe, 0x34, 0xbf, 0xdf, 0xd5, 0xc2, 0x25, 0xfd, 0xdd, 0x3f, 0xff, 0x3b, 0x00, 0x00, 0xff, + 0xff, 0x6b, 0x83, 0xa6, 0x5f, 0x9b, 0x06, 0x00, 0x00, } func (m *Options) Marshal() (dAtA []byte, err error) { @@ -418,6 +424,13 @@ func (m *Options) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintRunhcs(dAtA, i, uint64(len(m.LogLevel))) i += copy(dAtA[i:], m.LogLevel) } + if m.IoRetryTimeoutInSec != 0 { + dAtA[i] = 0x88 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRunhcs(dAtA, i, uint64(m.IoRetryTimeoutInSec)) + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -565,6 +578,9 @@ func (m *Options) Size() (n int) { if l > 0 { n += 2 + l + sovRunhcs(uint64(l)) } + if m.IoRetryTimeoutInSec != 0 { + n += 2 + sovRunhcs(uint64(m.IoRetryTimeoutInSec)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -645,6 +661,7 @@ func (this *Options) String() string { `ShareScratch:` + fmt.Sprintf("%v", this.ShareScratch) + `,`, `NCProxyAddr:` + fmt.Sprintf("%v", this.NCProxyAddr) + `,`, `LogLevel:` + fmt.Sprintf("%v", this.LogLevel) + `,`, + `IoRetryTimeoutInSec:` + fmt.Sprintf("%v", this.IoRetryTimeoutInSec) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -1104,6 +1121,25 @@ func (m *Options) Unmarshal(dAtA []byte) error { } m.LogLevel = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 17: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IoRetryTimeoutInSec", wireType) + } + m.IoRetryTimeoutInSec = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRunhcs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IoRetryTimeoutInSec |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipRunhcs(dAtA[iNdEx:]) diff --git a/cmd/containerd-shim-runhcs-v1/options/runhcs.proto b/cmd/containerd-shim-runhcs-v1/options/runhcs.proto index 90f3376ffc..60c89adbde 100644 --- a/cmd/containerd-shim-runhcs-v1/options/runhcs.proto +++ b/cmd/containerd-shim-runhcs-v1/options/runhcs.proto @@ -93,6 +93,11 @@ message Options { // logrus log levels: "trace", "debug", "info", "warn", "error", "fatal", "panic". This setting will override // the `debug` field if both are specified, unless the level specified is also "debug", as these are equivalent. string log_level = 16; + + // io_retry_timeout_in_sec is the timeout in seconds for how long to try and reconnect to an upstream IO provider if a connection is lost. + // The typical example is if Containerd has restarted but is expected to come back online. A 0 for this field is interpreted as an infinite + // timeout. + int32 io_retry_timeout_in_sec = 17; } // ProcessDetails contains additional information about a process. This is the additional diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs.go b/cmd/containerd-shim-runhcs-v1/task_hcs.go index 4ecb35716f..0f48c0f6a6 100644 --- a/cmd/containerd-shim-runhcs-v1/task_hcs.go +++ b/cmd/containerd-shim-runhcs-v1/task_hcs.go @@ -166,11 +166,6 @@ func newHcsTask( owner := filepath.Base(os.Args[0]) isTemplate := oci.ParseAnnotationsSaveAsTemplate(ctx, s) - io, err := cmd.NewUpstreamIO(ctx, req.ID, req.Stdout, req.Stderr, req.Stdin, req.Terminal) - if err != nil { - return nil, err - } - var netNS string if s.Windows != nil && s.Windows.Network != nil { @@ -186,22 +181,33 @@ func newHcsTask( shimOpts = v.(*runhcsopts.Options) } + // Default to an infinite timeout (zero value) + var ioRetryTimeout time.Duration + if shimOpts != nil { + ioRetryTimeout = time.Duration(shimOpts.IoRetryTimeoutInSec) * time.Second + } + io, err := cmd.NewUpstreamIO(ctx, req.ID, req.Stdout, req.Stderr, req.Stdin, req.Terminal, ioRetryTimeout) + if err != nil { + return nil, err + } + container, resources, err := createContainer(ctx, req.ID, owner, netNS, s, parent, shimOpts) if err != nil { return nil, err } ht := &hcsTask{ - events: events, - id: req.ID, - isWCOW: oci.IsWCOW(s), - c: container, - cr: resources, - ownsHost: ownsParent, - host: parent, - closed: make(chan struct{}), - taskSpec: s, - isTemplate: isTemplate, + events: events, + id: req.ID, + isWCOW: oci.IsWCOW(s), + c: container, + cr: resources, + ownsHost: ownsParent, + host: parent, + closed: make(chan struct{}), + taskSpec: s, + isTemplate: isTemplate, + ioRetryTimeout: ioRetryTimeout, } ht.init = newHcsExec( ctx, @@ -279,7 +285,21 @@ func newClonedHcsTask( return nil, fmt.Errorf("cloned task can only be created inside a windows host") } - io, err := cmd.NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal) + var shimOpts *runhcsopts.Options + if req.Options != nil { + v, err := typeurl.UnmarshalAny(req.Options) + if err != nil { + return nil, err + } + shimOpts = v.(*runhcsopts.Options) + } + + // Default to an infinite timeout (zero value) + var ioRetryTimeout time.Duration + if shimOpts != nil { + ioRetryTimeout = time.Duration(shimOpts.IoRetryTimeoutInSec) * time.Second + } + io, err := cmd.NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal, ioRetryTimeout) if err != nil { return nil, err } @@ -433,6 +453,9 @@ type hcsTask struct { // taskSpec represents the spec/configuration for this task. taskSpec *specs.Spec + + // ioRetryTimeout is the time for how long to try reconnecting to stdio pipes from containerd. + ioRetryTimeout time.Duration } func (ht *hcsTask) ID() string { @@ -453,7 +476,7 @@ func (ht *hcsTask) CreateExec(ctx context.Context, req *task.ExecProcessRequest, return errors.Wrapf(errdefs.ErrFailedPrecondition, "exec: '' in task: '%s' must be running to create additional execs", ht.id) } - io, err := cmd.NewUpstreamIO(ctx, req.ID, req.Stdout, req.Stderr, req.Stdin, req.Terminal) + io, err := cmd.NewUpstreamIO(ctx, req.ID, req.Stdout, req.Stderr, req.Stdin, req.Terminal, ht.ioRetryTimeout) if err != nil { return err } diff --git a/go.mod b/go.mod index 4d26c7c362..d08be0da17 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.17 + github.com/cenkalti/backoff/v4 v4.1.1 github.com/containerd/cgroups v1.0.1 github.com/containerd/console v1.0.2 github.com/containerd/containerd v1.5.7 diff --git a/go.sum b/go.sum index 572931b3bb..57b72d60d4 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index e91cf83b32..91a7da0277 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -131,24 +131,6 @@ func CommandContext(ctx context.Context, host cow.ProcessHost, name string, arg return cmd } -// relayIO is a glorified io.Copy that also logs when the copy has completed. -func relayIO(w io.Writer, r io.Reader, log *logrus.Entry, name string) (int64, error) { - n, err := io.Copy(w, r) - if log != nil { - lvl := logrus.DebugLevel - log = log.WithFields(logrus.Fields{ - "file": name, - "bytes": n, - }) - if err != nil { - lvl = logrus.ErrorLevel - log = log.WithError(err) - } - log.Log(lvl, "Cmd IO relay complete") - } - return n, err -} - // Start starts a command. The caller must ensure that if Start succeeds, // Wait is eventually called to clean up resources. func (c *Cmd) Start() error { diff --git a/internal/cmd/diag.go b/internal/cmd/diag.go index 092fcc207f..f4caff3251 100644 --- a/internal/cmd/diag.go +++ b/internal/cmd/diag.go @@ -15,7 +15,7 @@ func ExecInUvm(ctx context.Context, vm *uvm.UtilityVM, req *CmdProcessRequest) ( if len(req.Args) == 0 { return 0, errors.New("missing command") } - np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal) + np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal, 0) if err != nil { return 0, err } @@ -46,7 +46,7 @@ func ExecInShimHost(ctx context.Context, req *CmdProcessRequest) (int, error) { if len(req.Args) > 1 { cmdArgsWithoutName = req.Args[1:] } - np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal) + np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal, 0) if err != nil { return 0, err } diff --git a/internal/cmd/io.go b/internal/cmd/io.go index e2150c22aa..0912af160c 100644 --- a/internal/cmd/io.go +++ b/internal/cmd/io.go @@ -4,8 +4,10 @@ import ( "context" "io" "net/url" + "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // UpstreamIO is an interface describing the IO to connect to above the shim. @@ -43,12 +45,12 @@ type UpstreamIO interface { // NewUpstreamIO returns an UpstreamIO instance. Currently we only support named pipes and binary // logging driver for container IO. When using binary logger `stdout` and `stderr` are assumed to be // the same and the value of `stderr` is completely ignored. -func NewUpstreamIO(ctx context.Context, id string, stdout string, stderr string, stdin string, terminal bool) (UpstreamIO, error) { +func NewUpstreamIO(ctx context.Context, id, stdout, stderr, stdin string, terminal bool, ioRetryTimeout time.Duration) (UpstreamIO, error) { u, err := url.Parse(stdout) // Create IO with named pipes. if err != nil || u.Scheme == "" { - return NewNpipeIO(ctx, stdin, stdout, stderr, terminal) + return NewNpipeIO(ctx, stdin, stdout, stderr, terminal, ioRetryTimeout) } // Create IO for binary logging driver. @@ -58,3 +60,21 @@ func NewUpstreamIO(ctx context.Context, id string, stdout string, stderr string, return NewBinaryIO(ctx, id, u) } + +// relayIO is a glorified io.Copy that also logs when the copy has completed. +func relayIO(w io.Writer, r io.Reader, log *logrus.Entry, name string) (int64, error) { + n, err := io.Copy(w, r) + if log != nil { + lvl := logrus.DebugLevel + log = log.WithFields(logrus.Fields{ + "file": name, + "bytes": n, + }) + if err != nil { + lvl = logrus.ErrorLevel + log = log.WithError(err) + } + log.Log(lvl, "Cmd IO relay complete") + } + return n, err +} diff --git a/internal/cmd/io_npipe.go b/internal/cmd/io_npipe.go index e116362453..63b9f9b732 100644 --- a/internal/cmd/io_npipe.go +++ b/internal/cmd/io_npipe.go @@ -2,24 +2,37 @@ package cmd import ( "context" + "fmt" "io" + "math/rand" "net" "sync" + "syscall" + "time" winio "github.com/Microsoft/go-winio" "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/log" + "github.com/cenkalti/backoff/v4" "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" ) -// NewNpipeIO creates connected upstream io. It is the callers responsibility to -// validate that `if terminal == true`, `stderr == ""`. -func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool) (_ UpstreamIO, err error) { +func init() { + // Need to seed for the rng in backoff.NextBackoff() + rand.Seed(time.Now().UnixNano()) +} + +// NewNpipeIO creates connected upstream io. It is the callers responsibility to validate that `if terminal == true`, `stderr == ""`. retryTimeout +// refers to the timeout used to try and reconnect to the server end of the named pipe if the connection is severed. A value of 0 for retryTimeout +// is treated as an infinite timeout. +func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool, retryTimeout time.Duration) (_ UpstreamIO, err error) { log.G(ctx).WithFields(logrus.Fields{ "stdin": stdin, "stdout": stdout, "stderr": stderr, - "terminal": terminal}).Debug("NewNpipeIO") + "terminal": terminal, + }).Debug("NewNpipeIO") nio := &npipeio{ stdin: stdin, @@ -32,11 +45,16 @@ func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool nio.Close(ctx) } }() + if stdin != "" { c, err := winio.DialPipeContext(ctx, stdin) if err != nil { return nil, err } + // We don't have any retry logic for stdin as there's no good way to detect that we'd even need to retry. If the process forwarding + // stdin to the container (some client interface to exec a process in a container) exited, we'll get EOF which io.Copy treats as + // success. For fifos on Linux it seems if all fd's for the write end of the pipe dissappear, which is the same scenario, then + // the read end will get EOF as well. nio.sin = c } if stdout != "" { @@ -44,18 +62,112 @@ func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool if err != nil { return nil, err } - nio.sout = c + nio.sout = &nPipeRetryWriter{ctx, c, stdout, newBackOff(retryTimeout)} } if stderr != "" { c, err := winio.DialPipeContext(ctx, stderr) if err != nil { return nil, err } - nio.serr = c + nio.serr = &nPipeRetryWriter{ctx, c, stderr, newBackOff(retryTimeout)} } return nio, nil } +// nPipeRetryWriter is an io.Writer that wraps a net.Conn representing a named pipe connection. The retry logic is specifically only for +// disconnect scenarios (pipe broken, server went away etc.) to attempt to re-establish a connection, and is not for retrying writes on a busy pipe. +type nPipeRetryWriter struct { + ctx context.Context + net.Conn + pipePath string + backOff backoff.BackOff +} + +// newBackOff returns a new BackOff interface. The values chosen are fairly conservative, the main use is to get a somewhat random +// retry timeout on each ask. This can help avoid flooding a server all at once. +func newBackOff(timeout time.Duration) backoff.BackOff { + return &backoff.ExponentialBackOff{ + // First backoff timeout will be somewhere in the 100 - 300 ms range given the default multiplier. + InitialInterval: time.Millisecond * 200, + RandomizationFactor: backoff.DefaultRandomizationFactor, + Multiplier: backoff.DefaultMultiplier, + // Set the max interval to a minute, seems like a sane value. We don't know how long the server will be down for, and if we reached + // this point it's been down for quite awhile. + MaxInterval: time.Minute * 1, + // `backoff.ExponentialBackoff` treats a 0 timeout as infinite, which is ideal as it's the logic we desire. + MaxElapsedTime: timeout, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + } +} + +func (nprw *nPipeRetryWriter) Write(p []byte) (n int, err error) { + var currBufPos int + for { + // p[currBufPos:] to handle a case where we wrote n bytes but got disconnected and now we just need to write the rest of the buffer. If this is the + // first write then the current position is 0 so we just try and write the whole buffer as usual. + n, err = nprw.Conn.Write(p[currBufPos:]) + currBufPos += n + if err != nil { + // If the error is one that we can discern calls for a retry, attempt to redial the pipe. + if isDisconnectedErr(err) { + // Log that we're going to retry establishing the connection. + log.G(nprw.ctx).WithFields(logrus.Fields{ + "address": nprw.pipePath, + logrus.ErrorKey: err, + }).Error("Named pipe disconnected, retrying dial") + + // Close the old conn first. + nprw.Conn.Close() + newConn, retryErr := nprw.retryDialPipe() + if retryErr == nil { + log.G(nprw.ctx).WithField("address", nprw.pipePath).Info("Succeeded in reconnecting to named pipe") + + nprw.Conn = newConn + continue + } + err = retryErr + } + } + return currBufPos, err + } +} + +// retryDialPipe is a helper to retry dialing a named pipe until the timeout of nprw.BackOff or a successful connection. This is mainly to +// assist in scenarios where the server end of the pipe has crashed/went away and is no longer accepting new connections but may +// come back online. The backoff used inside is to try and space out connections to the server as to not flood it all at once with connection +// attempts at the same interval. +func (nprw *nPipeRetryWriter) retryDialPipe() (net.Conn, error) { + // Reset the backoff object as it starts ticking down when it's created. This also ensures we can re-use it in the event the server goes + // away more than once. + nprw.backOff.Reset() + for { + backOffTime := nprw.backOff.NextBackOff() + // We don't simply use a context with a timeout and pass it to DialPipe because DialPipe only retries the connection (and thus makes use of + // the timeout) if it sees that the pipe is busy. If the server isn't up/not listening it will just error out immediately and not make use + // of the timeout passed. That's the case we're most likely in right now so we need our own retry logic on top. + conn, err := winio.DialPipe(nprw.pipePath, nil) + if err == nil { + return conn, nil + } + // Next backoff would go over our timeout. We've tried once more above due to the ordering of this check, but now we need to bail out. + if backOffTime == backoff.Stop { + return nil, fmt.Errorf("reached timeout while retrying dial on %s", nprw.pipePath) + } + time.Sleep(backOffTime) + } +} + +// isDisconnectedErr is a helper to determine if the error received from writing to the server end of a named pipe indicates a disconnect/severed +// connection. This can be used to attempt a redial if it's expected that the server will come back online at some point. +func isDisconnectedErr(err error) bool { + if serr, ok := err.(syscall.Errno); ok { + // Server went away/something went wrong. + return serr == windows.ERROR_NO_DATA || serr == windows.ERROR_PIPE_NOT_CONNECTED || serr == windows.ERROR_BROKEN_PIPE + } + return false +} + var _ = (UpstreamIO)(&npipeio{}) type npipeio struct { diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index fcf41fba08..29bda6ccfc 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "strings" @@ -13,6 +12,7 @@ import ( "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 0042b3d24c..e62c9b4fe2 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -224,6 +224,7 @@ const ( // SecurityPolicy is used to specify a security policy for opengcs to enforce SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // ContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of // some kind (vhd, bind mount, fileshare mount etc.) diff --git a/test/go.sum b/test/go.sum index cb4394507a..fc7c6dfe83 100644 --- a/test/go.sum +++ b/test/go.sum @@ -78,6 +78,8 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/test/vendor/github.com/Microsoft/hcsshim/.golangci.yml b/test/vendor/github.com/Microsoft/hcsshim/.golangci.yml index 728178cff4..2830868fc4 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/.golangci.yml +++ b/test/vendor/github.com/Microsoft/hcsshim/.golangci.yml @@ -24,7 +24,7 @@ issues: linters: - stylecheck Text: "ST1003:" - + - path: cmd\\ncproxy\\nodenetsvc\\ linters: - stylecheck diff --git a/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go b/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go index fe11f8a2bb..89aff3723a 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go +++ b/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go @@ -139,7 +139,11 @@ type Options struct { // log_level specifies the logrus log level for the shim. Supported values are a string representation of the // logrus log levels: "trace", "debug", "info", "warn", "error", "fatal", "panic". This setting will override // the `debug` field if both are specified, unless the level specified is also "debug", as these are equivalent. - LogLevel string `protobuf:"bytes,16,opt,name=log_level,json=logLevel,proto3" json:"log_level,omitempty"` + LogLevel string `protobuf:"bytes,16,opt,name=log_level,json=logLevel,proto3" json:"log_level,omitempty"` + // io_retry_timeout_in_sec is the timeout in seconds for how long to try and reconnect to an upstream IO provider if a connection is lost. + // The typical example is if Containerd has restarted but is expected to come back online. A 0 for this field is interpreted as an infinite + // timeout. + IoRetryTimeoutInSec int32 `protobuf:"varint,17,opt,name=io_retry_timeout_in_sec,json=ioRetryTimeoutInSec,proto3" json:"io_retry_timeout_in_sec,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -238,65 +242,67 @@ func init() { } var fileDescriptor_b643df6839c75082 = []byte{ - // 920 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x95, 0x5b, 0x6f, 0xdb, 0x36, - 0x14, 0xc7, 0xad, 0x26, 0x71, 0xac, 0x93, 0x9b, 0xc3, 0x05, 0x98, 0x90, 0xac, 0xb6, 0x91, 0x0e, - 0x68, 0x8a, 0x35, 0x52, 0xd2, 0x3d, 0x6e, 0xc0, 0xd0, 0xd8, 0x4e, 0xeb, 0x21, 0x17, 0x43, 0xce, - 0xd2, 0x5d, 0x1e, 0x08, 0x5d, 0x18, 0x59, 0xa8, 0x28, 0x0a, 0x24, 0xed, 0xc5, 0x7d, 0xda, 0x47, - 0xd8, 0xc7, 0xca, 0xe3, 0x1e, 0x07, 0x0c, 0xc8, 0x56, 0x7f, 0x88, 0x3d, 0x0f, 0xa4, 0xa8, 0xb4, - 0x0b, 0x82, 0xbd, 0xec, 0xc9, 0xe4, 0xff, 0xfc, 0xf8, 0xe7, 0xe1, 0x21, 0x8f, 0x05, 0xe7, 0x49, - 0x2a, 0xc7, 0x93, 0xd0, 0x8d, 0x18, 0xf5, 0x4e, 0xd3, 0x88, 0x33, 0xc1, 0xae, 0xa4, 0x37, 0x8e, - 0x84, 0x18, 0xa7, 0xd4, 0x8b, 0x68, 0xec, 0x45, 0x2c, 0x97, 0x41, 0x9a, 0x13, 0x1e, 0xef, 0x2b, - 0x6d, 0x9f, 0x4f, 0xf2, 0x71, 0x24, 0xf6, 0xa7, 0x87, 0x1e, 0x2b, 0x64, 0xca, 0x72, 0xe1, 0x95, - 0x8a, 0x5b, 0x70, 0x26, 0x19, 0xda, 0xfa, 0xc0, 0xbb, 0x26, 0x30, 0x3d, 0xdc, 0xde, 0x4a, 0x58, - 0xc2, 0x34, 0xe0, 0xa9, 0x51, 0xc9, 0x6e, 0xb7, 0x13, 0xc6, 0x92, 0x8c, 0x78, 0x7a, 0x16, 0x4e, - 0xae, 0x3c, 0x99, 0x52, 0x22, 0x64, 0x40, 0x8b, 0x12, 0xd8, 0xfd, 0xbb, 0x0e, 0xcb, 0xe7, 0xe5, - 0x2e, 0x68, 0x0b, 0x96, 0x62, 0x12, 0x4e, 0x12, 0xc7, 0xea, 0x58, 0x7b, 0x0d, 0xbf, 0x9c, 0xa0, - 0x63, 0x00, 0x3d, 0xc0, 0x72, 0x56, 0x10, 0xe7, 0x51, 0xc7, 0xda, 0x5b, 0x7f, 0xf1, 0xd4, 0x7d, - 0x28, 0x07, 0xd7, 0x18, 0xb9, 0x3d, 0xc5, 0x5f, 0xcc, 0x0a, 0xe2, 0xdb, 0x71, 0x35, 0x44, 0x4f, - 0x60, 0x8d, 0x93, 0x24, 0x15, 0x92, 0xcf, 0x30, 0x67, 0x4c, 0x3a, 0x0b, 0x1d, 0x6b, 0xcf, 0xf6, - 0x57, 0x2b, 0xd1, 0x67, 0x4c, 0x2a, 0x48, 0x04, 0x79, 0x1c, 0xb2, 0x6b, 0x9c, 0xd2, 0x20, 0x21, - 0xce, 0x62, 0x09, 0x19, 0x71, 0xa0, 0x34, 0xf4, 0x0c, 0x9a, 0x15, 0x54, 0x64, 0x81, 0xbc, 0x62, - 0x9c, 0x3a, 0x4b, 0x9a, 0xdb, 0x30, 0xfa, 0xd0, 0xc8, 0xe8, 0x27, 0xd8, 0xbc, 0xf3, 0x13, 0x2c, - 0x0b, 0x54, 0x7e, 0x4e, 0x5d, 0x9f, 0xc1, 0xfd, 0xef, 0x33, 0x8c, 0xcc, 0x8e, 0xd5, 0x2a, 0xbf, - 0xda, 0xf3, 0x4e, 0x41, 0x1e, 0x6c, 0x85, 0x8c, 0x49, 0x7c, 0x95, 0x66, 0x44, 0xe8, 0x33, 0xe1, - 0x22, 0x90, 0x63, 0x67, 0x59, 0xe7, 0xb2, 0xa9, 0x62, 0xc7, 0x2a, 0xa4, 0x4e, 0x36, 0x0c, 0xe4, - 0x18, 0x3d, 0x07, 0x34, 0xa5, 0xb8, 0xe0, 0x2c, 0x22, 0x42, 0x30, 0x8e, 0x23, 0x36, 0xc9, 0xa5, - 0xd3, 0xe8, 0x58, 0x7b, 0x4b, 0x7e, 0x73, 0x4a, 0x87, 0x55, 0xa0, 0xab, 0x74, 0xe4, 0xc2, 0xd6, - 0x94, 0x62, 0x4a, 0x28, 0xe3, 0x33, 0x2c, 0xd2, 0x77, 0x04, 0xa7, 0x39, 0xa6, 0xa1, 0x63, 0x57, - 0xfc, 0xa9, 0x0e, 0x8d, 0xd2, 0x77, 0x64, 0x90, 0x9f, 0x86, 0xa8, 0x05, 0xf0, 0x6a, 0xf8, 0xdd, - 0xe5, 0xeb, 0x9e, 0xda, 0xcb, 0x01, 0x9d, 0xc4, 0x47, 0x0a, 0xfa, 0x1a, 0x76, 0x44, 0x14, 0x64, - 0x04, 0x47, 0xc5, 0x04, 0x67, 0x29, 0x4d, 0xa5, 0xc0, 0x92, 0x61, 0x73, 0x2c, 0x67, 0x45, 0x5f, - 0xfa, 0xa7, 0x1a, 0xe9, 0x16, 0x93, 0x13, 0x0d, 0x5c, 0x30, 0x53, 0x07, 0x74, 0x0a, 0x9f, 0xc7, - 0xe4, 0x2a, 0x98, 0x64, 0x12, 0xdf, 0xd5, 0x0d, 0x8b, 0x88, 0x07, 0x32, 0x1a, 0xdf, 0x65, 0x97, - 0x84, 0xce, 0xaa, 0xce, 0xae, 0x6d, 0xd8, 0x6e, 0x85, 0x8e, 0x4a, 0xb2, 0x4c, 0xf6, 0x55, 0x88, - 0xbe, 0x81, 0xc7, 0x95, 0xdd, 0x94, 0x3e, 0xe4, 0xb3, 0xa6, 0x7d, 0x1c, 0x03, 0x5d, 0xd2, 0xfb, - 0x06, 0xea, 0xa5, 0x8c, 0x03, 0x4e, 0xaa, 0xb5, 0xce, 0xba, 0xce, 0x7f, 0x55, 0x8b, 0x06, 0x46, - 0x1d, 0x58, 0x39, 0xeb, 0x0e, 0x39, 0xbb, 0x9e, 0xbd, 0x8c, 0x63, 0xee, 0x6c, 0xe8, 0x9a, 0x7c, - 0x2c, 0xa1, 0x1d, 0xb0, 0x33, 0x96, 0xe0, 0x8c, 0x4c, 0x49, 0xe6, 0x34, 0x75, 0xbc, 0x91, 0xb1, - 0xe4, 0x44, 0xcd, 0x77, 0x9f, 0x81, 0x7d, 0xf7, 0x94, 0x91, 0x0d, 0x4b, 0x67, 0xc3, 0xc1, 0xb0, - 0xdf, 0xac, 0xa1, 0x06, 0x2c, 0x1e, 0x0f, 0x4e, 0xfa, 0x4d, 0x0b, 0x2d, 0xc3, 0x42, 0xff, 0xe2, - 0x4d, 0xf3, 0xd1, 0xae, 0x07, 0xcd, 0xfb, 0x2f, 0x06, 0xad, 0xc0, 0xf2, 0xd0, 0x3f, 0xef, 0xf6, - 0x47, 0xa3, 0x66, 0x0d, 0xad, 0x03, 0xbc, 0xfe, 0x61, 0xd8, 0xf7, 0x2f, 0x07, 0xa3, 0x73, 0xbf, - 0x69, 0xed, 0xfe, 0xb1, 0x00, 0xeb, 0xe6, 0xc2, 0x7b, 0x44, 0x06, 0x69, 0x26, 0xd0, 0x63, 0x00, - 0xfd, 0xe8, 0x71, 0x1e, 0x50, 0xa2, 0x9b, 0xd0, 0xf6, 0x6d, 0xad, 0x9c, 0x05, 0x94, 0xa0, 0x2e, - 0x40, 0xc4, 0x49, 0x20, 0x49, 0x8c, 0x03, 0xa9, 0x1b, 0x71, 0xe5, 0xc5, 0xb6, 0x5b, 0x36, 0xb8, - 0x5b, 0x35, 0xb8, 0x7b, 0x51, 0x35, 0xf8, 0x51, 0xe3, 0xe6, 0xb6, 0x5d, 0xfb, 0xf5, 0xcf, 0xb6, - 0xe5, 0xdb, 0x66, 0xdd, 0x4b, 0x89, 0xbe, 0x00, 0xf4, 0x96, 0xf0, 0x9c, 0x64, 0x58, 0xfd, 0x13, - 0xe0, 0xc3, 0x83, 0x03, 0x9c, 0x0b, 0xdd, 0x8a, 0x8b, 0xfe, 0x46, 0x19, 0x51, 0x0e, 0x87, 0x07, - 0x07, 0x67, 0x02, 0xb9, 0xf0, 0x89, 0x79, 0x7e, 0x11, 0xa3, 0x34, 0x95, 0x38, 0x9c, 0x49, 0x22, - 0x74, 0x4f, 0x2e, 0xfa, 0x9b, 0x65, 0xa8, 0xab, 0x23, 0x47, 0x2a, 0x80, 0x8e, 0xa1, 0x63, 0xf8, - 0x9f, 0x19, 0x7f, 0x9b, 0xe6, 0x09, 0x16, 0x44, 0xe2, 0x82, 0xa7, 0xd3, 0x40, 0x12, 0xb3, 0x78, - 0x49, 0x2f, 0xfe, 0xac, 0xe4, 0xde, 0x94, 0xd8, 0x88, 0xc8, 0x61, 0x09, 0x95, 0x3e, 0x3d, 0x68, - 0x3f, 0xe0, 0xa3, 0x6f, 0x36, 0x36, 0x36, 0x75, 0x6d, 0xb3, 0x73, 0xdf, 0x66, 0xa4, 0x99, 0xd2, - 0xe5, 0x39, 0x80, 0x69, 0x35, 0x9c, 0xc6, 0xba, 0x29, 0xd7, 0x8e, 0xd6, 0xe6, 0xb7, 0x6d, 0xdb, - 0x94, 0x7d, 0xd0, 0xf3, 0x6d, 0x03, 0x0c, 0x62, 0xf4, 0x14, 0x9a, 0x13, 0x41, 0xf8, 0xbf, 0xca, - 0xd2, 0xd0, 0x9b, 0xac, 0x29, 0xfd, 0x43, 0x51, 0x9e, 0xc0, 0x32, 0xb9, 0x26, 0x91, 0xf2, 0x54, - 0x9d, 0x68, 0x1f, 0xc1, 0xfc, 0xb6, 0x5d, 0xef, 0x5f, 0x93, 0x68, 0xd0, 0xf3, 0xeb, 0x2a, 0x34, - 0x88, 0x8f, 0xe2, 0x9b, 0xf7, 0xad, 0xda, 0xef, 0xef, 0x5b, 0xb5, 0x5f, 0xe6, 0x2d, 0xeb, 0x66, - 0xde, 0xb2, 0x7e, 0x9b, 0xb7, 0xac, 0xbf, 0xe6, 0x2d, 0xeb, 0xc7, 0x6f, 0xff, 0xff, 0xe7, 0xe0, - 0x2b, 0xf3, 0xfb, 0x7d, 0x2d, 0xac, 0xeb, 0x7b, 0xff, 0xf2, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x84, 0x30, 0x9b, 0x7c, 0x65, 0x06, 0x00, 0x00, + // 953 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x5d, 0x6f, 0xdb, 0x36, + 0x17, 0xb6, 0xda, 0x24, 0xb6, 0x4e, 0xbe, 0x1c, 0x36, 0x40, 0x85, 0xe4, 0xad, 0x6d, 0xa4, 0x2f, + 0xd0, 0x14, 0x6b, 0xa4, 0xa4, 0xdb, 0xdd, 0x06, 0x0c, 0x8d, 0xed, 0xb4, 0x1e, 0xf2, 0x61, 0xc8, + 0x59, 0xba, 0x8f, 0x0b, 0x42, 0x1f, 0x8c, 0x4c, 0x54, 0x12, 0x05, 0x92, 0xf6, 0xe2, 0x5e, 0xed, + 0x27, 0xec, 0x87, 0xec, 0x87, 0xe4, 0x72, 0x97, 0x03, 0x06, 0x64, 0xab, 0x7f, 0xc9, 0x40, 0x8a, + 0x4a, 0xbb, 0x20, 0xd8, 0xcd, 0xae, 0x4c, 0x3e, 0xcf, 0xc3, 0x87, 0xe7, 0x1c, 0x9d, 0x43, 0xc3, + 0x59, 0x42, 0xe5, 0x78, 0x12, 0xba, 0x11, 0xcb, 0xbc, 0x13, 0x1a, 0x71, 0x26, 0xd8, 0xa5, 0xf4, + 0xc6, 0x91, 0x10, 0x63, 0x9a, 0x79, 0x51, 0x16, 0x7b, 0x11, 0xcb, 0x65, 0x40, 0x73, 0xc2, 0xe3, + 0x3d, 0x85, 0xed, 0xf1, 0x49, 0x3e, 0x8e, 0xc4, 0xde, 0xf4, 0xc0, 0x63, 0x85, 0xa4, 0x2c, 0x17, + 0x5e, 0x89, 0xb8, 0x05, 0x67, 0x92, 0xa1, 0xcd, 0x8f, 0x7a, 0xd7, 0x10, 0xd3, 0x83, 0xad, 0xcd, + 0x84, 0x25, 0x4c, 0x0b, 0x3c, 0xb5, 0x2a, 0xb5, 0x5b, 0xed, 0x84, 0xb1, 0x24, 0x25, 0x9e, 0xde, + 0x85, 0x93, 0x4b, 0x4f, 0xd2, 0x8c, 0x08, 0x19, 0x64, 0x45, 0x29, 0xd8, 0xf9, 0xb5, 0x0e, 0xf5, + 0xb3, 0xf2, 0x16, 0xb4, 0x09, 0x8b, 0x31, 0x09, 0x27, 0x89, 0x63, 0x75, 0xac, 0xdd, 0x86, 0x5f, + 0x6e, 0xd0, 0x11, 0x80, 0x5e, 0x60, 0x39, 0x2b, 0x88, 0xf3, 0xa0, 0x63, 0xed, 0xae, 0xbd, 0x7c, + 0xe6, 0xde, 0x17, 0x83, 0x6b, 0x8c, 0xdc, 0x9e, 0xd2, 0x9f, 0xcf, 0x0a, 0xe2, 0xdb, 0x71, 0xb5, + 0x44, 0x4f, 0x61, 0x95, 0x93, 0x84, 0x0a, 0xc9, 0x67, 0x98, 0x33, 0x26, 0x9d, 0x87, 0x1d, 0x6b, + 0xd7, 0xf6, 0x57, 0x2a, 0xd0, 0x67, 0x4c, 0x2a, 0x91, 0x08, 0xf2, 0x38, 0x64, 0x57, 0x98, 0x66, + 0x41, 0x42, 0x9c, 0x85, 0x52, 0x64, 0xc0, 0x81, 0xc2, 0xd0, 0x73, 0x68, 0x56, 0xa2, 0x22, 0x0d, + 0xe4, 0x25, 0xe3, 0x99, 0xb3, 0xa8, 0x75, 0xeb, 0x06, 0x1f, 0x1a, 0x18, 0xfd, 0x08, 0x1b, 0xb7, + 0x7e, 0x82, 0xa5, 0x81, 0x8a, 0xcf, 0x59, 0xd2, 0x39, 0xb8, 0xff, 0x9e, 0xc3, 0xc8, 0xdc, 0x58, + 0x9d, 0xf2, 0xab, 0x3b, 0x6f, 0x11, 0xe4, 0xc1, 0x66, 0xc8, 0x98, 0xc4, 0x97, 0x34, 0x25, 0x42, + 0xe7, 0x84, 0x8b, 0x40, 0x8e, 0x9d, 0xba, 0x8e, 0x65, 0x43, 0x71, 0x47, 0x8a, 0x52, 0x99, 0x0d, + 0x03, 0x39, 0x46, 0x2f, 0x00, 0x4d, 0x33, 0x5c, 0x70, 0x16, 0x11, 0x21, 0x18, 0xc7, 0x11, 0x9b, + 0xe4, 0xd2, 0x69, 0x74, 0xac, 0xdd, 0x45, 0xbf, 0x39, 0xcd, 0x86, 0x15, 0xd1, 0x55, 0x38, 0x72, + 0x61, 0x73, 0x9a, 0xe1, 0x8c, 0x64, 0x8c, 0xcf, 0xb0, 0xa0, 0xef, 0x09, 0xa6, 0x39, 0xce, 0x42, + 0xc7, 0xae, 0xf4, 0x27, 0x9a, 0x1a, 0xd1, 0xf7, 0x64, 0x90, 0x9f, 0x84, 0xa8, 0x05, 0xf0, 0x7a, + 0xf8, 0xed, 0xc5, 0x9b, 0x9e, 0xba, 0xcb, 0x01, 0x1d, 0xc4, 0x27, 0x08, 0xfa, 0x0a, 0xb6, 0x45, + 0x14, 0xa4, 0x04, 0x47, 0xc5, 0x04, 0xa7, 0x34, 0xa3, 0x52, 0x60, 0xc9, 0xb0, 0x49, 0xcb, 0x59, + 0xd6, 0x1f, 0xfd, 0xb1, 0x96, 0x74, 0x8b, 0xc9, 0xb1, 0x16, 0x9c, 0x33, 0x53, 0x07, 0x74, 0x02, + 0xff, 0x8f, 0xc9, 0x65, 0x30, 0x49, 0x25, 0xbe, 0xad, 0x1b, 0x16, 0x11, 0x0f, 0x64, 0x34, 0xbe, + 0x8d, 0x2e, 0x09, 0x9d, 0x15, 0x1d, 0x5d, 0xdb, 0x68, 0xbb, 0x95, 0x74, 0x54, 0x2a, 0xcb, 0x60, + 0x5f, 0x87, 0xe8, 0x6b, 0x78, 0x52, 0xd9, 0x4d, 0xb3, 0xfb, 0x7c, 0x56, 0xb5, 0x8f, 0x63, 0x44, + 0x17, 0xd9, 0x5d, 0x03, 0xd5, 0x29, 0xe3, 0x80, 0x93, 0xea, 0xac, 0xb3, 0xa6, 0xe3, 0x5f, 0xd1, + 0xa0, 0x11, 0xa3, 0x0e, 0x2c, 0x9f, 0x76, 0x87, 0x9c, 0x5d, 0xcd, 0x5e, 0xc5, 0x31, 0x77, 0xd6, + 0x75, 0x4d, 0x3e, 0x85, 0xd0, 0x36, 0xd8, 0x29, 0x4b, 0x70, 0x4a, 0xa6, 0x24, 0x75, 0x9a, 0x9a, + 0x6f, 0xa4, 0x2c, 0x39, 0x56, 0x7b, 0xf4, 0x05, 0x3c, 0xa6, 0x0c, 0x73, 0xa2, 0x5a, 0x56, 0x0d, + 0x0e, 0x9b, 0x48, 0x15, 0x9d, 0x20, 0x91, 0xb3, 0xa1, 0xc3, 0x7b, 0x44, 0x99, 0xaf, 0xd8, 0xf3, + 0x92, 0x1c, 0xe4, 0x23, 0x12, 0xed, 0x3c, 0x07, 0xfb, 0x76, 0x00, 0x90, 0x0d, 0x8b, 0xa7, 0xc3, + 0xc1, 0xb0, 0xdf, 0xac, 0xa1, 0x06, 0x2c, 0x1c, 0x0d, 0x8e, 0xfb, 0x4d, 0x0b, 0xd5, 0xe1, 0x61, + 0xff, 0xfc, 0x6d, 0xf3, 0xc1, 0x8e, 0x07, 0xcd, 0xbb, 0x7d, 0x86, 0x96, 0xa1, 0x3e, 0xf4, 0xcf, + 0xba, 0xfd, 0xd1, 0xa8, 0x59, 0x43, 0x6b, 0x00, 0x6f, 0xbe, 0x1f, 0xf6, 0xfd, 0x8b, 0xc1, 0xe8, + 0xcc, 0x6f, 0x5a, 0x3b, 0x7f, 0x3c, 0x84, 0x35, 0xd3, 0x26, 0x3d, 0x22, 0x03, 0x9a, 0x0a, 0xf4, + 0x04, 0x40, 0x8f, 0x0a, 0xce, 0x83, 0x8c, 0xe8, 0xd1, 0xb5, 0x7d, 0x5b, 0x23, 0xa7, 0x41, 0x46, + 0x50, 0x17, 0x20, 0xe2, 0x24, 0x90, 0x24, 0xc6, 0x81, 0xd4, 0xe3, 0xbb, 0xfc, 0x72, 0xcb, 0x2d, + 0x9f, 0x05, 0xb7, 0x7a, 0x16, 0xdc, 0xf3, 0xea, 0x59, 0x38, 0x6c, 0x5c, 0xdf, 0xb4, 0x6b, 0xbf, + 0xfc, 0xd9, 0xb6, 0x7c, 0xdb, 0x9c, 0x7b, 0x25, 0xd1, 0x67, 0x80, 0xde, 0x11, 0x9e, 0x93, 0x54, + 0x97, 0x01, 0x1f, 0xec, 0xef, 0xe3, 0x5c, 0xe8, 0x01, 0x5e, 0xf0, 0xd7, 0x4b, 0x46, 0x39, 0x1c, + 0xec, 0xef, 0x9f, 0x0a, 0xe4, 0xc2, 0x23, 0xd3, 0xb4, 0x11, 0xcb, 0x32, 0x2a, 0x71, 0x38, 0x93, + 0x44, 0xe8, 0x49, 0x5e, 0xf0, 0x37, 0x4a, 0xaa, 0xab, 0x99, 0x43, 0x45, 0xa0, 0x23, 0xe8, 0x18, + 0xfd, 0x4f, 0x8c, 0xbf, 0xa3, 0x79, 0x82, 0x05, 0x91, 0xb8, 0xe0, 0x74, 0x1a, 0x48, 0x62, 0x0e, + 0x2f, 0xea, 0xc3, 0xff, 0x2b, 0x75, 0x6f, 0x4b, 0xd9, 0x88, 0xc8, 0x61, 0x29, 0x2a, 0x7d, 0x7a, + 0xd0, 0xbe, 0xc7, 0x47, 0xf7, 0x43, 0x6c, 0x6c, 0x96, 0xb4, 0xcd, 0xf6, 0x5d, 0x9b, 0x91, 0xd6, + 0x94, 0x2e, 0x2f, 0x00, 0xcc, 0x80, 0x62, 0x1a, 0xeb, 0x51, 0x5e, 0x3d, 0x5c, 0x9d, 0xdf, 0xb4, + 0x6d, 0x53, 0xf6, 0x41, 0xcf, 0xb7, 0x8d, 0x60, 0x10, 0xa3, 0x67, 0xd0, 0x9c, 0x08, 0xc2, 0xff, + 0x51, 0x96, 0x86, 0xbe, 0x64, 0x55, 0xe1, 0x1f, 0x8b, 0xf2, 0x14, 0xea, 0xe4, 0x8a, 0x44, 0xca, + 0x53, 0xcd, 0xaf, 0x7d, 0x08, 0xf3, 0x9b, 0xf6, 0x52, 0xff, 0x8a, 0x44, 0x83, 0x9e, 0xbf, 0xa4, + 0xa8, 0x41, 0x7c, 0x18, 0x5f, 0x7f, 0x68, 0xd5, 0x7e, 0xff, 0xd0, 0xaa, 0xfd, 0x3c, 0x6f, 0x59, + 0xd7, 0xf3, 0x96, 0xf5, 0xdb, 0xbc, 0x65, 0xfd, 0x35, 0x6f, 0x59, 0x3f, 0x7c, 0xf3, 0xdf, 0xff, + 0x44, 0xbe, 0x34, 0xbf, 0xdf, 0xd5, 0xc2, 0x25, 0xfd, 0xdd, 0x3f, 0xff, 0x3b, 0x00, 0x00, 0xff, + 0xff, 0x6b, 0x83, 0xa6, 0x5f, 0x9b, 0x06, 0x00, 0x00, } func (m *Options) Marshal() (dAtA []byte, err error) { @@ -418,6 +424,13 @@ func (m *Options) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintRunhcs(dAtA, i, uint64(len(m.LogLevel))) i += copy(dAtA[i:], m.LogLevel) } + if m.IoRetryTimeoutInSec != 0 { + dAtA[i] = 0x88 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRunhcs(dAtA, i, uint64(m.IoRetryTimeoutInSec)) + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -565,6 +578,9 @@ func (m *Options) Size() (n int) { if l > 0 { n += 2 + l + sovRunhcs(uint64(l)) } + if m.IoRetryTimeoutInSec != 0 { + n += 2 + sovRunhcs(uint64(m.IoRetryTimeoutInSec)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -645,6 +661,7 @@ func (this *Options) String() string { `ShareScratch:` + fmt.Sprintf("%v", this.ShareScratch) + `,`, `NCProxyAddr:` + fmt.Sprintf("%v", this.NCProxyAddr) + `,`, `LogLevel:` + fmt.Sprintf("%v", this.LogLevel) + `,`, + `IoRetryTimeoutInSec:` + fmt.Sprintf("%v", this.IoRetryTimeoutInSec) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -1104,6 +1121,25 @@ func (m *Options) Unmarshal(dAtA []byte) error { } m.LogLevel = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 17: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IoRetryTimeoutInSec", wireType) + } + m.IoRetryTimeoutInSec = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRunhcs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IoRetryTimeoutInSec |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipRunhcs(dAtA[iNdEx:]) diff --git a/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto b/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto index 90f3376ffc..60c89adbde 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto +++ b/test/vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto @@ -93,6 +93,11 @@ message Options { // logrus log levels: "trace", "debug", "info", "warn", "error", "fatal", "panic". This setting will override // the `debug` field if both are specified, unless the level specified is also "debug", as these are equivalent. string log_level = 16; + + // io_retry_timeout_in_sec is the timeout in seconds for how long to try and reconnect to an upstream IO provider if a connection is lost. + // The typical example is if Containerd has restarted but is expected to come back online. A 0 for this field is interpreted as an infinite + // timeout. + int32 io_retry_timeout_in_sec = 17; } // ProcessDetails contains additional information about a process. This is the additional diff --git a/test/vendor/github.com/Microsoft/hcsshim/go.mod b/test/vendor/github.com/Microsoft/hcsshim/go.mod index 4d26c7c362..d08be0da17 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/go.mod +++ b/test/vendor/github.com/Microsoft/hcsshim/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.17 + github.com/cenkalti/backoff/v4 v4.1.1 github.com/containerd/cgroups v1.0.1 github.com/containerd/console v1.0.2 github.com/containerd/containerd v1.5.7 diff --git a/test/vendor/github.com/Microsoft/hcsshim/go.sum b/test/vendor/github.com/Microsoft/hcsshim/go.sum index 572931b3bb..57b72d60d4 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/go.sum +++ b/test/vendor/github.com/Microsoft/hcsshim/go.sum @@ -86,6 +86,8 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/cmd.go b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/cmd.go index e91cf83b32..91a7da0277 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/cmd.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/cmd.go @@ -131,24 +131,6 @@ func CommandContext(ctx context.Context, host cow.ProcessHost, name string, arg return cmd } -// relayIO is a glorified io.Copy that also logs when the copy has completed. -func relayIO(w io.Writer, r io.Reader, log *logrus.Entry, name string) (int64, error) { - n, err := io.Copy(w, r) - if log != nil { - lvl := logrus.DebugLevel - log = log.WithFields(logrus.Fields{ - "file": name, - "bytes": n, - }) - if err != nil { - lvl = logrus.ErrorLevel - log = log.WithError(err) - } - log.Log(lvl, "Cmd IO relay complete") - } - return n, err -} - // Start starts a command. The caller must ensure that if Start succeeds, // Wait is eventually called to clean up resources. func (c *Cmd) Start() error { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go index 092fcc207f..f4caff3251 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go @@ -15,7 +15,7 @@ func ExecInUvm(ctx context.Context, vm *uvm.UtilityVM, req *CmdProcessRequest) ( if len(req.Args) == 0 { return 0, errors.New("missing command") } - np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal) + np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal, 0) if err != nil { return 0, err } @@ -46,7 +46,7 @@ func ExecInShimHost(ctx context.Context, req *CmdProcessRequest) (int, error) { if len(req.Args) > 1 { cmdArgsWithoutName = req.Args[1:] } - np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal) + np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal, 0) if err != nil { return 0, err } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io.go b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io.go index e2150c22aa..0912af160c 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io.go @@ -4,8 +4,10 @@ import ( "context" "io" "net/url" + "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // UpstreamIO is an interface describing the IO to connect to above the shim. @@ -43,12 +45,12 @@ type UpstreamIO interface { // NewUpstreamIO returns an UpstreamIO instance. Currently we only support named pipes and binary // logging driver for container IO. When using binary logger `stdout` and `stderr` are assumed to be // the same and the value of `stderr` is completely ignored. -func NewUpstreamIO(ctx context.Context, id string, stdout string, stderr string, stdin string, terminal bool) (UpstreamIO, error) { +func NewUpstreamIO(ctx context.Context, id, stdout, stderr, stdin string, terminal bool, ioRetryTimeout time.Duration) (UpstreamIO, error) { u, err := url.Parse(stdout) // Create IO with named pipes. if err != nil || u.Scheme == "" { - return NewNpipeIO(ctx, stdin, stdout, stderr, terminal) + return NewNpipeIO(ctx, stdin, stdout, stderr, terminal, ioRetryTimeout) } // Create IO for binary logging driver. @@ -58,3 +60,21 @@ func NewUpstreamIO(ctx context.Context, id string, stdout string, stderr string, return NewBinaryIO(ctx, id, u) } + +// relayIO is a glorified io.Copy that also logs when the copy has completed. +func relayIO(w io.Writer, r io.Reader, log *logrus.Entry, name string) (int64, error) { + n, err := io.Copy(w, r) + if log != nil { + lvl := logrus.DebugLevel + log = log.WithFields(logrus.Fields{ + "file": name, + "bytes": n, + }) + if err != nil { + lvl = logrus.ErrorLevel + log = log.WithError(err) + } + log.Log(lvl, "Cmd IO relay complete") + } + return n, err +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io_npipe.go b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io_npipe.go index e116362453..63b9f9b732 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io_npipe.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/io_npipe.go @@ -2,24 +2,37 @@ package cmd import ( "context" + "fmt" "io" + "math/rand" "net" "sync" + "syscall" + "time" winio "github.com/Microsoft/go-winio" "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/log" + "github.com/cenkalti/backoff/v4" "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" ) -// NewNpipeIO creates connected upstream io. It is the callers responsibility to -// validate that `if terminal == true`, `stderr == ""`. -func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool) (_ UpstreamIO, err error) { +func init() { + // Need to seed for the rng in backoff.NextBackoff() + rand.Seed(time.Now().UnixNano()) +} + +// NewNpipeIO creates connected upstream io. It is the callers responsibility to validate that `if terminal == true`, `stderr == ""`. retryTimeout +// refers to the timeout used to try and reconnect to the server end of the named pipe if the connection is severed. A value of 0 for retryTimeout +// is treated as an infinite timeout. +func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool, retryTimeout time.Duration) (_ UpstreamIO, err error) { log.G(ctx).WithFields(logrus.Fields{ "stdin": stdin, "stdout": stdout, "stderr": stderr, - "terminal": terminal}).Debug("NewNpipeIO") + "terminal": terminal, + }).Debug("NewNpipeIO") nio := &npipeio{ stdin: stdin, @@ -32,11 +45,16 @@ func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool nio.Close(ctx) } }() + if stdin != "" { c, err := winio.DialPipeContext(ctx, stdin) if err != nil { return nil, err } + // We don't have any retry logic for stdin as there's no good way to detect that we'd even need to retry. If the process forwarding + // stdin to the container (some client interface to exec a process in a container) exited, we'll get EOF which io.Copy treats as + // success. For fifos on Linux it seems if all fd's for the write end of the pipe dissappear, which is the same scenario, then + // the read end will get EOF as well. nio.sin = c } if stdout != "" { @@ -44,18 +62,112 @@ func NewNpipeIO(ctx context.Context, stdin, stdout, stderr string, terminal bool if err != nil { return nil, err } - nio.sout = c + nio.sout = &nPipeRetryWriter{ctx, c, stdout, newBackOff(retryTimeout)} } if stderr != "" { c, err := winio.DialPipeContext(ctx, stderr) if err != nil { return nil, err } - nio.serr = c + nio.serr = &nPipeRetryWriter{ctx, c, stderr, newBackOff(retryTimeout)} } return nio, nil } +// nPipeRetryWriter is an io.Writer that wraps a net.Conn representing a named pipe connection. The retry logic is specifically only for +// disconnect scenarios (pipe broken, server went away etc.) to attempt to re-establish a connection, and is not for retrying writes on a busy pipe. +type nPipeRetryWriter struct { + ctx context.Context + net.Conn + pipePath string + backOff backoff.BackOff +} + +// newBackOff returns a new BackOff interface. The values chosen are fairly conservative, the main use is to get a somewhat random +// retry timeout on each ask. This can help avoid flooding a server all at once. +func newBackOff(timeout time.Duration) backoff.BackOff { + return &backoff.ExponentialBackOff{ + // First backoff timeout will be somewhere in the 100 - 300 ms range given the default multiplier. + InitialInterval: time.Millisecond * 200, + RandomizationFactor: backoff.DefaultRandomizationFactor, + Multiplier: backoff.DefaultMultiplier, + // Set the max interval to a minute, seems like a sane value. We don't know how long the server will be down for, and if we reached + // this point it's been down for quite awhile. + MaxInterval: time.Minute * 1, + // `backoff.ExponentialBackoff` treats a 0 timeout as infinite, which is ideal as it's the logic we desire. + MaxElapsedTime: timeout, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + } +} + +func (nprw *nPipeRetryWriter) Write(p []byte) (n int, err error) { + var currBufPos int + for { + // p[currBufPos:] to handle a case where we wrote n bytes but got disconnected and now we just need to write the rest of the buffer. If this is the + // first write then the current position is 0 so we just try and write the whole buffer as usual. + n, err = nprw.Conn.Write(p[currBufPos:]) + currBufPos += n + if err != nil { + // If the error is one that we can discern calls for a retry, attempt to redial the pipe. + if isDisconnectedErr(err) { + // Log that we're going to retry establishing the connection. + log.G(nprw.ctx).WithFields(logrus.Fields{ + "address": nprw.pipePath, + logrus.ErrorKey: err, + }).Error("Named pipe disconnected, retrying dial") + + // Close the old conn first. + nprw.Conn.Close() + newConn, retryErr := nprw.retryDialPipe() + if retryErr == nil { + log.G(nprw.ctx).WithField("address", nprw.pipePath).Info("Succeeded in reconnecting to named pipe") + + nprw.Conn = newConn + continue + } + err = retryErr + } + } + return currBufPos, err + } +} + +// retryDialPipe is a helper to retry dialing a named pipe until the timeout of nprw.BackOff or a successful connection. This is mainly to +// assist in scenarios where the server end of the pipe has crashed/went away and is no longer accepting new connections but may +// come back online. The backoff used inside is to try and space out connections to the server as to not flood it all at once with connection +// attempts at the same interval. +func (nprw *nPipeRetryWriter) retryDialPipe() (net.Conn, error) { + // Reset the backoff object as it starts ticking down when it's created. This also ensures we can re-use it in the event the server goes + // away more than once. + nprw.backOff.Reset() + for { + backOffTime := nprw.backOff.NextBackOff() + // We don't simply use a context with a timeout and pass it to DialPipe because DialPipe only retries the connection (and thus makes use of + // the timeout) if it sees that the pipe is busy. If the server isn't up/not listening it will just error out immediately and not make use + // of the timeout passed. That's the case we're most likely in right now so we need our own retry logic on top. + conn, err := winio.DialPipe(nprw.pipePath, nil) + if err == nil { + return conn, nil + } + // Next backoff would go over our timeout. We've tried once more above due to the ordering of this check, but now we need to bail out. + if backOffTime == backoff.Stop { + return nil, fmt.Errorf("reached timeout while retrying dial on %s", nprw.pipePath) + } + time.Sleep(backOffTime) + } +} + +// isDisconnectedErr is a helper to determine if the error received from writing to the server end of a named pipe indicates a disconnect/severed +// connection. This can be used to attempt a redial if it's expected that the server will come back online at some point. +func isDisconnectedErr(err error) bool { + if serr, ok := err.(syscall.Errno); ok { + // Server went away/something went wrong. + return serr == windows.ERROR_NO_DATA || serr == windows.ERROR_PIPE_NOT_CONNECTED || serr == windows.ERROR_BROKEN_PIPE + } + return false +} + var _ = (UpstreamIO)(&npipeio{}) type npipeio struct { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go index fcf41fba08..29bda6ccfc 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "strings" @@ -13,6 +12,7 @@ import ( "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go index 0042b3d24c..e62c9b4fe2 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go @@ -224,6 +224,7 @@ const ( // SecurityPolicy is used to specify a security policy for opengcs to enforce SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // ContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of // some kind (vhd, bind mount, fileshare mount etc.) diff --git a/test/vendor/github.com/cenkalti/backoff/v4/.gitignore b/test/vendor/github.com/cenkalti/backoff/v4/.gitignore new file mode 100644 index 0000000000..50d95c548b --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +# IDEs +.idea/ diff --git a/test/vendor/github.com/cenkalti/backoff/v4/.travis.yml b/test/vendor/github.com/cenkalti/backoff/v4/.travis.yml new file mode 100644 index 0000000000..c79105c2fb --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/.travis.yml @@ -0,0 +1,10 @@ +language: go +go: + - 1.13 + - 1.x + - tip +before_install: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover +script: + - $HOME/gopath/bin/goveralls -service=travis-ci diff --git a/test/vendor/github.com/cenkalti/backoff/v4/LICENSE b/test/vendor/github.com/cenkalti/backoff/v4/LICENSE new file mode 100644 index 0000000000..89b8179965 --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Cenk Altı + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/test/vendor/github.com/cenkalti/backoff/v4/README.md b/test/vendor/github.com/cenkalti/backoff/v4/README.md new file mode 100644 index 0000000000..16abdfc084 --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/README.md @@ -0,0 +1,32 @@ +# Exponential Backoff [![GoDoc][godoc image]][godoc] [![Build Status][travis image]][travis] [![Coverage Status][coveralls image]][coveralls] + +This is a Go port of the exponential backoff algorithm from [Google's HTTP Client Library for Java][google-http-java-client]. + +[Exponential backoff][exponential backoff wiki] +is an algorithm that uses feedback to multiplicatively decrease the rate of some process, +in order to gradually find an acceptable rate. +The retries exponentially increase and stop increasing when a certain threshold is met. + +## Usage + +Import path is `github.com/cenkalti/backoff/v4`. Please note the version part at the end. + +Use https://pkg.go.dev/github.com/cenkalti/backoff/v4 to view the documentation. + +## Contributing + +* I would like to keep this library as small as possible. +* Please don't send a PR without opening an issue and discussing it first. +* If proposed change is not a common use case, I will probably not accept it. + +[godoc]: https://pkg.go.dev/github.com/cenkalti/backoff/v4 +[godoc image]: https://godoc.org/github.com/cenkalti/backoff?status.png +[travis]: https://travis-ci.org/cenkalti/backoff +[travis image]: https://travis-ci.org/cenkalti/backoff.png?branch=master +[coveralls]: https://coveralls.io/github/cenkalti/backoff?branch=master +[coveralls image]: https://coveralls.io/repos/github/cenkalti/backoff/badge.svg?branch=master + +[google-http-java-client]: https://github.com/google/google-http-java-client/blob/da1aa993e90285ec18579f1553339b00e19b3ab5/google-http-client/src/main/java/com/google/api/client/util/ExponentialBackOff.java +[exponential backoff wiki]: http://en.wikipedia.org/wiki/Exponential_backoff + +[advanced example]: https://pkg.go.dev/github.com/cenkalti/backoff/v4?tab=doc#pkg-examples diff --git a/test/vendor/github.com/cenkalti/backoff/v4/backoff.go b/test/vendor/github.com/cenkalti/backoff/v4/backoff.go new file mode 100644 index 0000000000..3676ee405d --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/backoff.go @@ -0,0 +1,66 @@ +// Package backoff implements backoff algorithms for retrying operations. +// +// Use Retry function for retrying operations that may fail. +// If Retry does not meet your needs, +// copy/paste the function into your project and modify as you wish. +// +// There is also Ticker type similar to time.Ticker. +// You can use it if you need to work with channels. +// +// See Examples section below for usage examples. +package backoff + +import "time" + +// BackOff is a backoff policy for retrying an operation. +type BackOff interface { + // NextBackOff returns the duration to wait before retrying the operation, + // or backoff. Stop to indicate that no more retries should be made. + // + // Example usage: + // + // duration := backoff.NextBackOff(); + // if (duration == backoff.Stop) { + // // Do not retry operation. + // } else { + // // Sleep for duration and retry operation. + // } + // + NextBackOff() time.Duration + + // Reset to initial state. + Reset() +} + +// Stop indicates that no more retries should be made for use in NextBackOff(). +const Stop time.Duration = -1 + +// ZeroBackOff is a fixed backoff policy whose backoff time is always zero, +// meaning that the operation is retried immediately without waiting, indefinitely. +type ZeroBackOff struct{} + +func (b *ZeroBackOff) Reset() {} + +func (b *ZeroBackOff) NextBackOff() time.Duration { return 0 } + +// StopBackOff is a fixed backoff policy that always returns backoff.Stop for +// NextBackOff(), meaning that the operation should never be retried. +type StopBackOff struct{} + +func (b *StopBackOff) Reset() {} + +func (b *StopBackOff) NextBackOff() time.Duration { return Stop } + +// ConstantBackOff is a backoff policy that always returns the same backoff delay. +// This is in contrast to an exponential backoff policy, +// which returns a delay that grows longer as you call NextBackOff() over and over again. +type ConstantBackOff struct { + Interval time.Duration +} + +func (b *ConstantBackOff) Reset() {} +func (b *ConstantBackOff) NextBackOff() time.Duration { return b.Interval } + +func NewConstantBackOff(d time.Duration) *ConstantBackOff { + return &ConstantBackOff{Interval: d} +} diff --git a/test/vendor/github.com/cenkalti/backoff/v4/context.go b/test/vendor/github.com/cenkalti/backoff/v4/context.go new file mode 100644 index 0000000000..48482330eb --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/context.go @@ -0,0 +1,62 @@ +package backoff + +import ( + "context" + "time" +) + +// BackOffContext is a backoff policy that stops retrying after the context +// is canceled. +type BackOffContext interface { // nolint: golint + BackOff + Context() context.Context +} + +type backOffContext struct { + BackOff + ctx context.Context +} + +// WithContext returns a BackOffContext with context ctx +// +// ctx must not be nil +func WithContext(b BackOff, ctx context.Context) BackOffContext { // nolint: golint + if ctx == nil { + panic("nil context") + } + + if b, ok := b.(*backOffContext); ok { + return &backOffContext{ + BackOff: b.BackOff, + ctx: ctx, + } + } + + return &backOffContext{ + BackOff: b, + ctx: ctx, + } +} + +func getContext(b BackOff) context.Context { + if cb, ok := b.(BackOffContext); ok { + return cb.Context() + } + if tb, ok := b.(*backOffTries); ok { + return getContext(tb.delegate) + } + return context.Background() +} + +func (b *backOffContext) Context() context.Context { + return b.ctx +} + +func (b *backOffContext) NextBackOff() time.Duration { + select { + case <-b.ctx.Done(): + return Stop + default: + return b.BackOff.NextBackOff() + } +} diff --git a/test/vendor/github.com/cenkalti/backoff/v4/exponential.go b/test/vendor/github.com/cenkalti/backoff/v4/exponential.go new file mode 100644 index 0000000000..3d3453215b --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/exponential.go @@ -0,0 +1,158 @@ +package backoff + +import ( + "math/rand" + "time" +) + +/* +ExponentialBackOff is a backoff implementation that increases the backoff +period for each retry attempt using a randomization function that grows exponentially. + +NextBackOff() is calculated using the following formula: + + randomized interval = + RetryInterval * (random value in range [1 - RandomizationFactor, 1 + RandomizationFactor]) + +In other words NextBackOff() will range between the randomization factor +percentage below and above the retry interval. + +For example, given the following parameters: + + RetryInterval = 2 + RandomizationFactor = 0.5 + Multiplier = 2 + +the actual backoff period used in the next retry attempt will range between 1 and 3 seconds, +multiplied by the exponential, that is, between 2 and 6 seconds. + +Note: MaxInterval caps the RetryInterval and not the randomized interval. + +If the time elapsed since an ExponentialBackOff instance is created goes past the +MaxElapsedTime, then the method NextBackOff() starts returning backoff.Stop. + +The elapsed time can be reset by calling Reset(). + +Example: Given the following default arguments, for 10 tries the sequence will be, +and assuming we go over the MaxElapsedTime on the 10th try: + + Request # RetryInterval (seconds) Randomized Interval (seconds) + + 1 0.5 [0.25, 0.75] + 2 0.75 [0.375, 1.125] + 3 1.125 [0.562, 1.687] + 4 1.687 [0.8435, 2.53] + 5 2.53 [1.265, 3.795] + 6 3.795 [1.897, 5.692] + 7 5.692 [2.846, 8.538] + 8 8.538 [4.269, 12.807] + 9 12.807 [6.403, 19.210] + 10 19.210 backoff.Stop + +Note: Implementation is not thread-safe. +*/ +type ExponentialBackOff struct { + InitialInterval time.Duration + RandomizationFactor float64 + Multiplier float64 + MaxInterval time.Duration + // After MaxElapsedTime the ExponentialBackOff returns Stop. + // It never stops if MaxElapsedTime == 0. + MaxElapsedTime time.Duration + Stop time.Duration + Clock Clock + + currentInterval time.Duration + startTime time.Time +} + +// Clock is an interface that returns current time for BackOff. +type Clock interface { + Now() time.Time +} + +// Default values for ExponentialBackOff. +const ( + DefaultInitialInterval = 500 * time.Millisecond + DefaultRandomizationFactor = 0.5 + DefaultMultiplier = 1.5 + DefaultMaxInterval = 60 * time.Second + DefaultMaxElapsedTime = 15 * time.Minute +) + +// NewExponentialBackOff creates an instance of ExponentialBackOff using default values. +func NewExponentialBackOff() *ExponentialBackOff { + b := &ExponentialBackOff{ + InitialInterval: DefaultInitialInterval, + RandomizationFactor: DefaultRandomizationFactor, + Multiplier: DefaultMultiplier, + MaxInterval: DefaultMaxInterval, + MaxElapsedTime: DefaultMaxElapsedTime, + Stop: Stop, + Clock: SystemClock, + } + b.Reset() + return b +} + +type systemClock struct{} + +func (t systemClock) Now() time.Time { + return time.Now() +} + +// SystemClock implements Clock interface that uses time.Now(). +var SystemClock = systemClock{} + +// Reset the interval back to the initial retry interval and restarts the timer. +// Reset must be called before using b. +func (b *ExponentialBackOff) Reset() { + b.currentInterval = b.InitialInterval + b.startTime = b.Clock.Now() +} + +// NextBackOff calculates the next backoff interval using the formula: +// Randomized interval = RetryInterval * (1 ± RandomizationFactor) +func (b *ExponentialBackOff) NextBackOff() time.Duration { + // Make sure we have not gone over the maximum elapsed time. + elapsed := b.GetElapsedTime() + next := getRandomValueFromInterval(b.RandomizationFactor, rand.Float64(), b.currentInterval) + b.incrementCurrentInterval() + if b.MaxElapsedTime != 0 && elapsed+next > b.MaxElapsedTime { + return b.Stop + } + return next +} + +// GetElapsedTime returns the elapsed time since an ExponentialBackOff instance +// is created and is reset when Reset() is called. +// +// The elapsed time is computed using time.Now().UnixNano(). It is +// safe to call even while the backoff policy is used by a running +// ticker. +func (b *ExponentialBackOff) GetElapsedTime() time.Duration { + return b.Clock.Now().Sub(b.startTime) +} + +// Increments the current interval by multiplying it with the multiplier. +func (b *ExponentialBackOff) incrementCurrentInterval() { + // Check for overflow, if overflow is detected set the current interval to the max interval. + if float64(b.currentInterval) >= float64(b.MaxInterval)/b.Multiplier { + b.currentInterval = b.MaxInterval + } else { + b.currentInterval = time.Duration(float64(b.currentInterval) * b.Multiplier) + } +} + +// Returns a random value from the following interval: +// [currentInterval - randomizationFactor * currentInterval, currentInterval + randomizationFactor * currentInterval]. +func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration { + var delta = randomizationFactor * float64(currentInterval) + var minInterval = float64(currentInterval) - delta + var maxInterval = float64(currentInterval) + delta + + // Get a random value from the range [minInterval, maxInterval]. + // The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then + // we want a 33% chance for selecting either 1, 2 or 3. + return time.Duration(minInterval + (random * (maxInterval - minInterval + 1))) +} diff --git a/test/vendor/github.com/cenkalti/backoff/v4/go.mod b/test/vendor/github.com/cenkalti/backoff/v4/go.mod new file mode 100644 index 0000000000..f811bead99 --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/go.mod @@ -0,0 +1,3 @@ +module github.com/cenkalti/backoff/v4 + +go 1.13 diff --git a/test/vendor/github.com/cenkalti/backoff/v4/retry.go b/test/vendor/github.com/cenkalti/backoff/v4/retry.go new file mode 100644 index 0000000000..1ce2507ebc --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/retry.go @@ -0,0 +1,112 @@ +package backoff + +import ( + "errors" + "time" +) + +// An Operation is executing by Retry() or RetryNotify(). +// The operation will be retried using a backoff policy if it returns an error. +type Operation func() error + +// Notify is a notify-on-error function. It receives an operation error and +// backoff delay if the operation failed (with an error). +// +// NOTE that if the backoff policy stated to stop retrying, +// the notify function isn't called. +type Notify func(error, time.Duration) + +// Retry the operation o until it does not return error or BackOff stops. +// o is guaranteed to be run at least once. +// +// If o returns a *PermanentError, the operation is not retried, and the +// wrapped error is returned. +// +// Retry sleeps the goroutine for the duration returned by BackOff after a +// failed operation returns. +func Retry(o Operation, b BackOff) error { + return RetryNotify(o, b, nil) +} + +// RetryNotify calls notify function with the error and wait duration +// for each failed attempt before sleep. +func RetryNotify(operation Operation, b BackOff, notify Notify) error { + return RetryNotifyWithTimer(operation, b, notify, nil) +} + +// RetryNotifyWithTimer calls notify function with the error and wait duration using the given Timer +// for each failed attempt before sleep. +// A default timer that uses system timer is used when nil is passed. +func RetryNotifyWithTimer(operation Operation, b BackOff, notify Notify, t Timer) error { + var err error + var next time.Duration + if t == nil { + t = &defaultTimer{} + } + + defer func() { + t.Stop() + }() + + ctx := getContext(b) + + b.Reset() + for { + if err = operation(); err == nil { + return nil + } + + var permanent *PermanentError + if errors.As(err, &permanent) { + return permanent.Err + } + + if next = b.NextBackOff(); next == Stop { + if cerr := ctx.Err(); cerr != nil { + return cerr + } + + return err + } + + if notify != nil { + notify(err, next) + } + + t.Start(next) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C(): + } + } +} + +// PermanentError signals that the operation should not be retried. +type PermanentError struct { + Err error +} + +func (e *PermanentError) Error() string { + return e.Err.Error() +} + +func (e *PermanentError) Unwrap() error { + return e.Err +} + +func (e *PermanentError) Is(target error) bool { + _, ok := target.(*PermanentError) + return ok +} + +// Permanent wraps the given err in a *PermanentError. +func Permanent(err error) error { + if err == nil { + return nil + } + return &PermanentError{ + Err: err, + } +} diff --git a/test/vendor/github.com/cenkalti/backoff/v4/ticker.go b/test/vendor/github.com/cenkalti/backoff/v4/ticker.go new file mode 100644 index 0000000000..df9d68bce5 --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/ticker.go @@ -0,0 +1,97 @@ +package backoff + +import ( + "context" + "sync" + "time" +) + +// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff. +// +// Ticks will continue to arrive when the previous operation is still running, +// so operations that take a while to fail could run in quick succession. +type Ticker struct { + C <-chan time.Time + c chan time.Time + b BackOff + ctx context.Context + timer Timer + stop chan struct{} + stopOnce sync.Once +} + +// NewTicker returns a new Ticker containing a channel that will send +// the time at times specified by the BackOff argument. Ticker is +// guaranteed to tick at least once. The channel is closed when Stop +// method is called or BackOff stops. It is not safe to manipulate the +// provided backoff policy (notably calling NextBackOff or Reset) +// while the ticker is running. +func NewTicker(b BackOff) *Ticker { + return NewTickerWithTimer(b, &defaultTimer{}) +} + +// NewTickerWithTimer returns a new Ticker with a custom timer. +// A default timer that uses system timer is used when nil is passed. +func NewTickerWithTimer(b BackOff, timer Timer) *Ticker { + if timer == nil { + timer = &defaultTimer{} + } + c := make(chan time.Time) + t := &Ticker{ + C: c, + c: c, + b: b, + ctx: getContext(b), + timer: timer, + stop: make(chan struct{}), + } + t.b.Reset() + go t.run() + return t +} + +// Stop turns off a ticker. After Stop, no more ticks will be sent. +func (t *Ticker) Stop() { + t.stopOnce.Do(func() { close(t.stop) }) +} + +func (t *Ticker) run() { + c := t.c + defer close(c) + + // Ticker is guaranteed to tick at least once. + afterC := t.send(time.Now()) + + for { + if afterC == nil { + return + } + + select { + case tick := <-afterC: + afterC = t.send(tick) + case <-t.stop: + t.c = nil // Prevent future ticks from being sent to the channel. + return + case <-t.ctx.Done(): + return + } + } +} + +func (t *Ticker) send(tick time.Time) <-chan time.Time { + select { + case t.c <- tick: + case <-t.stop: + return nil + } + + next := t.b.NextBackOff() + if next == Stop { + t.Stop() + return nil + } + + t.timer.Start(next) + return t.timer.C() +} diff --git a/test/vendor/github.com/cenkalti/backoff/v4/timer.go b/test/vendor/github.com/cenkalti/backoff/v4/timer.go new file mode 100644 index 0000000000..8120d0213c --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/timer.go @@ -0,0 +1,35 @@ +package backoff + +import "time" + +type Timer interface { + Start(duration time.Duration) + Stop() + C() <-chan time.Time +} + +// defaultTimer implements Timer interface using time.Timer +type defaultTimer struct { + timer *time.Timer +} + +// C returns the timers channel which receives the current time when the timer fires. +func (t *defaultTimer) C() <-chan time.Time { + return t.timer.C +} + +// Start starts the timer to fire after the given duration +func (t *defaultTimer) Start(duration time.Duration) { + if t.timer == nil { + t.timer = time.NewTimer(duration) + } else { + t.timer.Reset(duration) + } +} + +// Stop is called when the timer is not used anymore and resources may be freed. +func (t *defaultTimer) Stop() { + if t.timer != nil { + t.timer.Stop() + } +} diff --git a/test/vendor/github.com/cenkalti/backoff/v4/tries.go b/test/vendor/github.com/cenkalti/backoff/v4/tries.go new file mode 100644 index 0000000000..28d58ca37c --- /dev/null +++ b/test/vendor/github.com/cenkalti/backoff/v4/tries.go @@ -0,0 +1,38 @@ +package backoff + +import "time" + +/* +WithMaxRetries creates a wrapper around another BackOff, which will +return Stop if NextBackOff() has been called too many times since +the last time Reset() was called + +Note: Implementation is not thread-safe. +*/ +func WithMaxRetries(b BackOff, max uint64) BackOff { + return &backOffTries{delegate: b, maxTries: max} +} + +type backOffTries struct { + delegate BackOff + maxTries uint64 + numTries uint64 +} + +func (b *backOffTries) NextBackOff() time.Duration { + if b.maxTries == 0 { + return Stop + } + if b.maxTries > 0 { + if b.maxTries <= b.numTries { + return Stop + } + b.numTries++ + } + return b.delegate.NextBackOff() +} + +func (b *backOffTries) Reset() { + b.numTries = 0 + b.delegate.Reset() +} diff --git a/test/vendor/modules.txt b/test/vendor/modules.txt index d297c9cfd9..f97e92c822 100644 --- a/test/vendor/modules.txt +++ b/test/vendor/modules.txt @@ -72,6 +72,8 @@ github.com/Microsoft/hcsshim/pkg/securitypolicy github.com/bits-and-blooms/bitset # github.com/blang/semver v3.5.1+incompatible github.com/blang/semver +# github.com/cenkalti/backoff/v4 v4.1.1 +github.com/cenkalti/backoff/v4 # github.com/containerd/cgroups v1.0.1 github.com/containerd/cgroups/stats/v1 # github.com/containerd/console v1.0.2 diff --git a/vendor/github.com/cenkalti/backoff/v4/.gitignore b/vendor/github.com/cenkalti/backoff/v4/.gitignore new file mode 100644 index 0000000000..50d95c548b --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +# IDEs +.idea/ diff --git a/vendor/github.com/cenkalti/backoff/v4/.travis.yml b/vendor/github.com/cenkalti/backoff/v4/.travis.yml new file mode 100644 index 0000000000..c79105c2fb --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/.travis.yml @@ -0,0 +1,10 @@ +language: go +go: + - 1.13 + - 1.x + - tip +before_install: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover +script: + - $HOME/gopath/bin/goveralls -service=travis-ci diff --git a/vendor/github.com/cenkalti/backoff/v4/LICENSE b/vendor/github.com/cenkalti/backoff/v4/LICENSE new file mode 100644 index 0000000000..89b8179965 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Cenk Altı + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/cenkalti/backoff/v4/README.md b/vendor/github.com/cenkalti/backoff/v4/README.md new file mode 100644 index 0000000000..16abdfc084 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/README.md @@ -0,0 +1,32 @@ +# Exponential Backoff [![GoDoc][godoc image]][godoc] [![Build Status][travis image]][travis] [![Coverage Status][coveralls image]][coveralls] + +This is a Go port of the exponential backoff algorithm from [Google's HTTP Client Library for Java][google-http-java-client]. + +[Exponential backoff][exponential backoff wiki] +is an algorithm that uses feedback to multiplicatively decrease the rate of some process, +in order to gradually find an acceptable rate. +The retries exponentially increase and stop increasing when a certain threshold is met. + +## Usage + +Import path is `github.com/cenkalti/backoff/v4`. Please note the version part at the end. + +Use https://pkg.go.dev/github.com/cenkalti/backoff/v4 to view the documentation. + +## Contributing + +* I would like to keep this library as small as possible. +* Please don't send a PR without opening an issue and discussing it first. +* If proposed change is not a common use case, I will probably not accept it. + +[godoc]: https://pkg.go.dev/github.com/cenkalti/backoff/v4 +[godoc image]: https://godoc.org/github.com/cenkalti/backoff?status.png +[travis]: https://travis-ci.org/cenkalti/backoff +[travis image]: https://travis-ci.org/cenkalti/backoff.png?branch=master +[coveralls]: https://coveralls.io/github/cenkalti/backoff?branch=master +[coveralls image]: https://coveralls.io/repos/github/cenkalti/backoff/badge.svg?branch=master + +[google-http-java-client]: https://github.com/google/google-http-java-client/blob/da1aa993e90285ec18579f1553339b00e19b3ab5/google-http-client/src/main/java/com/google/api/client/util/ExponentialBackOff.java +[exponential backoff wiki]: http://en.wikipedia.org/wiki/Exponential_backoff + +[advanced example]: https://pkg.go.dev/github.com/cenkalti/backoff/v4?tab=doc#pkg-examples diff --git a/vendor/github.com/cenkalti/backoff/v4/backoff.go b/vendor/github.com/cenkalti/backoff/v4/backoff.go new file mode 100644 index 0000000000..3676ee405d --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/backoff.go @@ -0,0 +1,66 @@ +// Package backoff implements backoff algorithms for retrying operations. +// +// Use Retry function for retrying operations that may fail. +// If Retry does not meet your needs, +// copy/paste the function into your project and modify as you wish. +// +// There is also Ticker type similar to time.Ticker. +// You can use it if you need to work with channels. +// +// See Examples section below for usage examples. +package backoff + +import "time" + +// BackOff is a backoff policy for retrying an operation. +type BackOff interface { + // NextBackOff returns the duration to wait before retrying the operation, + // or backoff. Stop to indicate that no more retries should be made. + // + // Example usage: + // + // duration := backoff.NextBackOff(); + // if (duration == backoff.Stop) { + // // Do not retry operation. + // } else { + // // Sleep for duration and retry operation. + // } + // + NextBackOff() time.Duration + + // Reset to initial state. + Reset() +} + +// Stop indicates that no more retries should be made for use in NextBackOff(). +const Stop time.Duration = -1 + +// ZeroBackOff is a fixed backoff policy whose backoff time is always zero, +// meaning that the operation is retried immediately without waiting, indefinitely. +type ZeroBackOff struct{} + +func (b *ZeroBackOff) Reset() {} + +func (b *ZeroBackOff) NextBackOff() time.Duration { return 0 } + +// StopBackOff is a fixed backoff policy that always returns backoff.Stop for +// NextBackOff(), meaning that the operation should never be retried. +type StopBackOff struct{} + +func (b *StopBackOff) Reset() {} + +func (b *StopBackOff) NextBackOff() time.Duration { return Stop } + +// ConstantBackOff is a backoff policy that always returns the same backoff delay. +// This is in contrast to an exponential backoff policy, +// which returns a delay that grows longer as you call NextBackOff() over and over again. +type ConstantBackOff struct { + Interval time.Duration +} + +func (b *ConstantBackOff) Reset() {} +func (b *ConstantBackOff) NextBackOff() time.Duration { return b.Interval } + +func NewConstantBackOff(d time.Duration) *ConstantBackOff { + return &ConstantBackOff{Interval: d} +} diff --git a/vendor/github.com/cenkalti/backoff/v4/context.go b/vendor/github.com/cenkalti/backoff/v4/context.go new file mode 100644 index 0000000000..48482330eb --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/context.go @@ -0,0 +1,62 @@ +package backoff + +import ( + "context" + "time" +) + +// BackOffContext is a backoff policy that stops retrying after the context +// is canceled. +type BackOffContext interface { // nolint: golint + BackOff + Context() context.Context +} + +type backOffContext struct { + BackOff + ctx context.Context +} + +// WithContext returns a BackOffContext with context ctx +// +// ctx must not be nil +func WithContext(b BackOff, ctx context.Context) BackOffContext { // nolint: golint + if ctx == nil { + panic("nil context") + } + + if b, ok := b.(*backOffContext); ok { + return &backOffContext{ + BackOff: b.BackOff, + ctx: ctx, + } + } + + return &backOffContext{ + BackOff: b, + ctx: ctx, + } +} + +func getContext(b BackOff) context.Context { + if cb, ok := b.(BackOffContext); ok { + return cb.Context() + } + if tb, ok := b.(*backOffTries); ok { + return getContext(tb.delegate) + } + return context.Background() +} + +func (b *backOffContext) Context() context.Context { + return b.ctx +} + +func (b *backOffContext) NextBackOff() time.Duration { + select { + case <-b.ctx.Done(): + return Stop + default: + return b.BackOff.NextBackOff() + } +} diff --git a/vendor/github.com/cenkalti/backoff/v4/exponential.go b/vendor/github.com/cenkalti/backoff/v4/exponential.go new file mode 100644 index 0000000000..3d3453215b --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/exponential.go @@ -0,0 +1,158 @@ +package backoff + +import ( + "math/rand" + "time" +) + +/* +ExponentialBackOff is a backoff implementation that increases the backoff +period for each retry attempt using a randomization function that grows exponentially. + +NextBackOff() is calculated using the following formula: + + randomized interval = + RetryInterval * (random value in range [1 - RandomizationFactor, 1 + RandomizationFactor]) + +In other words NextBackOff() will range between the randomization factor +percentage below and above the retry interval. + +For example, given the following parameters: + + RetryInterval = 2 + RandomizationFactor = 0.5 + Multiplier = 2 + +the actual backoff period used in the next retry attempt will range between 1 and 3 seconds, +multiplied by the exponential, that is, between 2 and 6 seconds. + +Note: MaxInterval caps the RetryInterval and not the randomized interval. + +If the time elapsed since an ExponentialBackOff instance is created goes past the +MaxElapsedTime, then the method NextBackOff() starts returning backoff.Stop. + +The elapsed time can be reset by calling Reset(). + +Example: Given the following default arguments, for 10 tries the sequence will be, +and assuming we go over the MaxElapsedTime on the 10th try: + + Request # RetryInterval (seconds) Randomized Interval (seconds) + + 1 0.5 [0.25, 0.75] + 2 0.75 [0.375, 1.125] + 3 1.125 [0.562, 1.687] + 4 1.687 [0.8435, 2.53] + 5 2.53 [1.265, 3.795] + 6 3.795 [1.897, 5.692] + 7 5.692 [2.846, 8.538] + 8 8.538 [4.269, 12.807] + 9 12.807 [6.403, 19.210] + 10 19.210 backoff.Stop + +Note: Implementation is not thread-safe. +*/ +type ExponentialBackOff struct { + InitialInterval time.Duration + RandomizationFactor float64 + Multiplier float64 + MaxInterval time.Duration + // After MaxElapsedTime the ExponentialBackOff returns Stop. + // It never stops if MaxElapsedTime == 0. + MaxElapsedTime time.Duration + Stop time.Duration + Clock Clock + + currentInterval time.Duration + startTime time.Time +} + +// Clock is an interface that returns current time for BackOff. +type Clock interface { + Now() time.Time +} + +// Default values for ExponentialBackOff. +const ( + DefaultInitialInterval = 500 * time.Millisecond + DefaultRandomizationFactor = 0.5 + DefaultMultiplier = 1.5 + DefaultMaxInterval = 60 * time.Second + DefaultMaxElapsedTime = 15 * time.Minute +) + +// NewExponentialBackOff creates an instance of ExponentialBackOff using default values. +func NewExponentialBackOff() *ExponentialBackOff { + b := &ExponentialBackOff{ + InitialInterval: DefaultInitialInterval, + RandomizationFactor: DefaultRandomizationFactor, + Multiplier: DefaultMultiplier, + MaxInterval: DefaultMaxInterval, + MaxElapsedTime: DefaultMaxElapsedTime, + Stop: Stop, + Clock: SystemClock, + } + b.Reset() + return b +} + +type systemClock struct{} + +func (t systemClock) Now() time.Time { + return time.Now() +} + +// SystemClock implements Clock interface that uses time.Now(). +var SystemClock = systemClock{} + +// Reset the interval back to the initial retry interval and restarts the timer. +// Reset must be called before using b. +func (b *ExponentialBackOff) Reset() { + b.currentInterval = b.InitialInterval + b.startTime = b.Clock.Now() +} + +// NextBackOff calculates the next backoff interval using the formula: +// Randomized interval = RetryInterval * (1 ± RandomizationFactor) +func (b *ExponentialBackOff) NextBackOff() time.Duration { + // Make sure we have not gone over the maximum elapsed time. + elapsed := b.GetElapsedTime() + next := getRandomValueFromInterval(b.RandomizationFactor, rand.Float64(), b.currentInterval) + b.incrementCurrentInterval() + if b.MaxElapsedTime != 0 && elapsed+next > b.MaxElapsedTime { + return b.Stop + } + return next +} + +// GetElapsedTime returns the elapsed time since an ExponentialBackOff instance +// is created and is reset when Reset() is called. +// +// The elapsed time is computed using time.Now().UnixNano(). It is +// safe to call even while the backoff policy is used by a running +// ticker. +func (b *ExponentialBackOff) GetElapsedTime() time.Duration { + return b.Clock.Now().Sub(b.startTime) +} + +// Increments the current interval by multiplying it with the multiplier. +func (b *ExponentialBackOff) incrementCurrentInterval() { + // Check for overflow, if overflow is detected set the current interval to the max interval. + if float64(b.currentInterval) >= float64(b.MaxInterval)/b.Multiplier { + b.currentInterval = b.MaxInterval + } else { + b.currentInterval = time.Duration(float64(b.currentInterval) * b.Multiplier) + } +} + +// Returns a random value from the following interval: +// [currentInterval - randomizationFactor * currentInterval, currentInterval + randomizationFactor * currentInterval]. +func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration { + var delta = randomizationFactor * float64(currentInterval) + var minInterval = float64(currentInterval) - delta + var maxInterval = float64(currentInterval) + delta + + // Get a random value from the range [minInterval, maxInterval]. + // The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then + // we want a 33% chance for selecting either 1, 2 or 3. + return time.Duration(minInterval + (random * (maxInterval - minInterval + 1))) +} diff --git a/vendor/github.com/cenkalti/backoff/v4/go.mod b/vendor/github.com/cenkalti/backoff/v4/go.mod new file mode 100644 index 0000000000..f811bead99 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/go.mod @@ -0,0 +1,3 @@ +module github.com/cenkalti/backoff/v4 + +go 1.13 diff --git a/vendor/github.com/cenkalti/backoff/v4/retry.go b/vendor/github.com/cenkalti/backoff/v4/retry.go new file mode 100644 index 0000000000..1ce2507ebc --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/retry.go @@ -0,0 +1,112 @@ +package backoff + +import ( + "errors" + "time" +) + +// An Operation is executing by Retry() or RetryNotify(). +// The operation will be retried using a backoff policy if it returns an error. +type Operation func() error + +// Notify is a notify-on-error function. It receives an operation error and +// backoff delay if the operation failed (with an error). +// +// NOTE that if the backoff policy stated to stop retrying, +// the notify function isn't called. +type Notify func(error, time.Duration) + +// Retry the operation o until it does not return error or BackOff stops. +// o is guaranteed to be run at least once. +// +// If o returns a *PermanentError, the operation is not retried, and the +// wrapped error is returned. +// +// Retry sleeps the goroutine for the duration returned by BackOff after a +// failed operation returns. +func Retry(o Operation, b BackOff) error { + return RetryNotify(o, b, nil) +} + +// RetryNotify calls notify function with the error and wait duration +// for each failed attempt before sleep. +func RetryNotify(operation Operation, b BackOff, notify Notify) error { + return RetryNotifyWithTimer(operation, b, notify, nil) +} + +// RetryNotifyWithTimer calls notify function with the error and wait duration using the given Timer +// for each failed attempt before sleep. +// A default timer that uses system timer is used when nil is passed. +func RetryNotifyWithTimer(operation Operation, b BackOff, notify Notify, t Timer) error { + var err error + var next time.Duration + if t == nil { + t = &defaultTimer{} + } + + defer func() { + t.Stop() + }() + + ctx := getContext(b) + + b.Reset() + for { + if err = operation(); err == nil { + return nil + } + + var permanent *PermanentError + if errors.As(err, &permanent) { + return permanent.Err + } + + if next = b.NextBackOff(); next == Stop { + if cerr := ctx.Err(); cerr != nil { + return cerr + } + + return err + } + + if notify != nil { + notify(err, next) + } + + t.Start(next) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C(): + } + } +} + +// PermanentError signals that the operation should not be retried. +type PermanentError struct { + Err error +} + +func (e *PermanentError) Error() string { + return e.Err.Error() +} + +func (e *PermanentError) Unwrap() error { + return e.Err +} + +func (e *PermanentError) Is(target error) bool { + _, ok := target.(*PermanentError) + return ok +} + +// Permanent wraps the given err in a *PermanentError. +func Permanent(err error) error { + if err == nil { + return nil + } + return &PermanentError{ + Err: err, + } +} diff --git a/vendor/github.com/cenkalti/backoff/v4/ticker.go b/vendor/github.com/cenkalti/backoff/v4/ticker.go new file mode 100644 index 0000000000..df9d68bce5 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/ticker.go @@ -0,0 +1,97 @@ +package backoff + +import ( + "context" + "sync" + "time" +) + +// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff. +// +// Ticks will continue to arrive when the previous operation is still running, +// so operations that take a while to fail could run in quick succession. +type Ticker struct { + C <-chan time.Time + c chan time.Time + b BackOff + ctx context.Context + timer Timer + stop chan struct{} + stopOnce sync.Once +} + +// NewTicker returns a new Ticker containing a channel that will send +// the time at times specified by the BackOff argument. Ticker is +// guaranteed to tick at least once. The channel is closed when Stop +// method is called or BackOff stops. It is not safe to manipulate the +// provided backoff policy (notably calling NextBackOff or Reset) +// while the ticker is running. +func NewTicker(b BackOff) *Ticker { + return NewTickerWithTimer(b, &defaultTimer{}) +} + +// NewTickerWithTimer returns a new Ticker with a custom timer. +// A default timer that uses system timer is used when nil is passed. +func NewTickerWithTimer(b BackOff, timer Timer) *Ticker { + if timer == nil { + timer = &defaultTimer{} + } + c := make(chan time.Time) + t := &Ticker{ + C: c, + c: c, + b: b, + ctx: getContext(b), + timer: timer, + stop: make(chan struct{}), + } + t.b.Reset() + go t.run() + return t +} + +// Stop turns off a ticker. After Stop, no more ticks will be sent. +func (t *Ticker) Stop() { + t.stopOnce.Do(func() { close(t.stop) }) +} + +func (t *Ticker) run() { + c := t.c + defer close(c) + + // Ticker is guaranteed to tick at least once. + afterC := t.send(time.Now()) + + for { + if afterC == nil { + return + } + + select { + case tick := <-afterC: + afterC = t.send(tick) + case <-t.stop: + t.c = nil // Prevent future ticks from being sent to the channel. + return + case <-t.ctx.Done(): + return + } + } +} + +func (t *Ticker) send(tick time.Time) <-chan time.Time { + select { + case t.c <- tick: + case <-t.stop: + return nil + } + + next := t.b.NextBackOff() + if next == Stop { + t.Stop() + return nil + } + + t.timer.Start(next) + return t.timer.C() +} diff --git a/vendor/github.com/cenkalti/backoff/v4/timer.go b/vendor/github.com/cenkalti/backoff/v4/timer.go new file mode 100644 index 0000000000..8120d0213c --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/timer.go @@ -0,0 +1,35 @@ +package backoff + +import "time" + +type Timer interface { + Start(duration time.Duration) + Stop() + C() <-chan time.Time +} + +// defaultTimer implements Timer interface using time.Timer +type defaultTimer struct { + timer *time.Timer +} + +// C returns the timers channel which receives the current time when the timer fires. +func (t *defaultTimer) C() <-chan time.Time { + return t.timer.C +} + +// Start starts the timer to fire after the given duration +func (t *defaultTimer) Start(duration time.Duration) { + if t.timer == nil { + t.timer = time.NewTimer(duration) + } else { + t.timer.Reset(duration) + } +} + +// Stop is called when the timer is not used anymore and resources may be freed. +func (t *defaultTimer) Stop() { + if t.timer != nil { + t.timer.Stop() + } +} diff --git a/vendor/github.com/cenkalti/backoff/v4/tries.go b/vendor/github.com/cenkalti/backoff/v4/tries.go new file mode 100644 index 0000000000..28d58ca37c --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/v4/tries.go @@ -0,0 +1,38 @@ +package backoff + +import "time" + +/* +WithMaxRetries creates a wrapper around another BackOff, which will +return Stop if NextBackOff() has been called too many times since +the last time Reset() was called + +Note: Implementation is not thread-safe. +*/ +func WithMaxRetries(b BackOff, max uint64) BackOff { + return &backOffTries{delegate: b, maxTries: max} +} + +type backOffTries struct { + delegate BackOff + maxTries uint64 + numTries uint64 +} + +func (b *backOffTries) NextBackOff() time.Duration { + if b.maxTries == 0 { + return Stop + } + if b.maxTries > 0 { + if b.maxTries <= b.numTries { + return Stop + } + b.numTries++ + } + return b.delegate.NextBackOff() +} + +func (b *backOffTries) Reset() { + b.numTries = 0 + b.delegate.Reset() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 54a5476ece..426beb5e50 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -9,6 +9,8 @@ github.com/Microsoft/go-winio/pkg/guid github.com/Microsoft/go-winio/pkg/process github.com/Microsoft/go-winio/pkg/security github.com/Microsoft/go-winio/vhd +# github.com/cenkalti/backoff/v4 v4.1.1 +github.com/cenkalti/backoff/v4 # github.com/containerd/cgroups v1.0.1 github.com/containerd/cgroups github.com/containerd/cgroups/stats/v1