diff --git a/frontend/dockerfile/docs/syntax.md b/frontend/dockerfile/docs/syntax.md index e86cad3129d0..9af7c823aa6c 100644 --- a/frontend/dockerfile/docs/syntax.md +++ b/frontend/dockerfile/docs/syntax.md @@ -12,7 +12,7 @@ If you are using Docker v18.09 or later, BuildKit mode can be enabled by setting BuildKit supports loading frontends dynamically from container images. Images for Dockerfile frontends are available at [`docker/dockerfile`](https://hub.docker.com/r/docker/dockerfile/tags/) repository. -To use the external frontend, the first line of your Dockerfile needs to be `# syntax=docker/dockerfile:v1.2` pointing to the +To use the external frontend, the first line of your Dockerfile needs to be `# syntax=docker/dockerfile:1.2` pointing to the specific image you want to use. BuildKit also ships with Dockerfile frontend builtin but it is recommended to use an external image to make sure that all @@ -73,7 +73,7 @@ it if more storage space is needed. #### Example: cache Go packages ```dockerfile -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.2 FROM golang ... RUN --mount=type=cache,target=/root/.cache/go-build go build ... @@ -82,7 +82,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build go build ... #### Example: cache apt packages ```dockerfile -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.2 FROM ubuntu RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ @@ -115,7 +115,7 @@ This mount type allows the build container to access secure files such as privat #### Example: access to S3 ```dockerfile -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.2 FROM python:3 RUN pip install awscli RUN --mount=type=secret,id=aws,target=/root/.aws/credentials aws s3 cp s3://... ... @@ -147,7 +147,7 @@ This mount type allows the build container to access SSH keys via SSH agents, wi #### Example: access to Gitlab ```dockerfile -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.2 FROM alpine RUN apk add --no-cache openssh-client RUN mkdir -p -m 0700 ~/.ssh && ssh-keyscan gitlab.com >> ~/.ssh/known_hosts @@ -193,7 +193,7 @@ Default sandbox mode can be activated via `--security=sandbox`, but that is no-o #### Example: check entitlements ```dockerfile -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.2-labs FROM ubuntu RUN --security=insecure cat /proc/self/status | grep CapEff ``` @@ -229,7 +229,7 @@ which needs to be enabled when starting the buildkitd daemon #### Example: isolating external effects ```dockerfile -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.2-labs FROM python:3.6 ADD mypackage.tgz wheels/ RUN --network=none pip install --find-links wheels mypackage diff --git a/frontend/gateway/container.go b/frontend/gateway/container.go index 1f79bb68f0b1..e124566da94a 100644 --- a/frontend/gateway/container.go +++ b/frontend/gateway/container.go @@ -127,6 +127,7 @@ type MountRef struct { type MountMutableRef struct { Ref cache.MutableRef MountIndex int + NoCommit bool } type MakeMutable func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error) @@ -196,6 +197,7 @@ func PrepareMounts(ctx context.Context, mm *mounts.MountManager, cm cache.Manage p.Actives = append(p.Actives, MountMutableRef{ MountIndex: i, Ref: active, + NoCommit: true, }) if m.Output != opspb.SkipOutput && ref != nil { p.OutputRefs = append(p.OutputRefs, MountRef{ diff --git a/solver/llbsolver/errdefs/exec.go b/solver/llbsolver/errdefs/exec.go index f60757104fff..ed3c0d4b0fae 100644 --- a/solver/llbsolver/errdefs/exec.go +++ b/solver/llbsolver/errdefs/exec.go @@ -21,10 +21,15 @@ func (e *ExecError) Unwrap() error { } func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) { + m := map[solver.Result]struct{}{} for _, res := range e.Inputs { if res == nil { continue } + if _, ok := m[res]; ok { + continue + } + m[res] = struct{}{} if err1 := fn(res); err1 != nil && err == nil { err = err1 } @@ -33,6 +38,10 @@ func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) { if res == nil { continue } + if _, ok := m[res]; ok { + continue + } + m[res] = struct{}{} if err1 := fn(res); err1 != nil && err == nil { err = err1 } diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 5ec45fafd5b0..24aa46aa9fb1 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -235,7 +235,7 @@ func (e *execOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu if m.Input == -1 { continue } - execInputs[i] = inputs[m.Input] + execInputs[i] = inputs[m.Input].Clone() } execMounts := make([]solver.Result, len(e.op.Mounts)) copy(execMounts, execInputs) @@ -243,12 +243,16 @@ func (e *execOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu execMounts[p.OutputRefs[i].MountIndex] = res } for _, active := range p.Actives { - ref, cerr := active.Ref.Commit(ctx) - if cerr != nil { - err = errors.Wrapf(err, "error committing %s: %s", active.Ref.ID(), cerr) - continue + if active.NoCommit { + active.Ref.Release(context.TODO()) + } else { + ref, cerr := active.Ref.Commit(ctx) + if cerr != nil { + err = errors.Wrapf(err, "error committing %s: %s", active.Ref.ID(), cerr) + continue + } + execMounts[active.MountIndex] = worker.NewWorkerRefResult(ref, e.w) } - execMounts[active.MountIndex] = worker.NewWorkerRefResult(ref, e.w) } err = errdefs.WithExecError(err, execInputs, execMounts) } else { diff --git a/solver/llbsolver/ops/file.go b/solver/llbsolver/ops/file.go index 548bae1928e6..554532c1845f 100644 --- a/solver/llbsolver/ops/file.go +++ b/solver/llbsolver/ops/file.go @@ -430,7 +430,6 @@ func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptyp if cerr == nil { outputRes[idx-len(inputs)] = worker.NewWorkerRefResult(ref.(cache.ImmutableRef), s.w) } - inpMount.Release(context.TODO()) } // If the action has a secondary input, commit it and set the ref on diff --git a/solver/result.go b/solver/result.go index 75b378c4fdbe..b35c74b49e8e 100644 --- a/solver/result.go +++ b/solver/result.go @@ -47,7 +47,7 @@ type splitResult struct { func (r *splitResult) Release(ctx context.Context) error { if atomic.AddInt64(&r.released, 1) > 1 { - err := errors.Errorf("releasing already released reference") + err := errors.Errorf("releasing already released reference %+v", r.Result.ID()) logrus.Error(err) return err } @@ -78,10 +78,14 @@ func NewSharedCachedResult(res CachedResult) *SharedCachedResult { } } -func (r *SharedCachedResult) Clone() CachedResult { +func (r *SharedCachedResult) CloneCachedResult() CachedResult { return &clonedCachedResult{Result: r.SharedResult.Clone(), cr: r.CachedResult} } +func (r *SharedCachedResult) Clone() Result { + return r.CloneCachedResult() +} + func (r *SharedCachedResult) Release(ctx context.Context) error { return r.SharedResult.Release(ctx) } diff --git a/solver/scheduler.go b/solver/scheduler.go index 5b598bd6b996..bcad4dcdcc60 100644 --- a/solver/scheduler.go +++ b/solver/scheduler.go @@ -244,7 +244,7 @@ func (s *scheduler) build(ctx context.Context, edge Edge) (CachedResult, error) if err := p.Receiver.Status().Err; err != nil { return nil, err } - return p.Receiver.Status().Value.(*edgeState).result.Clone(), nil + return p.Receiver.Status().Value.(*edgeState).result.CloneCachedResult(), nil } // newPipe creates a new request pipe between two edges diff --git a/solver/scheduler_test.go b/solver/scheduler_test.go index 96a6dc391b3d..4c58f3e1ddbc 100644 --- a/solver/scheduler_test.go +++ b/solver/scheduler_test.go @@ -3640,6 +3640,7 @@ type dummyResult struct { func (r *dummyResult) ID() string { return r.id } func (r *dummyResult) Release(context.Context) error { return nil } func (r *dummyResult) Sys() interface{} { return r } +func (r *dummyResult) Clone() Result { return r } func testOpResolver(v Vertex, b Builder) (Op, error) { if op, ok := v.Sys().(Op); ok { diff --git a/solver/types.go b/solver/types.go index 76e4c91163b3..f9de6a93b260 100644 --- a/solver/types.go +++ b/solver/types.go @@ -61,6 +61,7 @@ type Result interface { ID() string Release(context.Context) error Sys() interface{} + Clone() Result } // CachedResult is a result connected with its cache key diff --git a/worker/result.go b/worker/result.go index e178ef3fffe1..9f4cdc5bfa80 100644 --- a/worker/result.go +++ b/worker/result.go @@ -52,3 +52,11 @@ func (r *workerRefResult) Release(ctx context.Context) error { func (r *workerRefResult) Sys() interface{} { return r.WorkerRef } + +func (r *workerRefResult) Clone() solver.Result { + r2 := *r + if r.ImmutableRef != nil { + r.ImmutableRef = r.ImmutableRef.Clone() + } + return &r2 +}