From 2eb359e799c42948a3bb4c3f31e31589ef94d2bd Mon Sep 17 00:00:00 2001 From: STRRL Date: Sun, 31 Jul 2022 11:13:10 +0800 Subject: [PATCH 1/4] feat: system prune also cleanup build cache Signed-off-by: Zhou Zhiqiang --- cmd/nerdctl/system_prune.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/nerdctl/system_prune.go b/cmd/nerdctl/system_prune.go index 0eab374a6ae..6832a3c7c51 100644 --- a/cmd/nerdctl/system_prune.go +++ b/cmd/nerdctl/system_prune.go @@ -72,6 +72,7 @@ func systemPruneAction(cmd *cobra.Command, args []string) error { } msg += ` - all images without at least one container associated to them + - all build cache ` msg += "\nAre you sure you want to continue? [y/N] " fmt.Fprintf(cmd.OutOrStdout(), "WARNING! %s", msg) @@ -99,5 +100,8 @@ func systemPruneAction(cmd *cobra.Command, args []string) error { return err } } - return imagePrune(cmd, client, ctx) + if err := imagePrune(cmd, client, ctx); err != nil { + return err + } + return builderPruneAction(cmd, args) } From ec602a602b6c9057e4e013858d0ae1b27887de9b Mon Sep 17 00:00:00 2001 From: STRRL Date: Wed, 3 Aug 2022 18:16:12 +0800 Subject: [PATCH 2/4] feat: new util for parsing buildctl prune output Signed-off-by: Zhou Zhiqiang --- .../buildctl_prune_output_parser.go | 65 +++++++++++ .../buildctl_prune_output_parser_test.go | 109 ++++++++++++++++++ pkg/tabutil/tabutil.go | 12 +- pkg/tabutil/tabutil_test.go | 34 ++++++ 4 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 pkg/buildkitutil/buildctl_prune_output_parser.go create mode 100644 pkg/buildkitutil/buildctl_prune_output_parser_test.go diff --git a/pkg/buildkitutil/buildctl_prune_output_parser.go b/pkg/buildkitutil/buildctl_prune_output_parser.go new file mode 100644 index 00000000000..41914a807d0 --- /dev/null +++ b/pkg/buildkitutil/buildctl_prune_output_parser.go @@ -0,0 +1,65 @@ +package buildkitutil + +import ( + "bufio" + "bytes" + "fmt" + "github.com/containerd/nerdctl/pkg/tabutil" + "strings" +) + +type BuildctlPruneOutput struct { + TotalSize string + Rows []BuildctlPruneOutputRow +} + +type BuildctlPruneOutputRow struct { + ID string + Reclaimable string + Size string + LastAccessed string +} + +const HeaderID = "ID" +const HeaderReclaimable = "RECLAIMABLE" +const HeaderSize = "SIZE" +const HeaderLastAccessed = "LAST ACCESSED" +const FinalizerTotal = "Total:" + +func ParseBuildctlPruneOutput(out []byte) (*BuildctlPruneOutput, error) { + tabReader := tabutil.NewReader(fmt.Sprintf("%s\t%s\t%s\t%s", HeaderID, HeaderReclaimable, HeaderSize, HeaderLastAccessed)) + scanner := bufio.NewScanner(bytes.NewReader(out)) + firstLineParsed := false + + var rows []BuildctlPruneOutputRow + totalSize := "" + for scanner.Scan() { + line := scanner.Text() + // parse total + if strings.HasPrefix(line, FinalizerTotal) { + totalSize = strings.TrimSpace(line[len(FinalizerTotal):]) + break + } + // parse header + if !firstLineParsed { + _ = tabReader.ParseHeader(line) + firstLineParsed = true + continue + } + // best effort parse row + id, _ := tabReader.ReadRow(line, HeaderID) + reclaimable, _ := tabReader.ReadRow(line, HeaderReclaimable) + size, _ := tabReader.ReadRow(line, HeaderSize) + lastAccessed, _ := tabReader.ReadRow(line, HeaderLastAccessed) + rows = append(rows, BuildctlPruneOutputRow{ + ID: id, + Reclaimable: reclaimable, + Size: size, + LastAccessed: lastAccessed, + }) + } + return &BuildctlPruneOutput{ + TotalSize: totalSize, + Rows: rows, + }, nil +} diff --git a/pkg/buildkitutil/buildctl_prune_output_parser_test.go b/pkg/buildkitutil/buildctl_prune_output_parser_test.go new file mode 100644 index 00000000000..f22298e9a0c --- /dev/null +++ b/pkg/buildkitutil/buildctl_prune_output_parser_test.go @@ -0,0 +1,109 @@ +package buildkitutil + +import ( + "reflect" + "testing" +) + +func TestParseBuildctlPruneOutput(t *testing.T) { + type args struct { + out []byte + } + tests := []struct { + name string + args args + want *BuildctlPruneOutput + wantErr bool + }{ + { + name: "builtctl prune frees several spaces", + args: args{out: []byte(`ID RECLAIMABLE SIZE LAST ACCESSED +spr33ail5pfddqvf25mnxg013 true 0B +akklguozvrppljhuw3v9t45se* true 4.10kB +skr3tplsx990ttmvyymww9pf2* true 0B +hevhw67hx6yv6zlkob00lkaxm true 6.84MB +lqq6c21uz1e697mgvah93govi true 8.19kB +3qi6zgbbup6h6nuc2axyx39ed true 140.81MB +ise2an3ziszpxqfbm0iwiow6p true 327.58MB +0st72zjm0ei4r60hnbmr0thye true 109.07MB +Total: 584.31MB`)}, + want: &BuildctlPruneOutput{ + TotalSize: "584.31MB", + Rows: []BuildctlPruneOutputRow{ + { + ID: "spr33ail5pfddqvf25mnxg013", + Reclaimable: "true", + Size: "0B", + LastAccessed: "", + }, + { + ID: "akklguozvrppljhuw3v9t45se*", + Reclaimable: "true", + Size: "4.10kB", + LastAccessed: "", + }, + { + ID: "skr3tplsx990ttmvyymww9pf2*", + Reclaimable: "true", + Size: "0B", + LastAccessed: "", + }, + { + ID: "hevhw67hx6yv6zlkob00lkaxm", + Reclaimable: "true", + Size: "6.84MB", + LastAccessed: "", + }, + { + ID: "lqq6c21uz1e697mgvah93govi", + Reclaimable: "true", + Size: "8.19kB", + LastAccessed: "", + }, + { + ID: "3qi6zgbbup6h6nuc2axyx39ed", + Reclaimable: "true", + Size: "140.81MB", + LastAccessed: "", + }, + { + ID: "ise2an3ziszpxqfbm0iwiow6p", + Reclaimable: "true", + Size: "327.58MB", + LastAccessed: "", + }, + { + ID: "0st72zjm0ei4r60hnbmr0thye", + Reclaimable: "true", + Size: "109.07MB", + LastAccessed: "", + }, + }, + }, + wantErr: false, + }, + { + name: "buildctl prune frees no spaces", + args: args{ + out: []byte(`Total: 0B`), + }, + want: &BuildctlPruneOutput{ + TotalSize: "0B", + Rows: nil, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseBuildctlPruneOutput(tt.args.out) + if (err != nil) != tt.wantErr { + t.Errorf("ParseBuildctlPruneOutput() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseBuildctlPruneOutput() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/tabutil/tabutil.go b/pkg/tabutil/tabutil.go index b1986ed4c77..a036aa13cc2 100644 --- a/pkg/tabutil/tabutil.go +++ b/pkg/tabutil/tabutil.go @@ -61,11 +61,21 @@ func (r *TabReader) ReadRow(row, key string) (string, bool) { if !ok { return "", false } + if idx.start > len(row) { + return "", false + } var value string if idx.end == -1 { value = row[idx.start:] } else { - value = row[idx.start:idx.end] + end := min(idx.end, len(row)) + value = row[idx.start:end] } return strings.TrimSpace(value), true } +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/pkg/tabutil/tabutil_test.go b/pkg/tabutil/tabutil_test.go index e490434f347..1705f46ab65 100644 --- a/pkg/tabutil/tabutil_test.go +++ b/pkg/tabutil/tabutil_test.go @@ -44,3 +44,37 @@ func TestTabReader(t *testing.T) { value, _ = reader.ReadRow(tabRows[2], "b") assert.Equal(t, value, "456") } + +func TestTabReaderParseTrimmedRow(t *testing.T) { + // for example, row 2 and row 3 are trimmed because they do not contain value for the header c + tabRows := strings.Split(`a b c +1 2 +123 456`, "\n") + reader := NewReader("a\tb\tc\t") + + err := reader.ParseHeader(tabRows[0]) + assert.NilError(t, err) + + var ( + value string + ok bool + ) + value, ok = reader.ReadRow(tabRows[1], "a") + assert.Equal(t, value, "1") + assert.Equal(t, ok, true) + value, ok = reader.ReadRow(tabRows[1], "b") + assert.Equal(t, value, "2") + assert.Equal(t, ok, true) + value, ok = reader.ReadRow(tabRows[1], "c") + assert.Equal(t, value, "") + assert.Equal(t, ok, false) + value, ok = reader.ReadRow(tabRows[2], "a") + assert.Equal(t, value, "123") + assert.Equal(t, ok, true) + value, ok = reader.ReadRow(tabRows[2], "b") + assert.Equal(t, value, "456") + assert.Equal(t, ok, true) + value, ok = reader.ReadRow(tabRows[2], "c") + assert.Equal(t, value, "") + assert.Equal(t, ok, false) +} From 8972e73d74cfa1b3575849c84d1c268339b60c6b Mon Sep 17 00:00:00 2001 From: Zhou Zhiqiang Date: Wed, 3 Aug 2022 19:46:57 +0800 Subject: [PATCH 3/4] chore: pretty print build cache prune Signed-off-by: Zhou Zhiqiang --- cmd/nerdctl/system_prune.go | 22 ++++++++++++++++++- .../buildctl_prune_output_parser.go | 2 +- .../buildctl_prune_output_parser_test.go | 6 ++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/cmd/nerdctl/system_prune.go b/cmd/nerdctl/system_prune.go index 6832a3c7c51..9441983ac2e 100644 --- a/cmd/nerdctl/system_prune.go +++ b/cmd/nerdctl/system_prune.go @@ -17,7 +17,10 @@ package main import ( + "bufio" + "bytes" "fmt" + "github.com/containerd/nerdctl/pkg/buildkitutil" "strings" "github.com/sirupsen/logrus" @@ -103,5 +106,22 @@ func systemPruneAction(cmd *cobra.Command, args []string) error { if err := imagePrune(cmd, client, ctx); err != nil { return err } - return builderPruneAction(cmd, args) + + var buf bytes.Buffer + outWriter := bufio.NewWriter(&buf) + copiedCmd := *cmd + copiedCmd.SetOut(outWriter) + if err := builderPruneAction(&copiedCmd, args); err != nil { + return err + } + parsedOutput, err := buildkitutil.ParseBuildctlPruneTableOutput(buf.Bytes()) + if err != nil { + return err + } + // pretty print the output of prune build cache + out := cmd.OutOrStdout() + for _, row := range parsedOutput.Rows { + _, _ = fmt.Fprintf(out, "%s\n", row.ID) + } + return err } diff --git a/pkg/buildkitutil/buildctl_prune_output_parser.go b/pkg/buildkitutil/buildctl_prune_output_parser.go index 41914a807d0..439012215bb 100644 --- a/pkg/buildkitutil/buildctl_prune_output_parser.go +++ b/pkg/buildkitutil/buildctl_prune_output_parser.go @@ -26,7 +26,7 @@ const HeaderSize = "SIZE" const HeaderLastAccessed = "LAST ACCESSED" const FinalizerTotal = "Total:" -func ParseBuildctlPruneOutput(out []byte) (*BuildctlPruneOutput, error) { +func ParseBuildctlPruneTableOutput(out []byte) (*BuildctlPruneOutput, error) { tabReader := tabutil.NewReader(fmt.Sprintf("%s\t%s\t%s\t%s", HeaderID, HeaderReclaimable, HeaderSize, HeaderLastAccessed)) scanner := bufio.NewScanner(bytes.NewReader(out)) firstLineParsed := false diff --git a/pkg/buildkitutil/buildctl_prune_output_parser_test.go b/pkg/buildkitutil/buildctl_prune_output_parser_test.go index f22298e9a0c..003f3fc422c 100644 --- a/pkg/buildkitutil/buildctl_prune_output_parser_test.go +++ b/pkg/buildkitutil/buildctl_prune_output_parser_test.go @@ -96,13 +96,13 @@ Total: 584.31MB`)}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ParseBuildctlPruneOutput(tt.args.out) + got, err := ParseBuildctlPruneTableOutput(tt.args.out) if (err != nil) != tt.wantErr { - t.Errorf("ParseBuildctlPruneOutput() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("ParseBuildctlPruneTableOutput() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ParseBuildctlPruneOutput() got = %v, want %v", got, tt.want) + t.Errorf("ParseBuildctlPruneTableOutput() got = %v, want %v", got, tt.want) } }) } From 332b68fb389634dcacc8480bf6b5d45743729b53 Mon Sep 17 00:00:00 2001 From: Zhou Zhiqiang Date: Wed, 3 Aug 2022 20:24:11 +0800 Subject: [PATCH 4/4] chore: update testcases for builtctl output Signed-off-by: Zhou Zhiqiang --- cmd/nerdctl/system_prune.go | 5 ++ .../buildctl_prune_output_parser.go | 68 +++++++++++---- .../buildctl_prune_output_parser_test.go | 87 ++++++++++++++----- 3 files changed, 118 insertions(+), 42 deletions(-) diff --git a/cmd/nerdctl/system_prune.go b/cmd/nerdctl/system_prune.go index 9441983ac2e..99e28f4b554 100644 --- a/cmd/nerdctl/system_prune.go +++ b/cmd/nerdctl/system_prune.go @@ -19,6 +19,7 @@ package main import ( "bufio" "bytes" + "encoding/json" "fmt" "github.com/containerd/nerdctl/pkg/buildkitutil" "strings" @@ -123,5 +124,9 @@ func systemPruneAction(cmd *cobra.Command, args []string) error { for _, row := range parsedOutput.Rows { _, _ = fmt.Fprintf(out, "%s\n", row.ID) } + j, _ := json.Marshal(parsedOutput) + fmt.Fprintf(out, "%s\n", j) + + fmt.Fprintf(out, "%s", buf.Bytes()) return err } diff --git a/pkg/buildkitutil/buildctl_prune_output_parser.go b/pkg/buildkitutil/buildctl_prune_output_parser.go index 439012215bb..4fde2c17aea 100644 --- a/pkg/buildkitutil/buildctl_prune_output_parser.go +++ b/pkg/buildkitutil/buildctl_prune_output_parser.go @@ -1,8 +1,29 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Portions from https://github.com/docker/cli/blob/v20.10.9/cli/command/image/build/context.go + Copyright (C) Docker authors. + Licensed under the Apache License, Version 2.0 + NOTICE: https://github.com/docker/cli/blob/v20.10.9/NOTICE +*/ + package buildkitutil import ( - "bufio" - "bytes" "fmt" "github.com/containerd/nerdctl/pkg/tabutil" "strings" @@ -28,24 +49,27 @@ const FinalizerTotal = "Total:" func ParseBuildctlPruneTableOutput(out []byte) (*BuildctlPruneOutput, error) { tabReader := tabutil.NewReader(fmt.Sprintf("%s\t%s\t%s\t%s", HeaderID, HeaderReclaimable, HeaderSize, HeaderLastAccessed)) - scanner := bufio.NewScanner(bytes.NewReader(out)) - firstLineParsed := false + lines := strings.Split(string(out), "\n") + + totalSize, err := parseTotalSize(lines[len(lines)-2]) + if err != nil { + return nil, err + } + + if len(lines) == 2 { + return &BuildctlPruneOutput{ + TotalSize: totalSize, + Rows: nil, + }, nil + } + + if err := tabReader.ParseHeader(lines[0]); err != nil { + return nil, err + } var rows []BuildctlPruneOutputRow - totalSize := "" - for scanner.Scan() { - line := scanner.Text() - // parse total - if strings.HasPrefix(line, FinalizerTotal) { - totalSize = strings.TrimSpace(line[len(FinalizerTotal):]) - break - } - // parse header - if !firstLineParsed { - _ = tabReader.ParseHeader(line) - firstLineParsed = true - continue - } + + for _, line := range lines[1 : len(lines)-2] { // best effort parse row id, _ := tabReader.ReadRow(line, HeaderID) reclaimable, _ := tabReader.ReadRow(line, HeaderReclaimable) @@ -58,8 +82,16 @@ func ParseBuildctlPruneTableOutput(out []byte) (*BuildctlPruneOutput, error) { LastAccessed: lastAccessed, }) } + return &BuildctlPruneOutput{ TotalSize: totalSize, Rows: rows, }, nil } + +func parseTotalSize(line string) (string, error) { + if strings.HasPrefix(line, FinalizerTotal) { + return strings.TrimSpace(line[len(FinalizerTotal):]), nil + } + return "", fmt.Errorf("parse total size from buildctl prune command ouput, unexpected line, does not contains total size: %s", line) +} diff --git a/pkg/buildkitutil/buildctl_prune_output_parser_test.go b/pkg/buildkitutil/buildctl_prune_output_parser_test.go index 003f3fc422c..215561e5053 100644 --- a/pkg/buildkitutil/buildctl_prune_output_parser_test.go +++ b/pkg/buildkitutil/buildctl_prune_output_parser_test.go @@ -1,3 +1,26 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Portions from https://github.com/docker/cli/blob/v20.10.9/cli/command/image/build/context.go + Copyright (C) Docker authors. + Licensed under the Apache License, Version 2.0 + NOTICE: https://github.com/docker/cli/blob/v20.10.9/NOTICE +*/ + package buildkitutil import ( @@ -17,65 +40,80 @@ func TestParseBuildctlPruneOutput(t *testing.T) { }{ { name: "builtctl prune frees several spaces", - args: args{out: []byte(`ID RECLAIMABLE SIZE LAST ACCESSED -spr33ail5pfddqvf25mnxg013 true 0B -akklguozvrppljhuw3v9t45se* true 4.10kB -skr3tplsx990ttmvyymww9pf2* true 0B -hevhw67hx6yv6zlkob00lkaxm true 6.84MB -lqq6c21uz1e697mgvah93govi true 8.19kB -3qi6zgbbup6h6nuc2axyx39ed true 140.81MB -ise2an3ziszpxqfbm0iwiow6p true 327.58MB -0st72zjm0ei4r60hnbmr0thye true 109.07MB -Total: 584.31MB`)}, + args: args{out: []byte(`ID RECLAIMABLE SIZE LAST ACCESSED +4kaw6aqapf3qvmqj3jskct89r true 0B +i0ys2vka15idnu952zc3le36o* true 0B +zn1cxnxqyxa18mzi5u2bnnqqk* true 4.10kB +x43clde10rasyy42vvzwo5w1r true 156B +kt007lnewdnssgukycqux8rcc true 0B +th54xgnz12r1rsgaoy55pfla1 true 0B +66b6kbtv4iul87bzibzxwgo51 true 0B +nlanvxjtqluiipuwva3uwfwvk true 0B +yh3oha6crdq9y34lmi2jshwhb true 0B +hai2h463u7o8xuybj0339e139 true 0B +Total: 4.25kB +`)}, want: &BuildctlPruneOutput{ TotalSize: "584.31MB", Rows: []BuildctlPruneOutputRow{ { - ID: "spr33ail5pfddqvf25mnxg013", + ID: "4kaw6aqapf3qvmqj3jskct89r", + Reclaimable: "true", + Size: "0B", + LastAccessed: "", + }, + { + ID: "i0ys2vka15idnu952zc3le36o*", Reclaimable: "true", Size: "0B", LastAccessed: "", }, { - ID: "akklguozvrppljhuw3v9t45se*", + ID: "zn1cxnxqyxa18mzi5u2bnnqqk*", Reclaimable: "true", Size: "4.10kB", LastAccessed: "", }, { - ID: "skr3tplsx990ttmvyymww9pf2*", + ID: "x43clde10rasyy42vvzwo5w1r", + Reclaimable: "true", + Size: "156B", + LastAccessed: "", + }, + { + ID: "kt007lnewdnssgukycqux8rcc", Reclaimable: "true", Size: "0B", LastAccessed: "", }, { - ID: "hevhw67hx6yv6zlkob00lkaxm", + ID: "th54xgnz12r1rsgaoy55pfla1", Reclaimable: "true", - Size: "6.84MB", + Size: "0B", LastAccessed: "", }, { - ID: "lqq6c21uz1e697mgvah93govi", + ID: "66b6kbtv4iul87bzibzxwgo51", Reclaimable: "true", - Size: "8.19kB", + Size: "0B", LastAccessed: "", }, { - ID: "3qi6zgbbup6h6nuc2axyx39ed", + ID: "nlanvxjtqluiipuwva3uwfwvk", Reclaimable: "true", - Size: "140.81MB", + Size: "0B", LastAccessed: "", }, { - ID: "ise2an3ziszpxqfbm0iwiow6p", + ID: "yh3oha6crdq9y34lmi2jshwhb", Reclaimable: "true", - Size: "327.58MB", + Size: "0B", LastAccessed: "", }, { - ID: "0st72zjm0ei4r60hnbmr0thye", + ID: "hai2h463u7o8xuybj0339e139", Reclaimable: "true", - Size: "109.07MB", + Size: "0B", LastAccessed: "", }, }, @@ -85,7 +123,8 @@ Total: 584.31MB`)}, { name: "buildctl prune frees no spaces", args: args{ - out: []byte(`Total: 0B`), + out: []byte(`Total: 0B +`), }, want: &BuildctlPruneOutput{ TotalSize: "0B",