From 0623a034601b27acfbefd11bd859ea6fe34575f6 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Sun, 3 Apr 2022 17:19:19 +0000 Subject: [PATCH 1/2] Add support for CRIU's 'ignore' manage-cgroup mode This adds an already existing CRIU cgroup mode to be selectable from runc: 'ignore'. This is a fix for errors seen with Podman containers that have been restored multiple times into new cgroups. CRIU fails to checkpoint a container that has already been restored once in a cgroup named differently than the original container. CRIU keeps information about the original cgroup and during restore in a differently named cgroup CRIU still tries to restore to original cgroup. This leads to a process with wrong cgroup settings and a second checkpoint of that container fails. With 'ignore' CRIU will ignore any cgroups and leave the cgroup setting to runc. Signed-off-by: Adrian Reber --- checkpoint.go | 4 +++- man/runc-checkpoint.8.md | 2 +- man/runc-restore.8.md | 2 +- restore.go | 5 ++++- tests/integration/checkpoint.bats | 37 +++++++++++++++++++++++++++++++ 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/checkpoint.go b/checkpoint.go index 7597e552df9..b805542b9eb 100644 --- a/checkpoint.go +++ b/checkpoint.go @@ -38,7 +38,7 @@ checkpointed.`, cli.StringFlag{Name: "page-server", Value: "", Usage: "ADDRESS:PORT of the page server"}, cli.BoolFlag{Name: "file-locks", Usage: "handle file locks, for safety"}, cli.BoolFlag{Name: "pre-dump", Usage: "dump container's memory information only, leave the container running after this"}, - cli.StringFlag{Name: "manage-cgroups-mode", Value: "", Usage: "cgroups mode: 'soft' (default), 'full' and 'strict'"}, + cli.StringFlag{Name: "manage-cgroups-mode", Value: "", Usage: "cgroups mode: 'soft' (default), 'ignore', 'full' and 'strict'"}, cli.StringSliceFlag{Name: "empty-ns", Usage: "create a namespace, but don't restore its properties"}, cli.BoolFlag{Name: "auto-dedup", Usage: "enable auto deduplication of memory images"}, }, @@ -147,6 +147,8 @@ func setManageCgroupsMode(context *cli.Context, options *libcontainer.CriuOpts) options.ManageCgroupsMode = criu.CriuCgMode_FULL case "strict": options.ManageCgroupsMode = criu.CriuCgMode_STRICT + case "ignore": + options.ManageCgroupsMode = criu.CriuCgMode_IGNORE default: return errors.New("Invalid manage cgroups mode") } diff --git a/man/runc-checkpoint.8.md b/man/runc-checkpoint.8.md index 373259d4ccf..545d160dd26 100644 --- a/man/runc-checkpoint.8.md +++ b/man/runc-checkpoint.8.md @@ -57,7 +57,7 @@ together with **criu lazy-pages**. See : Do a pre-dump, i.e. dump container's memory information only, leaving the container running. See [criu iterative migration](https://criu.org/Iterative_migration). -**--manage-cgroups-mode** **soft**|**full**|**strict**. +**--manage-cgroups-mode** **soft**|**ignore**|**full**|**strict**. : Cgroups mode. Default is **soft**. See [criu --manage-cgroups option](https://criu.org/CLI/opt/--manage-cgroups). diff --git a/man/runc-restore.8.md b/man/runc-restore.8.md index a2b3da6c6fa..f214f7e3e77 100644 --- a/man/runc-restore.8.md +++ b/man/runc-restore.8.md @@ -37,7 +37,7 @@ image files directory. : Allow checkpoint/restore of file locks. See [criu --file-locks option](https://criu.org/CLI/opt/--file-locks). -**--manage-cgroups-mode** **soft**|**full**|**strict**. +**--manage-cgroups-mode** **soft**|**ignore**|**full**|**strict**. : Cgroups mode. Default is **soft**. See [criu --manage-cgroups option](https://criu.org/CLI/opt/--manage-cgroups). diff --git a/restore.go b/restore.go index ccd1b232bc9..73b4f48bf36 100644 --- a/restore.go +++ b/restore.go @@ -53,7 +53,7 @@ using the runc checkpoint command.`, cli.StringFlag{ Name: "manage-cgroups-mode", Value: "", - Usage: "cgroups mode: 'soft' (default), 'full' and 'strict'", + Usage: "cgroups mode: 'soft' (default), 'ignore', 'full' and 'strict'", }, cli.StringFlag{ Name: "bundle, b", @@ -116,6 +116,9 @@ using the runc checkpoint command.`, if err := setEmptyNsMask(context, options); err != nil { return err } + if err := setManageCgroupsMode(context, options); err != nil { + return err + } status, err := startContainer(context, CT_ACT_RESTORE, options) if err != nil { return err diff --git a/tests/integration/checkpoint.bats b/tests/integration/checkpoint.bats index 615ef8cb65f..87e0f619252 100644 --- a/tests/integration/checkpoint.bats +++ b/tests/integration/checkpoint.bats @@ -405,3 +405,40 @@ function simple_cr() { # busybox should be back up and running testcontainer test_busybox running } + +@test "checkpoint and restore into a different cgroup" { + local pid name=test-diff-cgroup + + set_cgroups_path + runc run -d --console-socket "$CONSOLE_SOCKET" --pid-file pid "$name" + [ "$status" -eq 0 ] + + testcontainer "$name" running + # Check the cgroup. + pid="$(cat pid)" + original_cgroup="$OCI_CGROUPS_PATH" + echo "OCI_CGROUPS_PATH=$OCI_CGROUPS_PATH" + cat "/proc/$pid/cgroup" + + runc checkpoint --manage-cgroups-mode ignore --work-path ./work-dir "$name" + grep -B 5 Error ./work-dir/dump.log || true + [ "$status" -eq 0 ] + + testcontainer "$name" checkpointed + + # Modify the cgroup path. + set_cgroups_path + runc restore --manage-cgroups-mode ignore --pid-file pid -d \ + --work-path ./work-dir --console-socket "$CONSOLE_SOCKET" "$name" + grep -B 5 Error ./work-dir/restore.log || true + [ "$status" -eq 0 ] + + testcontainer "$name" running + + # Check the cgroup name has changed. + pid="$(cat pid)" + echo "OCI_CGROUPS_PATH=$OCI_CGROUPS_PATH" + cat "/proc/$pid/cgroup" + new_cgroup="$OCI_CGROUPS_PATH" + [ "$original_cgroup" != "$new_cgroup" ] +} From 0e8b1e7ec86d953afc4a7de363899aff9ab53a30 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Fri, 29 Jul 2022 16:06:57 -0700 Subject: [PATCH 2/2] tests/rootless.sh: add debug Signed-off-by: Kir Kolyshkin --- tests/rootless.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/rootless.sh b/tests/rootless.sh index 83be248a108..c487db81a65 100755 --- a/tests/rootless.sh +++ b/tests/rootless.sh @@ -146,7 +146,16 @@ function disable_cgroup() { [ -d "$CGROUP_MOUNT/$cg$CGROUP_PATH" ] && rmdir "$CGROUP_MOUNT/$cg$CGROUP_PATH" done # cgroup v2 - [ -d "$CGROUP_MOUNT/$CGROUP_PATH" ] && rmdir "$CGROUP_MOUNT/$CGROUP_PATH" + if [ -d "$CGROUP_MOUNT$CGROUP_PATH" ]; then + if ! rmdir "$CGROUP_MOUNT$CGROUP_PATH"; then + echo "ERROR: could not remove $CGROUP_MOUNT$CGROUP_PATH" + ls -lR "$CGROUP_MOUNT$CGROUP_PATH" + for pid in $(find "$CGROUP_MOUNT$CGROUP_PATH" -name cgroup.procs -exec cat '{}' +); do + ps -c $pid + done + return 1 + fi + fi return 0 }