From ef51b37df5dabdd2d3640e3529f56e03ab309673 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 25 Sep 2018 21:35:28 -0400 Subject: [PATCH 1/2] WIP: run compose in small supermin VM Allows for unprivileged runs of rpm-ostree compose tree. --- src/cmd-build | 25 ++++---- src/cmd-init | 5 +- src/cmdlib.sh | 107 +++++++++++++++++++++++++++++++--- src/deps.txt | 3 + src/supermin-init | 144 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 260 insertions(+), 24 deletions(-) create mode 100755 src/supermin-init diff --git a/src/cmd-build b/src/cmd-build index c460370108..5af9217bd3 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -29,11 +29,18 @@ EOF composejson=$(pwd)/tmp/compose.json changed_stamp=$(pwd)/tmp/treecompose.changed # --cache-only is here since `fetch` is a separate verb. -runcompose --cache-only --add-metadata-from-json ${commitmeta_input_json} \ - --touch-if-changed "${changed_stamp}" \ - --write-composejson-to ${composejson} -commit=$(ostree --repo=${workdir}/repo-build rev-parse "${ref}") -version=$(ostree --repo=${workdir}/repo-build show --print-metadata-key=version ${commit} | sed -e "s,',,g") +args=" --cache-only --add-metadata-from-json ${commitmeta_input_json}" +args+=" --touch-if-changed ${changed_stamp}" +args+=" --write-composejson-to ${composejson}" + +runcompose_in_vm ${previous_commit:-null} $ref $args + +# update summary file +ostree --repo=${workdir}/repo summary -u + +commit=$(ostree --repo=${workdir}/repo rev-parse "${ref}") +version=$(ostree --repo=${workdir}/repo show --print-metadata-key=version ${commit} | sed -e "s,',,g") + # Very special handling for --write-composejson-to as rpm-ostree doesn't # write it if the commit didn't change. if [ -f "${changed_stamp}" ]; then @@ -45,14 +52,6 @@ else # Grab the previous JSON cp -a --reflink=auto ${workdir}/tmp/compose-${commit}.json ${composejson} fi -# https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872 -# The passwd files (among others) don't have world readability. This won't -# actually corrupt the repository as the *canonical* permissions are stored -# as xattrs. Probably what we should do is have an ostree option to specify -# a permission mask for objects. -sudo chmod -R a+rX ${workdir}/repo-build/objects -ostree --repo=${workdir}/repo pull-local ${workdir}/repo-build "${ref}" -ostree --repo=${workdir}/repo summary -u sha256sum_str() { sha256sum | cut -f 1 -d ' ' diff --git a/src/cmd-init b/src/cmd-init index 4f16e83910..a7b59eac6a 100755 --- a/src/cmd-init +++ b/src/cmd-init @@ -102,8 +102,9 @@ mkdir -p installer fi ) -mkdir -p cache mkdir -p builds mkdir -p tmp ostree --repo=repo init --mode=archive -ostree --repo=repo-build init --mode=bare-user + +# prepare the unprivileged vm to run the compose in +prepare_vm diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 1293f5ec88..65678804c2 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -1,5 +1,13 @@ # Shared shell script library +# Global variables +export workdir=$(pwd) +export configdir=${workdir}/src/config +export manifest=${configdir}/manifest.yaml +export superminpreparedir="${workdir}/tmp/supermin-prepare.d" +export superminbuilddir="${workdir}/tmp/supermin-build.d" +export cachesimg="${workdir}/caches.qcow2" + fatal() { echo "error: $@" 1>&2; exit 1 } @@ -28,13 +36,13 @@ preflight() { fatal "Unable to find /dev/kvm" fi - if ! capsh --print | grep -q 'Current.*cap_sys_admin'; then - fatal "This container must currently be run with --privileged" - fi +# if ! capsh --print | grep -q 'Current.*cap_sys_admin'; then +# fatal "This container must currently be run with --privileged" +# fi - if ! sudo true; then - fatal "The user must currently have sudo privileges" - fi +# if ! sudo true; then +# fatal "The user must currently have sudo privileges" +# fi # permissions on /dev/kvm vary by (host) distro. If it's # not writable, recreate it. @@ -51,9 +59,12 @@ prepare_build() { fatal "No $(pwd)/repo found; did you run coreos-assembler init?" fi - export workdir=$(pwd) - export configdir=${workdir}/src/config - export manifest=${configdir}/manifest.yaml +# export workdir=$(pwd) +# export configdir=${workdir}/src/config +# export manifest=${configdir}/manifest.yaml +# export superminpreparedir="${workdir}/tmp/supermin-prepare.d" +# export superminbuilddir="${workdir}/tmp/supermin-build.d" +# export cachesimg="${workdir}/caches.qcow2" if ! [ -f "${manifest}" ]; then fatal "Failed to find ${manifest}" @@ -102,3 +113,81 @@ runcompose() { ${TREECOMPOSE_FLAGS:-} ${manifest} "$@" set +x } + +prepare_vm() { + # rpms to create appliance VM out of + rpms=' bash vim-minimal coreutils util-linux procps-ng kmod kernel-modules' + rpms+=' cifs-utils' # for samba + rpms+=' systemd' # for clean reboot + rpms+=' dhcp-client bind-export-libs iproute' # networking + rpms+=' rpm-ostree distribution-gpg-keys' # to run the compose + rpms+=' selinux-policy selinux-policy-targeted policycoreutils' #selinux + + # prepare appliance VM + supermin -v --prepare --use-installed $rpms -o "${superminpreparedir}" + supermin -v --build "${superminpreparedir}" \ + --include-packagelist --size 4G -f ext2 -o "${superminbuilddir}" + + # Create a disk image for our caches (pkgcache and bare-user build repo) + [ ! -d "${workdir}/tmp/emptydir" ] && mkdir "${workdir}/tmp/emptydir" + if [ ! -f "${cachesimg}" ]; then + virt-make-fs --format=qcow2 --type=xfs \ + --size=10G "${workdir}/tmp/emptydir" "${cachesimg}" + fi +} + +run_vm() { + init=$1 + set -x + + # Copy in the init script with the code we want to run + chmod +x "${init}" + virt-copy-in -a "${superminbuilddir}/root" "${init}" / + + # Execute unprivileged VM to run compose. Some notes: + # - smb="${workdir}" - we'll serve our workdir over samba + # - sda - rootfs from supermin build + # - sdb - caches.qcow2 - we store the bare-user repo and pkgcache here + # - discard=unmap - using virtio-scsi disks and discard=unmap so we can fstrim + # and recover disk space from the VM + qemu-kvm -nodefaults -nographic -m 2048 -no-reboot \ + -kernel "${superminbuilddir}/kernel" \ + -initrd "${superminbuilddir}/initrd" \ + -netdev user,id=eth0,hostname=supermin,smb="${workdir}",hostfwd=tcp:127.0.0.1:8000-:8000 \ + -device virtio-net-pci,netdev=eth0 \ + -device virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 \ + -drive if=none,id=drive-scsi0-0-0-0,snapshot=on,file="${superminbuilddir}/root" \ + -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=1 \ + -drive if=none,id=drive-scsi0-0-0-1,discard=unmap,file="${cachesimg}" \ + -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1 \ + -serial stdio -append "root=/dev/sda console=ttyS0 selinux=1 enforcing=0 autorelabel=1" + + if [ -f "${workdir}/tmp/supermin-failure" ]; then + rm -f "${workdir}/tmp/supermin-failure" + fatal "Detected failure from compose VM run" + fi +} + +runcompose_in_vm() { + previous_commit=$1 + ref=$2 + shift; shift; + + local treecompose_args="" + if ! grep -q '^# disable-unified-core' "${manifest}"; then + treecompose_args="${treecompose_args} --unified-core" + fi + + cmd="rpm-ostree compose tree --repo=${workdir}/repo-build" + cmd+=" --cachedir=${workdir}/cache ${treecompose_args}" + cmd+=" ${TREECOMPOSE_FLAGS:-} ${manifest} $@" + + # populate variables in init script + template=$(dirname $0)/supermin-init + sed -e "s|^workdir=|workdir=\"${workdir}\"|" \ + -e "s|^previous_commit=|previous_commit=\"${previous_commit}\"|" \ + -e "s|^ref=|ref=\"${ref}\"|" \ + -e "s|^cmd=|cmd=\"${cmd}\"|" $template > "${superminbuilddir}/init" + + run_vm "${superminbuilddir}/init" +} diff --git a/src/deps.txt b/src/deps.txt index 52ce40edd7..47ad715b6a 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -36,3 +36,6 @@ jq awscli # For ignition file validation in cmd-run ignition + +# For running supermin VM via qemu and sharing via samba +samba cifs-utils diff --git a/src/supermin-init b/src/supermin-init new file mode 100755 index 0000000000..7ee3d663c7 --- /dev/null +++ b/src/supermin-init @@ -0,0 +1,144 @@ +#!/bin/sh +set -ux + +trap fatal SIGHUP SIGINT SIGQUIT SIGABRT + +export PATH=/usr/sbin/:$PATH + +mount -t proc /proc /proc +mount -t sysfs /sys /sys +mount -t devtmpfs devtmpfs /dev + +# Load selinux policy +LANG=C /sbin/load_policy -i + +# Load kernel module for 9pnet_virtio for 9pfs mount +#/sbin/modprobe 9pnet_virtio + +# Need fuse module for rofiles-fuse/bwrap during post scripts run +/sbin/modprobe fuse + +# set up networking +/usr/sbin/dhclient eth0 & +sleep 2 # wait for dhcp + + +# these varaibles will be populated in this script before running +workdir= +ref= +cmd= +previous_commit= + +# set a variable for our mountpoint for hostworkdir +hostworkdir='/hostworkdir' + +# If usuing unified core we need to use the repo-build bare-user repo. +# If not use the repo archive repo. +if [[ "$cmd" =~ unified-core ]]; then + unifiedcore=1 + repo='repo-build' +else + unifiedcore=1 + repo='repo' +fi + + +[ -d "${workdir}" ] || mkdir -p "${workdir}" +[ -d "${hostworkdir}" ] || mkdir -p "${hostworkdir}" + +# mount cache disk +mount /dev/sdb "${workdir}" + +#bash -i + +# mount host "${workdir}" via samba +#ount -t 9p -o rw,trans=virtio,version=9p2000.L srv /hostsrv/ +# cat /proc/fs/cifs/DebugData +mount -t cifs -o 'username=root,password=,cache=loose' '\\10.0.2.4\qemu' "${hostworkdir}" + +# symlink some things from $workdir to $hostworkdir +[ -e "${workdir}/tmp" ] || ln -s "${hostworkdir}/tmp" "${workdir}/tmp" +[ -e "${workdir}/src" ] || ln -s "${hostworkdir}/src" "${workdir}/src" +[ -e "${workdir}/localrepo" ] || ln -s "${hostworkdir}/localrepo" "${workdir}/localrepo" + +# create a few directories if they don't exist +[ -d "${workdir}/cache" ] || mkdir "${workdir}/cache" +[ -d "${workdir}/repo-build" ] || mkdir "${workdir}/repo-build" +[ -d "${workdir}/repo" ] || mkdir "${workdir}/repo" + +# initialize the repos if they don't exist +[ -f "${workdir}/repo-build/config" ] || ostree init --mode=bare-user --repo="${workdir}/repo-build/" +[ -f "${workdir}/repo/config" ] || ostree init --mode=archive --repo="${workdir}/repo/" + + +#bash -i + + +compose() { + # make sure previous_commit exists in the build repo + if [ "${previous_commit}" != "null" ]; then + pc=$(ostree --repo="${workdir}/${repo}" rev-parse "${ref}" || true) + if [ "${previous_commit}" != "${pc}" ]; then + # is it possible to use --commit-metadata-only here ? + ostree pull-local --repo="${workdir}/${repo}" "${hostworkdir}/repo/" "$ref" + fi + fi + + # run the command from given to us + $cmd --repo="${workdir}/${repo}" || fatal + + # https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872 + # The passwd files (among others) don't have world readability. This won't + # actually corrupt the repository as the *canonical* permissions are stored + # as xattrs. Probably what we should do is have an ostree option to specify + # a permission mask for objects. + [ "${unifiedcore}" = "1" ] && chmod -R a+rX "${workdir}/repo-build/objects" + + #bash -i + + # Sync over to repo in $hostworkdir + if [ "${unifiedcore}" = "1" ]; then + # Need to do this in two steps instead of one + # https://github.com/ostreedev/ostree/issues/1734 + ostree pull-local --repo="${workdir}/repo/" "${workdir}/repo-build/" $ref || fatal + ostree pull-local --repo="${hostworkdir}/repo/" "${workdir}/repo/" $ref || fatal + else + ostree pull-local --repo="${hostworkdir}/repo/" "${workdir}/repo/" $ref || fatal + fi + + # prune - we'll leave just one commit in repo + if [ "${unifiedcore}" = "1" ]; then + ostree prune --repo="${workdir}/repo-build/" --only-branch "${ref}" --depth=0 || fatal + ostree prune --repo="${workdir}/repo/" --only-branch "${ref}" --depth=0 || fatal + else + ostree prune --repo="${workdir}/repo/" --only-branch "${ref}" --depth=0 || fatal + fi + + #bash -i +} + +fatal() { + echo "Failure from inside supermin VM" + touch "${workdir}/tmp/supermin-failure" + cleanup +} + +cleanup() { + set +eu + # release disk space back to the host and unmount + fstrim -v "${workdir}" + sync + umount "${hostworkdir}" + umount "${workdir}" + + # shutdown + reboot -f +# echo 1 > /proc/sys/kernel/sysrq +# echo o > /proc/sysrq-trigger +# sleep 10 +} + +# run the compose +compose +# run the cleanup +cleanup From 965a0772a5f82f7fbfb204dedb34c8ee8246230c Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 28 Sep 2018 10:33:11 -0400 Subject: [PATCH 2/2] build: supermin: assume unified-core is the default This simplifies things a bit. Also the xattr copying from bare-user to archive repo over smb file share has been fixed. --- src/supermin-init | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/src/supermin-init b/src/supermin-init index 7ee3d663c7..34da06d56f 100755 --- a/src/supermin-init +++ b/src/supermin-init @@ -32,17 +32,6 @@ previous_commit= # set a variable for our mountpoint for hostworkdir hostworkdir='/hostworkdir' -# If usuing unified core we need to use the repo-build bare-user repo. -# If not use the repo archive repo. -if [[ "$cmd" =~ unified-core ]]; then - unifiedcore=1 - repo='repo-build' -else - unifiedcore=1 - repo='repo' -fi - - [ -d "${workdir}" ] || mkdir -p "${workdir}" [ -d "${hostworkdir}" ] || mkdir -p "${hostworkdir}" @@ -77,42 +66,31 @@ mount -t cifs -o 'username=root,password=,cache=loose' '\\10.0.2.4\qemu' "${host compose() { # make sure previous_commit exists in the build repo if [ "${previous_commit}" != "null" ]; then - pc=$(ostree --repo="${workdir}/${repo}" rev-parse "${ref}" || true) + pc=$(ostree --repo="${workdir}/repo-build" rev-parse "${ref}" || true) if [ "${previous_commit}" != "${pc}" ]; then # is it possible to use --commit-metadata-only here ? - ostree pull-local --repo="${workdir}/${repo}" "${hostworkdir}/repo/" "$ref" + ostree pull-local --repo="${workdir}/repo-build" "${hostworkdir}/repo/" "$ref" fi fi # run the command from given to us - $cmd --repo="${workdir}/${repo}" || fatal + $cmd || fatal # https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872 # The passwd files (among others) don't have world readability. This won't # actually corrupt the repository as the *canonical* permissions are stored # as xattrs. Probably what we should do is have an ostree option to specify # a permission mask for objects. - [ "${unifiedcore}" = "1" ] && chmod -R a+rX "${workdir}/repo-build/objects" + chmod -R a+rX "${workdir}/repo-build/objects" #bash -i # Sync over to repo in $hostworkdir - if [ "${unifiedcore}" = "1" ]; then - # Need to do this in two steps instead of one - # https://github.com/ostreedev/ostree/issues/1734 - ostree pull-local --repo="${workdir}/repo/" "${workdir}/repo-build/" $ref || fatal - ostree pull-local --repo="${hostworkdir}/repo/" "${workdir}/repo/" $ref || fatal - else - ostree pull-local --repo="${hostworkdir}/repo/" "${workdir}/repo/" $ref || fatal - fi + ostree pull-local --repo="${hostworkdir}/repo/" "${workdir}/repo-build/" $ref || fatal + # Sync over to repo in $hostworkdir # prune - we'll leave just one commit in repo - if [ "${unifiedcore}" = "1" ]; then - ostree prune --repo="${workdir}/repo-build/" --only-branch "${ref}" --depth=0 || fatal - ostree prune --repo="${workdir}/repo/" --only-branch "${ref}" --depth=0 || fatal - else - ostree prune --repo="${workdir}/repo/" --only-branch "${ref}" --depth=0 || fatal - fi + ostree prune --repo="${workdir}/repo-build/" --only-branch "${ref}" --depth=0 || fatal #bash -i } @@ -133,9 +111,6 @@ cleanup() { # shutdown reboot -f -# echo 1 > /proc/sys/kernel/sysrq -# echo o > /proc/sysrq-trigger -# sleep 10 } # run the compose