WIP: run compose in small supermin VM

src/cmd-build

@@ -29,11 +29,18 @@ EOF
 composejson=$(pwd)/tmp/compose.json
 changed_stamp=$(pwd)/tmp/treecompose.changed
 # --cache-only is here since `fetch` is a separate verb.
-runcompose --cache-only --add-metadata-from-json ${commitmeta_input_json} \
-           --touch-if-changed "${changed_stamp}" \
-           --write-composejson-to ${composejson}
-commit=$(ostree --repo=${workdir}/repo-build rev-parse "${ref}")
-version=$(ostree --repo=${workdir}/repo-build show --print-metadata-key=version ${commit} | sed -e "s,',,g")
+args=" --cache-only --add-metadata-from-json ${commitmeta_input_json}"
+args+=" --touch-if-changed ${changed_stamp}"
+args+=" --write-composejson-to ${composejson}"
+
+runcompose_in_vm ${previous_commit:-null} $ref $args
+
+# update summary file
+ostree --repo=${workdir}/repo summary -u
+
+commit=$(ostree --repo=${workdir}/repo rev-parse "${ref}")
+version=$(ostree --repo=${workdir}/repo show --print-metadata-key=version ${commit} | sed -e "s,',,g")
+
 # Very special handling for --write-composejson-to as rpm-ostree doesn't
 # write it if the commit didn't change.
 if [ -f "${changed_stamp}" ]; then
@@ -45,14 +52,6 @@ else
     # Grab the previous JSON
     cp -a --reflink=auto ${workdir}/tmp/compose-${commit}.json ${composejson}
 fi
-# https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872
-# The passwd files (among others) don't have world readability.  This won't
-# actually corrupt the repository as the *canonical* permissions are stored
-# as xattrs.  Probably what we should do is have an ostree option to specify
-# a permission mask for objects.
-sudo chmod -R a+rX ${workdir}/repo-build/objects
-ostree --repo=${workdir}/repo pull-local ${workdir}/repo-build "${ref}"
-ostree --repo=${workdir}/repo summary -u
 
 sha256sum_str() {
     sha256sum | cut -f 1 -d ' '

src/cmd-init

@@ -102,8 +102,9 @@ mkdir -p installer
  fi
 )
 
-mkdir -p cache
 mkdir -p builds
 mkdir -p tmp
 ostree --repo=repo init --mode=archive
-ostree --repo=repo-build init --mode=bare-user
+
+# prepare the unprivileged vm to run the compose in
+prepare_vm

src/cmdlib.sh

@@ -1,5 +1,13 @@
 # Shared shell script library
 
+# Global variables
+export workdir=$(pwd)
+export configdir=${workdir}/src/config
+export manifest=${configdir}/manifest.yaml
+export superminpreparedir="${workdir}/tmp/supermin-prepare.d"
+export superminbuilddir="${workdir}/tmp/supermin-build.d"
+export cachesimg="${workdir}/caches.qcow2"
+
 fatal() {
     echo "error: $@" 1>&2; exit 1
 }
@@ -28,13 +36,13 @@ preflight() {
         fatal "Unable to find /dev/kvm"
     fi
 
-    if ! capsh --print | grep -q 'Current.*cap_sys_admin'; then
-        fatal "This container must currently be run with --privileged"
-    fi
+#   if ! capsh --print | grep -q 'Current.*cap_sys_admin'; then
+#       fatal "This container must currently be run with --privileged"
+#   fi
 
-    if ! sudo true; then
-        fatal "The user must currently have sudo privileges"
-    fi
+#   if ! sudo true; then
+#       fatal "The user must currently have sudo privileges"
+#   fi
 
     # permissions on /dev/kvm vary by (host) distro.  If it's
     # not writable, recreate it.
@@ -51,9 +59,12 @@ prepare_build() {
         fatal "No $(pwd)/repo found; did you run coreos-assembler init?"
     fi
 
-    export workdir=$(pwd)
-    export configdir=${workdir}/src/config
-    export manifest=${configdir}/manifest.yaml
+#   export workdir=$(pwd)
+#   export configdir=${workdir}/src/config
+#   export manifest=${configdir}/manifest.yaml
+#   export superminpreparedir="${workdir}/tmp/supermin-prepare.d"
+#   export superminbuilddir="${workdir}/tmp/supermin-build.d"
+#   export cachesimg="${workdir}/caches.qcow2"
 
     if ! [ -f "${manifest}" ]; then
         fatal "Failed to find ${manifest}"
@@ -102,3 +113,81 @@ runcompose() {
          ${TREECOMPOSE_FLAGS:-} ${manifest} "$@"
     set +x
 }
+
+prepare_vm() {
+    # rpms to create appliance VM out of
+    rpms=' bash vim-minimal coreutils util-linux procps-ng kmod kernel-modules'
+    rpms+=' cifs-utils' # for samba
+    rpms+=' systemd' # for clean reboot
+    rpms+=' dhcp-client bind-export-libs iproute' # networking
+    rpms+=' rpm-ostree distribution-gpg-keys' # to run the compose
+    rpms+=' selinux-policy selinux-policy-targeted policycoreutils' #selinux
+
+    # prepare appliance VM
+    supermin -v --prepare --use-installed $rpms -o "${superminpreparedir}"
+    supermin -v --build "${superminpreparedir}" \
+             --include-packagelist --size 4G -f ext2 -o "${superminbuilddir}"
+
+    # Create a disk image for our caches (pkgcache and bare-user build repo)
+    [ ! -d "${workdir}/tmp/emptydir" ] && mkdir "${workdir}/tmp/emptydir"
+    if [ ! -f "${cachesimg}" ]; then
+        virt-make-fs --format=qcow2 --type=xfs \
+                     --size=10G "${workdir}/tmp/emptydir" "${cachesimg}"
+    fi
+}
+
+run_vm() {
+    init=$1
+    set -x
+
+    # Copy in the init script with the code we want to run  
+    chmod +x "${init}"
+    virt-copy-in -a "${superminbuilddir}/root" "${init}" /
+
+    # Execute unprivileged VM to run compose. Some notes:
+    # - smb="${workdir}" - we'll serve our workdir over samba
+    # - sda - rootfs from supermin build
+    # - sdb - caches.qcow2 - we store the bare-user repo and pkgcache here
+    # - discard=unmap - using virtio-scsi disks and discard=unmap so we can fstrim
+    #                   and recover disk space from the VM
+    qemu-kvm -nodefaults -nographic -m 2048 -no-reboot \
+              -kernel "${superminbuilddir}/kernel" \
+              -initrd "${superminbuilddir}/initrd" \
+              -netdev user,id=eth0,hostname=supermin,smb="${workdir}",hostfwd=tcp:127.0.0.1:8000-:8000 \
+              -device virtio-net-pci,netdev=eth0 \
+              -device virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 \
+              -drive if=none,id=drive-scsi0-0-0-0,snapshot=on,file="${superminbuilddir}/root" \
+              -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=1 \
+              -drive if=none,id=drive-scsi0-0-0-1,discard=unmap,file="${cachesimg}" \
+              -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1 \
+              -serial stdio -append "root=/dev/sda console=ttyS0 selinux=1 enforcing=0 autorelabel=1"
+
+    if [ -f "${workdir}/tmp/supermin-failure" ]; then
+        rm -f "${workdir}/tmp/supermin-failure"
+        fatal "Detected failure from compose VM run"
+    fi
+}
+
+runcompose_in_vm() {
+    previous_commit=$1
+    ref=$2
+    shift; shift;
+
+    local treecompose_args=""
+    if ! grep -q '^# disable-unified-core' "${manifest}"; then
+        treecompose_args="${treecompose_args} --unified-core"
+    fi
+
+    cmd="rpm-ostree compose tree --repo=${workdir}/repo-build"
+    cmd+=" --cachedir=${workdir}/cache ${treecompose_args}"
+    cmd+=" ${TREECOMPOSE_FLAGS:-} ${manifest} $@"
+
+    # populate variables in init script
+    template=$(dirname $0)/supermin-init
+    sed -e "s|^workdir=|workdir=\"${workdir}\"|" \
+        -e "s|^previous_commit=|previous_commit=\"${previous_commit}\"|" \
+        -e "s|^ref=|ref=\"${ref}\"|" \
+        -e "s|^cmd=|cmd=\"${cmd}\"|" $template > "${superminbuilddir}/init"
+
+    run_vm "${superminbuilddir}/init"
+}

src/deps.txt

@@ -36,3 +36,6 @@ jq awscli
 
 # For ignition file validation in cmd-run
 ignition
+
+# For running supermin VM via qemu and sharing via samba
+samba cifs-utils

src/supermin-init

@@ -0,0 +1,119 @@
+#!/bin/sh
+set -ux
+
+trap fatal SIGHUP SIGINT SIGQUIT SIGABRT
+
+export PATH=/usr/sbin/:$PATH
+
+mount -t proc /proc /proc
+mount -t sysfs /sys /sys
+mount -t devtmpfs devtmpfs /dev
+
+# Load selinux policy
+LANG=C /sbin/load_policy  -i
+
+# Load kernel module for 9pnet_virtio for 9pfs mount
+#/sbin/modprobe 9pnet_virtio
+
+# Need fuse module for rofiles-fuse/bwrap during post scripts run
+/sbin/modprobe fuse
+
+# set up networking
+/usr/sbin/dhclient eth0 & 
+sleep 2 # wait for dhcp
+
+
+# these varaibles will be populated in this script before running
+workdir=
+ref=
+cmd=
+previous_commit=
+
+# set a variable for our mountpoint for hostworkdir
+hostworkdir='/hostworkdir'
+
+[ -d "${workdir}" ]     || mkdir -p "${workdir}"
+[ -d "${hostworkdir}" ] || mkdir -p "${hostworkdir}"
+
+# mount cache disk 
+mount /dev/sdb "${workdir}"
+
+#bash -i 
+
+# mount host "${workdir}" via samba
+#ount -t 9p -o rw,trans=virtio,version=9p2000.L srv /hostsrv/
+# cat /proc/fs/cifs/DebugData
+mount -t cifs -o 'username=root,password=,cache=loose' '\\10.0.2.4\qemu' "${hostworkdir}"
+
+# symlink some things from $workdir to $hostworkdir
+[ -e "${workdir}/tmp" ]       || ln -s "${hostworkdir}/tmp"       "${workdir}/tmp"
+[ -e "${workdir}/src" ]       || ln -s "${hostworkdir}/src"       "${workdir}/src"
+[ -e "${workdir}/localrepo" ] || ln -s "${hostworkdir}/localrepo" "${workdir}/localrepo"
+
+# create a few directories if they don't exist
+[ -d "${workdir}/cache" ]      || mkdir "${workdir}/cache"
+[ -d "${workdir}/repo-build" ] || mkdir "${workdir}/repo-build"
+[ -d "${workdir}/repo" ]       || mkdir "${workdir}/repo"
+
+# initialize the repos if they don't exist
+[ -f "${workdir}/repo-build/config" ] || ostree init --mode=bare-user --repo="${workdir}/repo-build/"
+[ -f "${workdir}/repo/config" ]       || ostree init --mode=archive --repo="${workdir}/repo/"
+
+
+#bash -i
+
+
+compose() {
+    # make sure previous_commit exists in the build repo
+    if [ "${previous_commit}" != "null" ]; then
+        pc=$(ostree --repo="${workdir}/repo-build" rev-parse "${ref}" || true)
+        if [ "${previous_commit}" != "${pc}" ]; then
+            # is it possible to use --commit-metadata-only here ? 
+            ostree pull-local --repo="${workdir}/repo-build" "${hostworkdir}/repo/" "$ref"
+        fi
+    fi
+
+    # run the command from given to us
+    $cmd || fatal
+
+    # https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872
+    # The passwd files (among others) don't have world readability.  This won't
+    # actually corrupt the repository as the *canonical* permissions are stored
+    # as xattrs.  Probably what we should do is have an ostree option to specify
+    # a permission mask for objects.
+    chmod -R a+rX "${workdir}/repo-build/objects"
+
+    #bash -i
+
+    # Sync over to repo in $hostworkdir
+    ostree pull-local --repo="${hostworkdir}/repo/" "${workdir}/repo-build/" $ref || fatal
+    # Sync over to repo in $hostworkdir
+
+    # prune - we'll leave just one commit in repo
+    ostree prune --repo="${workdir}/repo-build/" --only-branch "${ref}" --depth=0 || fatal
+
+    #bash -i
+}
+
+fatal() {
+    echo "Failure from inside supermin VM"
+    touch "${workdir}/tmp/supermin-failure"
+    cleanup
+}
+
+cleanup() {
+    set +eu
+    # release disk space back to the host and unmount
+    fstrim -v "${workdir}"
+    sync
+    umount "${hostworkdir}"
+    umount "${workdir}"
+
+    # shutdown
+    reboot -f 
+}
+
+# run the compose
+compose
+# run the cleanup
+cleanup