diff --git a/src/cmd-build b/src/cmd-build index c460370108..5af9217bd3 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -29,11 +29,18 @@ EOF composejson=$(pwd)/tmp/compose.json changed_stamp=$(pwd)/tmp/treecompose.changed # --cache-only is here since `fetch` is a separate verb. -runcompose --cache-only --add-metadata-from-json ${commitmeta_input_json} \ - --touch-if-changed "${changed_stamp}" \ - --write-composejson-to ${composejson} -commit=$(ostree --repo=${workdir}/repo-build rev-parse "${ref}") -version=$(ostree --repo=${workdir}/repo-build show --print-metadata-key=version ${commit} | sed -e "s,',,g") +args=" --cache-only --add-metadata-from-json ${commitmeta_input_json}" +args+=" --touch-if-changed ${changed_stamp}" +args+=" --write-composejson-to ${composejson}" + +runcompose_in_vm ${previous_commit:-null} $ref $args + +# update summary file +ostree --repo=${workdir}/repo summary -u + +commit=$(ostree --repo=${workdir}/repo rev-parse "${ref}") +version=$(ostree --repo=${workdir}/repo show --print-metadata-key=version ${commit} | sed -e "s,',,g") + # Very special handling for --write-composejson-to as rpm-ostree doesn't # write it if the commit didn't change. if [ -f "${changed_stamp}" ]; then @@ -45,14 +52,6 @@ else # Grab the previous JSON cp -a --reflink=auto ${workdir}/tmp/compose-${commit}.json ${composejson} fi -# https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872 -# The passwd files (among others) don't have world readability. This won't -# actually corrupt the repository as the *canonical* permissions are stored -# as xattrs. Probably what we should do is have an ostree option to specify -# a permission mask for objects. -sudo chmod -R a+rX ${workdir}/repo-build/objects -ostree --repo=${workdir}/repo pull-local ${workdir}/repo-build "${ref}" -ostree --repo=${workdir}/repo summary -u sha256sum_str() { sha256sum | cut -f 1 -d ' ' diff --git a/src/cmd-init b/src/cmd-init index 4f16e83910..a7b59eac6a 100755 --- a/src/cmd-init +++ b/src/cmd-init @@ -102,8 +102,9 @@ mkdir -p installer fi ) -mkdir -p cache mkdir -p builds mkdir -p tmp ostree --repo=repo init --mode=archive -ostree --repo=repo-build init --mode=bare-user + +# prepare the unprivileged vm to run the compose in +prepare_vm diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 1293f5ec88..65678804c2 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -1,5 +1,13 @@ # Shared shell script library +# Global variables +export workdir=$(pwd) +export configdir=${workdir}/src/config +export manifest=${configdir}/manifest.yaml +export superminpreparedir="${workdir}/tmp/supermin-prepare.d" +export superminbuilddir="${workdir}/tmp/supermin-build.d" +export cachesimg="${workdir}/caches.qcow2" + fatal() { echo "error: $@" 1>&2; exit 1 } @@ -28,13 +36,13 @@ preflight() { fatal "Unable to find /dev/kvm" fi - if ! capsh --print | grep -q 'Current.*cap_sys_admin'; then - fatal "This container must currently be run with --privileged" - fi +# if ! capsh --print | grep -q 'Current.*cap_sys_admin'; then +# fatal "This container must currently be run with --privileged" +# fi - if ! sudo true; then - fatal "The user must currently have sudo privileges" - fi +# if ! sudo true; then +# fatal "The user must currently have sudo privileges" +# fi # permissions on /dev/kvm vary by (host) distro. If it's # not writable, recreate it. @@ -51,9 +59,12 @@ prepare_build() { fatal "No $(pwd)/repo found; did you run coreos-assembler init?" fi - export workdir=$(pwd) - export configdir=${workdir}/src/config - export manifest=${configdir}/manifest.yaml +# export workdir=$(pwd) +# export configdir=${workdir}/src/config +# export manifest=${configdir}/manifest.yaml +# export superminpreparedir="${workdir}/tmp/supermin-prepare.d" +# export superminbuilddir="${workdir}/tmp/supermin-build.d" +# export cachesimg="${workdir}/caches.qcow2" if ! [ -f "${manifest}" ]; then fatal "Failed to find ${manifest}" @@ -102,3 +113,81 @@ runcompose() { ${TREECOMPOSE_FLAGS:-} ${manifest} "$@" set +x } + +prepare_vm() { + # rpms to create appliance VM out of + rpms=' bash vim-minimal coreutils util-linux procps-ng kmod kernel-modules' + rpms+=' cifs-utils' # for samba + rpms+=' systemd' # for clean reboot + rpms+=' dhcp-client bind-export-libs iproute' # networking + rpms+=' rpm-ostree distribution-gpg-keys' # to run the compose + rpms+=' selinux-policy selinux-policy-targeted policycoreutils' #selinux + + # prepare appliance VM + supermin -v --prepare --use-installed $rpms -o "${superminpreparedir}" + supermin -v --build "${superminpreparedir}" \ + --include-packagelist --size 4G -f ext2 -o "${superminbuilddir}" + + # Create a disk image for our caches (pkgcache and bare-user build repo) + [ ! -d "${workdir}/tmp/emptydir" ] && mkdir "${workdir}/tmp/emptydir" + if [ ! -f "${cachesimg}" ]; then + virt-make-fs --format=qcow2 --type=xfs \ + --size=10G "${workdir}/tmp/emptydir" "${cachesimg}" + fi +} + +run_vm() { + init=$1 + set -x + + # Copy in the init script with the code we want to run + chmod +x "${init}" + virt-copy-in -a "${superminbuilddir}/root" "${init}" / + + # Execute unprivileged VM to run compose. Some notes: + # - smb="${workdir}" - we'll serve our workdir over samba + # - sda - rootfs from supermin build + # - sdb - caches.qcow2 - we store the bare-user repo and pkgcache here + # - discard=unmap - using virtio-scsi disks and discard=unmap so we can fstrim + # and recover disk space from the VM + qemu-kvm -nodefaults -nographic -m 2048 -no-reboot \ + -kernel "${superminbuilddir}/kernel" \ + -initrd "${superminbuilddir}/initrd" \ + -netdev user,id=eth0,hostname=supermin,smb="${workdir}",hostfwd=tcp:127.0.0.1:8000-:8000 \ + -device virtio-net-pci,netdev=eth0 \ + -device virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 \ + -drive if=none,id=drive-scsi0-0-0-0,snapshot=on,file="${superminbuilddir}/root" \ + -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=1 \ + -drive if=none,id=drive-scsi0-0-0-1,discard=unmap,file="${cachesimg}" \ + -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1 \ + -serial stdio -append "root=/dev/sda console=ttyS0 selinux=1 enforcing=0 autorelabel=1" + + if [ -f "${workdir}/tmp/supermin-failure" ]; then + rm -f "${workdir}/tmp/supermin-failure" + fatal "Detected failure from compose VM run" + fi +} + +runcompose_in_vm() { + previous_commit=$1 + ref=$2 + shift; shift; + + local treecompose_args="" + if ! grep -q '^# disable-unified-core' "${manifest}"; then + treecompose_args="${treecompose_args} --unified-core" + fi + + cmd="rpm-ostree compose tree --repo=${workdir}/repo-build" + cmd+=" --cachedir=${workdir}/cache ${treecompose_args}" + cmd+=" ${TREECOMPOSE_FLAGS:-} ${manifest} $@" + + # populate variables in init script + template=$(dirname $0)/supermin-init + sed -e "s|^workdir=|workdir=\"${workdir}\"|" \ + -e "s|^previous_commit=|previous_commit=\"${previous_commit}\"|" \ + -e "s|^ref=|ref=\"${ref}\"|" \ + -e "s|^cmd=|cmd=\"${cmd}\"|" $template > "${superminbuilddir}/init" + + run_vm "${superminbuilddir}/init" +} diff --git a/src/deps.txt b/src/deps.txt index 52ce40edd7..47ad715b6a 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -36,3 +36,6 @@ jq awscli # For ignition file validation in cmd-run ignition + +# For running supermin VM via qemu and sharing via samba +samba cifs-utils diff --git a/src/supermin-init b/src/supermin-init new file mode 100755 index 0000000000..34da06d56f --- /dev/null +++ b/src/supermin-init @@ -0,0 +1,119 @@ +#!/bin/sh +set -ux + +trap fatal SIGHUP SIGINT SIGQUIT SIGABRT + +export PATH=/usr/sbin/:$PATH + +mount -t proc /proc /proc +mount -t sysfs /sys /sys +mount -t devtmpfs devtmpfs /dev + +# Load selinux policy +LANG=C /sbin/load_policy -i + +# Load kernel module for 9pnet_virtio for 9pfs mount +#/sbin/modprobe 9pnet_virtio + +# Need fuse module for rofiles-fuse/bwrap during post scripts run +/sbin/modprobe fuse + +# set up networking +/usr/sbin/dhclient eth0 & +sleep 2 # wait for dhcp + + +# these varaibles will be populated in this script before running +workdir= +ref= +cmd= +previous_commit= + +# set a variable for our mountpoint for hostworkdir +hostworkdir='/hostworkdir' + +[ -d "${workdir}" ] || mkdir -p "${workdir}" +[ -d "${hostworkdir}" ] || mkdir -p "${hostworkdir}" + +# mount cache disk +mount /dev/sdb "${workdir}" + +#bash -i + +# mount host "${workdir}" via samba +#ount -t 9p -o rw,trans=virtio,version=9p2000.L srv /hostsrv/ +# cat /proc/fs/cifs/DebugData +mount -t cifs -o 'username=root,password=,cache=loose' '\\10.0.2.4\qemu' "${hostworkdir}" + +# symlink some things from $workdir to $hostworkdir +[ -e "${workdir}/tmp" ] || ln -s "${hostworkdir}/tmp" "${workdir}/tmp" +[ -e "${workdir}/src" ] || ln -s "${hostworkdir}/src" "${workdir}/src" +[ -e "${workdir}/localrepo" ] || ln -s "${hostworkdir}/localrepo" "${workdir}/localrepo" + +# create a few directories if they don't exist +[ -d "${workdir}/cache" ] || mkdir "${workdir}/cache" +[ -d "${workdir}/repo-build" ] || mkdir "${workdir}/repo-build" +[ -d "${workdir}/repo" ] || mkdir "${workdir}/repo" + +# initialize the repos if they don't exist +[ -f "${workdir}/repo-build/config" ] || ostree init --mode=bare-user --repo="${workdir}/repo-build/" +[ -f "${workdir}/repo/config" ] || ostree init --mode=archive --repo="${workdir}/repo/" + + +#bash -i + + +compose() { + # make sure previous_commit exists in the build repo + if [ "${previous_commit}" != "null" ]; then + pc=$(ostree --repo="${workdir}/repo-build" rev-parse "${ref}" || true) + if [ "${previous_commit}" != "${pc}" ]; then + # is it possible to use --commit-metadata-only here ? + ostree pull-local --repo="${workdir}/repo-build" "${hostworkdir}/repo/" "$ref" + fi + fi + + # run the command from given to us + $cmd || fatal + + # https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872 + # The passwd files (among others) don't have world readability. This won't + # actually corrupt the repository as the *canonical* permissions are stored + # as xattrs. Probably what we should do is have an ostree option to specify + # a permission mask for objects. + chmod -R a+rX "${workdir}/repo-build/objects" + + #bash -i + + # Sync over to repo in $hostworkdir + ostree pull-local --repo="${hostworkdir}/repo/" "${workdir}/repo-build/" $ref || fatal + # Sync over to repo in $hostworkdir + + # prune - we'll leave just one commit in repo + ostree prune --repo="${workdir}/repo-build/" --only-branch "${ref}" --depth=0 || fatal + + #bash -i +} + +fatal() { + echo "Failure from inside supermin VM" + touch "${workdir}/tmp/supermin-failure" + cleanup +} + +cleanup() { + set +eu + # release disk space back to the host and unmount + fstrim -v "${workdir}" + sync + umount "${hostworkdir}" + umount "${workdir}" + + # shutdown + reboot -f +} + +# run the compose +compose +# run the cleanup +cleanup