From 27e553dc33f43173bc37ca83fd7ac476ef809063 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 8 Jun 2020 17:27:00 -0400 Subject: [PATCH 1/4] travis: cache the chroot we use for package builds Prior to this change, we would debootstrap for every single integration test run in Travis. This changes that, so we will store the chroot created by debootstrap in Travis' cache, and use it if available. This saves 2-3 minutes, or ~1/3rd of the integration test run time (which is our longest run). --- .travis.yml | 70 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84f38533c16..b60ccb4682f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,18 +3,25 @@ dist: bionic # We use two different caching strategies. The default is to cache pip # packages (as most of our jobs use pip packages), which is configured here. -# For the integration tests, we instead want to cache the lxd images. The -# directory in which the images are stored (/var/snap/lxd/common/lxd/images/) -# is not readable/writeable by the default user (which is a requirement for -# caching), so we instead cache the `lxd_images/` directory. We move lxd -# images out of there before we run tests and back in once tests are complete. -# We _move_ the images out and only copy the most recent lxd image back into -# the cache, to avoid our cache growing without bound. (We only need the most -# recent lxd image because the integration tests only use a single image.) +# For the integration tests, we instead want to cache the lxd images and +# package build schroot. # # We cache the lxd images because this saves a few seconds in the general # case, but provides substantial speed-ups when cloud-images.ubuntu.com, the -# source of the images, is under heavy load. +# source of the images, is under heavy load. The directory in which the lxd +# images are stored (/var/snap/lxd/common/lxd/images/) is not +# readable/writeable by the default user (which is a requirement for caching), +# so we instead cache the `lxd_images/` directory. We move lxd images out of +# there before we run tests and back in once tests are complete. We _move_ the +# images out and only copy the most recent lxd image back into the cache, to +# avoid our cache growing without bound. (We only need the most recent lxd +# image because the integration tests only use a single image.) +# +# We cache the package build schroot because it saves 2-3 minutes per build. +# Without caching, we have to perform a debootstrap for every build. We update +# the schroot before storing it back in the cache, to ensure that we aren't +# just using an increasingly-old schroot as time passes. The cached schroot is +# stored as a tarball, to preserve permissions/ownership. cache: pip install: @@ -37,6 +44,7 @@ matrix: cache: - directories: - lxd_images + - chroots before_cache: - | # Find the most recent image file @@ -45,6 +53,14 @@ matrix: latest_file="$(basename $latest_file .rootfs)" # Find all files with that prefix and copy them to our cache dir sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} lxd_images/ \; + - | + # If a schroot exists (i.e. we didn't fail before its + # creation), tar it up (to preserve ownership/permissions) + # and move it into the cached dir; no need to compress it + # because Travis will do that anyway + if [ -e /var/lib/schroot/chroots/xenial-amd64 ]; then + sudo tar cf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 . + fi install: - git fetch --unshallow - sudo apt-get build-dep -y cloud-init @@ -68,8 +84,40 @@ matrix: script: # Ubuntu LTS: Build - ./packages/bddeb -S - # Use this to get a new shell where we're in the sbuild group - - sudo -E su $USER -c 'mk-sbuild xenial' + - | + if [ -e chroots/xenial-amd64.tar ]; then + # If we have a cached chroot, move it into place + sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64 + sudo tar xf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 + # Write its configuration + cat > sbuild-xenial-amd64 << EOM + [xenial-amd64] + description=xenial-amd64 + groups=sbuild,root,admin + root-groups=sbuild,root,admin + # Uncomment these lines to allow members of these groups to access + # the -source chroots directly (useful for automated updates, etc). + #source-root-users=sbuild,root,admin + #source-root-groups=sbuild,root,admin + type=directory + profile=sbuild + union-type=overlay + directory=/var/lib/schroot/chroots/xenial-amd64 + EOM + sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/ + sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64 + # And ensure it's up-to-date. (TODO: At the moment, even + # if there were no upgrades applied, because we tar up the + # schroot, Travis always detects changes and so + # recompresses/stores the cache every build. If we only + # replaced the cached tarball if upgrades occurred, we + # could save an extra ~30s on most builds.) + sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade" + else + # Otherwise, create the chroot + sudo -E su $USER -c 'mk-sbuild xenial' + fi + # Use sudo to get a new shell where we're in the sbuild group - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' # Ubuntu LTS: Integration - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb' From 6643c429f4aadd07a9becbfe14370eddc07d8dbd Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 11 Jun 2020 16:58:34 -0400 Subject: [PATCH 2/4] incorporate Ryan's comments to improve our tar calls --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b60ccb4682f..1feb0097207 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ matrix: # and move it into the cached dir; no need to compress it # because Travis will do that anyway if [ -e /var/lib/schroot/chroots/xenial-amd64 ]; then - sudo tar cf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 . + sudo tar --sparse --xattrs --xattrs-include=* cf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 . fi install: - git fetch --unshallow @@ -88,7 +88,7 @@ matrix: if [ -e chroots/xenial-amd64.tar ]; then # If we have a cached chroot, move it into place sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64 - sudo tar xf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 + sudo tar --sparse --xattrs --preserve-permissions --numeric-owner xf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 # Write its configuration cat > sbuild-xenial-amd64 << EOM [xenial-amd64] From 8be157ec188bf7c89c9d443fde2717ac661d9cf8 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 12 Jun 2020 09:38:54 -0400 Subject: [PATCH 3/4] use TRAVIS_BUILD_DIR to make chroot caching more robust --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1feb0097207..5948a90e8ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ matrix: # and move it into the cached dir; no need to compress it # because Travis will do that anyway if [ -e /var/lib/schroot/chroots/xenial-amd64 ]; then - sudo tar --sparse --xattrs --xattrs-include=* cf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 . + sudo tar --sparse --xattrs --xattrs-include=* cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 . fi install: - git fetch --unshallow @@ -85,10 +85,10 @@ matrix: # Ubuntu LTS: Build - ./packages/bddeb -S - | - if [ -e chroots/xenial-amd64.tar ]; then + if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then # If we have a cached chroot, move it into place sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64 - sudo tar --sparse --xattrs --preserve-permissions --numeric-owner xf chroots/xenial-amd64.tar -C /var/lib/schroot/chroots/xenial-amd64 + sudo tar --sparse --xattrs --preserve-permissions --numeric-owner xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 # Write its configuration cat > sbuild-xenial-amd64 << EOM [xenial-amd64] From fa1d414b0764c83b910fb9390945baf9a493fcc1 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 12 Jun 2020 09:50:46 -0400 Subject: [PATCH 4/4] fix tar options --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5948a90e8ff..b600eccdea0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ matrix: # and move it into the cached dir; no need to compress it # because Travis will do that anyway if [ -e /var/lib/schroot/chroots/xenial-amd64 ]; then - sudo tar --sparse --xattrs --xattrs-include=* cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 . + sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 . fi install: - git fetch --unshallow @@ -88,7 +88,7 @@ matrix: if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then # If we have a cached chroot, move it into place sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64 - sudo tar --sparse --xattrs --preserve-permissions --numeric-owner xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 + sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 # Write its configuration cat > sbuild-xenial-amd64 << EOM [xenial-amd64]