Speed up builds by caching /tmp/wheelhouse directory, speed up EL 8 build step by not running slow Requires / Provides Python generation step

.circle/buildenv_st2.sh

@@ -56,5 +56,8 @@ re="\\b$DISTRO\\b"
 ST2_CIRCLE_URL=${CIRCLE_BUILD_URL}
 
 write_env ST2_GITURL ST2_GITREV ST2PKG_VERSION ST2PKG_RELEASE DISTRO TESTING ST2_CIRCLE_URL
-
 cat ~/.buildenv
+
+# Workaround for cache key since we can't reference arbitrary environment variables for cache keys
+echo "${DISTRO}" > /tmp/distro-version
+echo "${ST2PKG_VERSION}" > /tmp/st2-version

.circleci/config.yml

@@ -31,7 +31,7 @@ jobs:
           exclusive: true   # default - true
           # Temporary workaround for Circle CI issue
           # https://discuss.circleci.com/t/setup-remote-docker-connection-failures/26434
-          version: 18.05.0-ce 
+          version: 18.05.0-ce
       - run:
           name: Ensure installation scripts are synced with their templates
           command: make .generated-files-check
@@ -70,21 +70,57 @@ jobs:
       - run:
           name: Pull dependent Docker Images
           command: .circle/docker-compose2.sh pull ${DISTRO}
+      - restore_cache:
+          name: Restore Wheelhouse Cache
+          keys:
+            - wheelhouse-{{ checksum "/tmp/distro-version" }}-{{ checksum "/tmp/st2-version" }}
+            - wheelhouse-
       - run:
           name: Build the ${DISTRO} Packages
           command: |
+            # Create necessary directories
+            # TODO: Per distro directory?
+            mkdir -p ~/st2-packages/build/${DISTRO}/log/
+            mkdir -p ~/st2-packages/wheelhouse/
+
+            # List volume content to see if there is any cached dasta
+            # Uncomment if you are troubleshooting wheelhouse caching issues
+            # echo "local wheelhouse cache directory content"
+            # echo ""
+            # ls -la ~/st2-packages/wheelhouse/
+            # du -hs ~/st2-packages/wheelhouse/
+            # echo ""
+
+            # Copy over cached wheelhouse packages (if any exist)
+            docker cp ~/st2-packages/wheelhouse/. st2-packages-vol:/tmp/wheelhouse
+
+            # List volume content to see if there is any cached data
+            # echo "docker volume wheelhouse cache directory content"
+            # echo ""
+            # docker run --rm -i -v=st2-packages-vol:/tmp/wheelhouse busybox find /tmp/wheelhouse
+            # echo ""
+
+            # Run the build
             .circle/docker-compose2.sh build ${DISTRO}
+
             # Once build container finishes we can copy packages directly from it
-            mkdir -p ~/st2-packages/build/${DISTRO}/log/
+
+            # Copy built packages
             docker cp st2-packages-vol:/root/build/. ~/st2-packages/build/${DISTRO}
-#      # TODO: It works! (~0.5-1min speed-up) Enable CircleCI2.0 cache for pip and wheelhouse later
-#      - run:
-#          name: Build the ${DISTRO} Packages 2nd time (compare with pip/wheelhouse cached)
-#          command: |
-#            .circle/docker-compose2.sh build ${DISTRO}
-#            # Once build container finishes we can copy packages directly from it
-#            docker cp st2-packages-vol:/root/build /tmp/st2-packages
-#          working_directory: ~/st2-packages
+
+            # Copy all the files from /tmp/wheelhouse so we can cache it and substantially speed
+            # up subsequent builds
+            docker cp st2-packages-vol:/tmp/wheelhouse/. ~/st2-packages/wheelhouse/
+
+            # echo "wheelhouse cache directory content"
+            # Uncomment if you are troubleshooting wheelhouse caching issues
+            # ls -la ~/st2-packages/wheelhouse/
+            # du -hs ~/st2-packages/wheelhouse/
+      - save_cache:
+          name: Store Wheelhouse Cache
+          key: wheelhouse-{{ checksum "/tmp/distro-version" }}-{{ checksum "/tmp/st2-version" }}
+          paths:
+            - ~/st2-packages/wheelhouse/
       - run:
           name: Test the Packages
           command: .circle/docker-compose2.sh test ${DISTRO}

packages/st2/component.makefile

@@ -85,7 +85,7 @@ bdist_wheel: .stamp-bdist_wheel
 # We need to install these python packages to handle rpmbuild 4.14 in EL8
 ifeq ($(EL_VERSION),8)
 	$(PIP_BINARY) install wheel setuptools virtualenv
-	$(PIP_BINARY) install cryptography --no-binary cryptography
+	$(PIP_BINARY) install cryptography
 endif
 	$(PYTHON_BINARY) setup.py bdist_wheel -d $(WHEELDIR) || \
 		$(PYTHON_BINARY) setup.py bdist_wheel -d $(WHEELDIR)

packages/st2/rpm/st2.spec

@@ -10,7 +10,7 @@
 Epoch: %{epoch}
 %endif
 
-%if 0%{?rhel} >= 8
+%if 0%{?rhel} == 8
 %global _build_id_links none
 %endif
 
@@ -19,12 +19,30 @@ Requires: python3-devel, openssl-devel, libffi-devel, git, pam, openssh-server,
 # EL8 requires a few python packages available within 'BUILDROOT' when outside venv
 # These are in the el8 packagingbuild dockerfile
 # Reference https://fossies.org/linux/ansible/packaging/rpm/ansible.spec
-%if 0%{?rhel} >= 8
+%if 0%{?rhel} == 8
 # Will use the python3 stdlib venv
 BuildRequires: python3-devel
 BuildRequires: python3-setuptools
 %endif  # Requires for RHEL 8
 
+%if 0%{?rhel} == 8
+# By default on EL 8, RPM helper scripts will try to generate Requires: section which lists every
+# Python dependencies. That process / script works by recursively scanning all the package Python
+# dependencies which is very slow (5-6 minutes).
+# Our package bundles virtualenv with all the dependendencies and doesn't rely on this metadata
+# so we skip that step to vastly speed up the build.
+# Technically we also don't Require or Provide any of those libraries auto-detected by that script
+# because those are only used internally inside a package specific virtual environment.
+# Same step also does not run on EL7.
+# See https://github.com/StackStorm/st2-packages/pull/697#issuecomment-808971874 and that PR for
+# more details.
+# That issue was found by enabling rpmbuild -vv flag.
+%undefine __pythondist_provides
+%undefine __pythondist_requires
+%undefine __python_provides
+%undefine __python_requires
+%endif
+
 Summary: StackStorm all components bundle
 Conflicts: st2common
 

rake/formatter.rb

@@ -89,7 +89,7 @@ def command_headers(command)
         ShellOut::HEADERS_LIST.inject('') do |result, k|
           color = command.failure? ? [:red, :bold] : Array(ShellOut::HEADERS_COLORS[k])
           header = send(:"header_#{k}", command) if respond_to?(:"header_#{k}", true)
-          header ? result << '[%s]' % colorize(header, *color) : result
+          header ? result << '[%s] [%s]' % [colorize(header, *color), Time.at(Time.new).utc.strftime("%H:%M:%S")] : result
         end
       end
 

rpmspec/package_venv.spec

@@ -12,7 +12,7 @@
 %define pin_pip %{venv_python} %{venv_bin}/pip3 install pip==20.3.3
 %define install_venvctrl python3 -m pip install venvctrl
 %if 0%{?rhel} == 8
-%define install_crypto %{venv_python} %{venv_bin}/pip install cryptography==2.8 --no-binary cryptography
+%define install_crypto %{venv_python} %{venv_bin}/pip install cryptography==2.8
 %else
 %define install_crypto %{nil}
 %endif

scripts/build_os_package.sh

@@ -13,10 +13,16 @@ platform() {
   echo 'rpm'
 }
 
+# NOTE: If you want to troubleshoot rpmbuild, add -vv flag to enable debug mode
 build_rpm() { rpmbuild -bb --define '_topdir %(readlink -f build)' rpm/"$package_name".spec; }
 build_deb() { dpkg-buildpackage -b -uc -us -j"$cores_num"; }
 
-copy_rpm() { sudo cp -v build/RPMS/*/$1*.rpm "$artifact_dir"; }
+copy_rpm() {
+    sudo cp -v build/RPMS/*/$1*.rpm "$artifact_dir";
+    # Also print some package info for easier troubleshooting
+    rpm -q --requires -p build/RPMS/*/$1*.rpm
+    rpm -q --provides -p build/RPMS/*/$1*.rpm
+}
 copy_deb() {
   sudo cp -v ../"$package_name"*.deb "$artifact_dir" || { echo "Failed to copy .deb file into artifact directory \`$artifact_dir'" ; exit 1; }
   sudo cp -v ../"$package_name"{*.changes,*.dsc} "$artifact_dir" || :;