feat: buildkit volume caching (experimental)

AGENTS.md

@@ -0,0 +1,10 @@
+# setup-earthbuild GitHub Action
+
+## Tooling
+
+The primary development tool is "earth". You can use it to build and test the action.
+
+* `earth +compile` - Build the action.
+* `earth +test` - Run the tests.
+* `earth +all` - Build, lint, and test the action.
+* `earth doc` - List all targets.

action.yml

@@ -16,6 +16,10 @@ inputs:
     default: 'true'
   github-token:
     description: 'GitHub token for fetching EarthBuild version list.'
+  experimental-buildkit-volume-cache:
+    description: 'whether to use the experimental buildkit volume caching'
+    default: 'false'
+
 runs:
   using: node24
   main: dist/setup/index.js

dist/cache-save/index.js

@@ -38911,8 +38911,9 @@ var __webpack_exports__ = {};
 
 // EXPORTS
 __nccwpck_require__.d(__webpack_exports__, {
-  "$": () => (/* binding */ cacheBinary),
-  "K": () => (/* binding */ run)
+  "$H": () => (/* binding */ cacheBinary),
+  "KH": () => (/* binding */ run),
+  "bS": () => (/* binding */ saveBuildkitCache)
 });
 
 // NAMESPACE OBJECT: ./node_modules/@azure/storage-blob/dist/esm/generated/src/models/mappers.js
@@ -39100,9 +39101,6 @@ __nccwpck_require__.d(mappers_namespaceObject, {
   "UserDelegationKey": () => (UserDelegationKey)
 });
 
-;// CONCATENATED MODULE: external "fs"
-const external_fs_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("fs");
-var external_fs_default = /*#__PURE__*/__nccwpck_require__.n(external_fs_namespaceObject);
 // EXTERNAL MODULE: external "os"
 var external_os_ = __nccwpck_require__(2037);
 ;// CONCATENATED MODULE: ./node_modules/@actions/core/lib/utils.js
@@ -39235,6 +39233,9 @@ function escapeProperty(s) {
 //# sourceMappingURL=command.js.map
 ;// CONCATENATED MODULE: external "crypto"
 const external_crypto_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("crypto");
+;// CONCATENATED MODULE: external "fs"
+const external_fs_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("fs");
+var external_fs_default = /*#__PURE__*/__nccwpck_require__.n(external_fs_namespaceObject);
 ;// CONCATENATED MODULE: ./node_modules/@actions/core/lib/file-command.js
 // For internal use, subject to change.
 // We use any as a valid input type
@@ -88102,18 +88103,6 @@ function saveCacheV2(paths_1, key_1, options_1) {
     });
 }
 //# sourceMappingURL=cache.js.map
-;// CONCATENATED MODULE: ./src/constants.ts
-var State;
-(function (State) {
-    State["CachePrimaryKey"] = "EARTHBUILD_CACHE_KEY";
-    State["CacheMatchedKey"] = "EARTHBUILD_CACHE_RESULT";
-    State["BinaryPath"] = "EARTHBUILD_BINARY_PATH";
-})(State || (State = {}));
-var Outputs;
-(function (Outputs) {
-    Outputs["CacheHit"] = "cache-hit";
-})(Outputs || (Outputs = {}));
-
 ;// CONCATENATED MODULE: ./src/cache-utils.ts
 
 
@@ -88134,12 +88123,28 @@ function isCacheFeatureAvailable() {
     return true;
 }
 
+;// CONCATENATED MODULE: ./src/constants.ts
+var State;
+(function (State) {
+    State["CachePrimaryKey"] = "EARTHBUILD_CACHE_KEY";
+    State["CacheMatchedKey"] = "EARTHBUILD_CACHE_RESULT";
+    State["BinaryPath"] = "EARTHBUILD_BINARY_PATH";
+    State["BuildkitCacheMatchedKey"] = "BUILDKIT_CACHE_RESULT";
+})(State || (State = {}));
+var Outputs;
+(function (Outputs) {
+    Outputs["CacheHit"] = "cache-hit";
+})(Outputs || (Outputs = {}));
+
 ;// CONCATENATED MODULE: ./src/cache-save.ts
 
 
 
 
 
+
+
+
 // Catch and log any unhandled exceptions.  These exceptions can leak out of the uploadChunk method in
 // @actions/toolkit when a failed upload closes the file descriptor causing any in-process reads to
 // throw an uncaught exception.  Instead of failing this action, just warn.
@@ -88150,6 +88155,7 @@ process.on('uncaughtException', (e) => {
 async function run() {
     try {
         await cacheBinary();
+        await saveBuildkitCache();
     }
     catch (error) {
         if (error instanceof Error) {
@@ -88201,12 +88207,63 @@ const cacheBinary = async () => {
         }
     }
 };
+const saveBuildkitCache = async () => {
+    if (!isCacheFeatureAvailable()) {
+        return;
+    }
+    const useBuildkitCache = getInput('experimental-buildkit-volume-cache') === 'true';
+    if (!useBuildkitCache) {
+        return;
+    }
+    const state = getState(State.BuildkitCacheMatchedKey);
+    const primaryKey = `earth-volume-cache-'}`;
+    const containerName = process.env.EARTHLY_INSTALLATION_NAME || 'earth-buildkitd';
+    const volumeName = 'earth-cache';
+    if (primaryKey === state) {
+        info(`Buildkit cache hit occurred on the primary key ${primaryKey}, not saving cache.`);
+        return;
+    }
+    try {
+        info(`Stopping buildkit container ${containerName}...`);
+        await exec_exec('docker', ['stop', containerName], { ignoreReturnCode: true });
+        const cacheFile = external_path_.join(process.env.RUNNER_TEMP || external_os_.tmpdir(), 'earthbuild-buildkit-cache.tar.zst');
+        try {
+            const volumePath = `/var/lib/docker/volumes/${volumeName}/_data`;
+            info(`Compressing buildkit volume ${volumePath} to ${cacheFile}...`);
+            await exec_exec('sudo', ['tar', '-c', '--use-compress-program=zstd -T0', '-f', cacheFile, '-C', volumePath, '.']);
+            await exec_exec('sudo', ['chmod', '666', cacheFile]);
+            await cache_saveCache([cacheFile], primaryKey);
+            info(`Buildkit cache saved with the key: ${primaryKey}`);
+        }
+        finally {
+            await external_fs_default().promises.rm(cacheFile, { force: true });
+        }
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            if (error.name === ValidationError.name) {
+                throw error;
+            }
+            else if (error.name === ReserveCacheError.name) {
+                info(error.message);
+            }
+            else {
+                warning(error.message);
+            }
+        }
+        else {
+            core_error(`unknown error encountered saving buildkit cache: ${String(error)}`);
+            throw error;
+        }
+    }
+};
 void run();
 
 })();
 
-var __webpack_exports__cacheBinary = __webpack_exports__.$;
-var __webpack_exports__run = __webpack_exports__.K;
-export { __webpack_exports__cacheBinary as cacheBinary, __webpack_exports__run as run };
+var __webpack_exports__cacheBinary = __webpack_exports__.$H;
+var __webpack_exports__run = __webpack_exports__.KH;
+var __webpack_exports__saveBuildkitCache = __webpack_exports__.bS;
+export { __webpack_exports__cacheBinary as cacheBinary, __webpack_exports__run as run, __webpack_exports__saveBuildkitCache as saveBuildkitCache };
 
 //# sourceMappingURL=index.js.map

dist/setup/index.js

@@ -39007,6 +39007,18 @@ module.exports = JSON.parse('{"name":"@actions/cache","version":"6.0.0","descrip
 /******/ }
 /******/ 
 /************************************************************************/
+/******/ /* webpack/runtime/compat get default export */
+/******/ (() => {
+/******/ 	// getDefaultExport function for compatibility with non-harmony modules
+/******/ 	__nccwpck_require__.n = (module) => {
+/******/ 		var getter = module && module.__esModule ?
+/******/ 			() => (module['default']) :
+/******/ 			() => (module);
+/******/ 		__nccwpck_require__.d(getter, { a: getter });
+/******/ 		return getter;
+/******/ 	};
+/******/ })();
+/******/ 
 /******/ /* webpack/runtime/create fake namespace object */
 /******/ (() => {
 /******/ 	var getProto = Object.getPrototypeOf ? (obj) => (Object.getPrototypeOf(obj)) : (obj) => (obj.__proto__);
@@ -39261,6 +39273,7 @@ __nccwpck_require__.d(mappers_namespaceObject, {
 
 ;// CONCATENATED MODULE: external "node:fs/promises"
 const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:fs/promises");
+var promises_default = /*#__PURE__*/__nccwpck_require__.n(promises_namespaceObject);
 // EXTERNAL MODULE: external "os"
 var external_os_ = __nccwpck_require__(2037);
 // EXTERNAL MODULE: external "path"
@@ -89057,6 +89070,7 @@ var State;
     State["CachePrimaryKey"] = "EARTHBUILD_CACHE_KEY";
     State["CacheMatchedKey"] = "EARTHBUILD_CACHE_RESULT";
     State["BinaryPath"] = "EARTHBUILD_BINARY_PATH";
+    State["BuildkitCacheMatchedKey"] = "BUILDKIT_CACHE_RESULT";
 })(State || (State = {}));
 var Outputs;
 (function (Outputs) {
@@ -89088,6 +89102,10 @@ function isCacheFeatureAvailable() {
 
 
 
+
+
+
+
 const cache_restore_restoreCache = async (path, version) => {
     if (!isCacheFeatureAvailable()) {
         return false;
@@ -89108,6 +89126,43 @@ const cache_restore_restoreCache = async (path, version) => {
     info(`Cache restored from key: ${cacheKey}`);
     return Boolean(cacheKey);
 };
+const restoreBuildkitCache = async () => {
+    if (!isCacheFeatureAvailable()) {
+        return false;
+    }
+    const useBuildkitCache = getInput('experimental-buildkit-volume-cache') === 'true';
+    if (!useBuildkitCache) {
+        return false;
+    }
+    const cacheKeyInput = `earth-volume-cache-${process.env.GITHUB_SHA || 'unknown'}`;
+    const restoreKeys = ['earth-volume-cache-'];
+    const volumeName = 'earth-cache';
+    const cacheFile = external_path_.join(process.env.RUNNER_TEMP || external_os_.tmpdir(), 'earthbuild-buildkit-cache.tar.zst');
+    try {
+        const cacheKey = await restoreCache([cacheFile], cacheKeyInput, restoreKeys);
+        if (!cacheKey) {
+            info('EarthBuild buildkit volume cache not found');
+            return false;
+        }
+        saveState(State.BuildkitCacheMatchedKey, cacheKey);
+        info(`Buildkit cache restored from key: ${cacheKey}`);
+        const volumePath = `/var/lib/docker/volumes/${volumeName}/_data`;
+        info(`Extracting cache to ${volumePath}`);
+        await exec_exec('sudo', ['mkdir', '-p', volumePath]);
+        try {
+            await promises_default().access(cacheFile);
+            await exec_exec('sudo', ['tar', '-xf', cacheFile, '-C', volumePath]);
+            info('Buildkit cache successfully extracted');
+        }
+        catch (err) {
+            warning(`Failed to extract buildkit cache: ${String(err)}`);
+        }
+        return true;
+    }
+    finally {
+        await promises_default().rm(cacheFile, { force: true });
+    }
+};
 
 ;// CONCATENATED MODULE: ./node_modules/universal-user-agent/index.js
 function getUserAgent() {
@@ -90930,13 +90985,15 @@ async function run() {
         if (toolcacheDir) {
             addPath(toolcacheDir);
             info(`using earthbuild from toolcache (${toolcacheDir})`);
+            await restoreBuildkitCache();
             return;
         }
         // then try to restore earthbuild from the github action cache
         addPath(installationDir);
         const restored = await cache_restore_restoreCache(installationPath, node_modules_semver.clean(tag_name) || tag_name.substring(1));
         if (restored) {
             await promises_namespaceObject.chmod(installationPath, 0o755);
+            await restoreBuildkitCache();
             return;
         }
         // finally, dowload EarthBuild release binary
@@ -90948,6 +91005,7 @@ async function run() {
         core_debug(`successfully downloaded ${buildURL} to ${downloaded}`);
         await promises_namespaceObject.chmod(installationPath, 0o755);
         await cacheDir(external_path_.join(destination, 'bin'), pkgName, node_modules_semver.clean(tag_name) || tag_name.substring(1), external_os_.arch());
+        await restoreBuildkitCache();
     }
     catch (error) {
         if (error instanceof Error) {