From 0e507d31b95b481a3653f512e51cb7cef2545eb7 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 2 Apr 2025 18:30:17 -0700 Subject: [PATCH 01/14] Fix a build error in release.yml that was caused by renaming --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7f1823..ed6fcdf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,12 +78,12 @@ jobs: # Add any other essential files here echo "Copying built artifacts..." - ACTIONS=("build-docker-image" "run-build-script-in-docker" "setup-cmake" "setup-vcpkg") + ACTIONS=("build-docker-image" "run-build-script-in-docker" "setup-build-tools") BUILD_DIR_RELATIVE="build" # Relative to MAIN_REPO_PATH for action_name in "${ACTIONS[@]}"; do echo "Processing action: $action_name" - DEST_ACTION_DIR="actions/$action_name" + DEST_ACTION_DIR="$action_name" DEST_DIST_DIR="$DEST_ACTION_DIR/dist" SRC_ACTION_PATH="$MAIN_REPO_PATH/$BUILD_DIR_RELATIVE/$action_name" From cc62460f760a752c2b92e90b55098728ad680ce0 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 4 Apr 2025 16:25:58 -0700 Subject: [PATCH 02/14] Create build-and-prep-ort-files and build-minimal-ort-and-run-tests --- .prettierignore | 10 + .prettierrc.json | 8 + actions/build-and-prep-ort-files/action.yml | 5 + .../action.yml | 28 + actions/setup-build-tools/README.md | 60 + package-lock.json | 1367 ++++++++++++++++- package.json | 1 + src/build-and-prep-ort-files/index.js | 228 +++ src/build-minimal-ort-and-run-tests/index.js | 232 +++ 9 files changed, 1909 insertions(+), 30 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc.json create mode 100644 actions/build-and-prep-ort-files/action.yml create mode 100644 actions/build-minimal-ort-and-run-tests/action.yml create mode 100644 actions/setup-build-tools/README.md create mode 100644 src/build-and-prep-ort-files/index.js create mode 100644 src/build-minimal-ort-and-run-tests/index.js diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..ac8f96c --- /dev/null +++ b/.prettierignore @@ -0,0 +1,10 @@ +# Ignore build output and coverage reports +build/ +coverage/ + +# Ignore test C++ code if any +test/cpp/ + +# Ignore lock files (though Prettier usually skips them anyway) +package-lock.json +yarn.lock \ No newline at end of file diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..7ef273f --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,8 @@ +{ + "semi": true, + "singleQuote": true, + "trailingComma": "es5", + "tabWidth": 2, + "printWidth": 120, + "arrowParens": "always" +} diff --git a/actions/build-and-prep-ort-files/action.yml b/actions/build-and-prep-ort-files/action.yml new file mode 100644 index 0000000..36cd9dc --- /dev/null +++ b/actions/build-and-prep-ort-files/action.yml @@ -0,0 +1,5 @@ +name: 'Build Full ORT and Prepare Test Files' +description: 'Installs requirements, builds full ORT wheel, installs it, and generates ORT format models and config files for minimal build tests in $RUNNER_TEMP.' +runs: + using: 'node20' + main: 'dist/index.js' \ No newline at end of file diff --git a/actions/build-minimal-ort-and-run-tests/action.yml b/actions/build-minimal-ort-and-run-tests/action.yml new file mode 100644 index 0000000..908d528 --- /dev/null +++ b/actions/build-minimal-ort-and-run-tests/action.yml @@ -0,0 +1,28 @@ +name: 'Build Minimal ORT and Run Tests' +description: 'Downloads test_data artifact, builds minimal ORT based on config, runs tests, and uploads binary size report.' +inputs: + reduced-ops-config-file: + description: 'Path (relative to downloaded test_data artifact) to the reduced Ops config file (e.g., required_ops.ort_models.config).' + required: true + enable-type-reduction: + description: 'Build with type reduction enabled.' + required: false + default: 'false' # String 'false' for getBooleanInput + enable-custom-ops: + description: 'Build with custom op support enabled.' + required: false + default: 'false' # String 'false' for getBooleanInput + skip-model-tests: + description: 'Skip running the E2E model tests with onnx_test_runner.' + required: false + default: 'false' # String 'false' for getBooleanInput + binary-size-report-name-prefix: + description: 'Optional prefix for the uploaded binary size artifact name.' + required: false + default: '' + size-threshold: + description: 'Optional threshold in bytes passed to check_build_binary_size.py.' + required: false +runs: + using: 'node20' + main: 'dist/index.js' \ No newline at end of file diff --git a/actions/setup-build-tools/README.md b/actions/setup-build-tools/README.md new file mode 100644 index 0000000..64eccd6 --- /dev/null +++ b/actions/setup-build-tools/README.md @@ -0,0 +1,60 @@ +# Setup Build Tools (CMake & vcpkg) Action + +## Description + +This GitHub Action streamlines the setup of essential C/C++ build tools by downloading, verifying, extracting, and caching specified versions of both **CMake** and **vcpkg**. + +It performs the setup for CMake first, followed by vcpkg. It leverages `@actions/tool-cache` for efficiency across workflow runs. Additionally, it can automatically add the required CMake version to the system `PATH` and sets the `VCPKG_INSTALLATION_ROOT` environment variable for use in subsequent build steps. + +This action combines and replaces the functionality of the previous separate `setup-cmake` and `setup-vcpkg` actions. + +## Features + +- Installs specific versions of CMake (including `latest`) and vcpkg (by tag). +- Supports Windows, Linux, and macOS runners. +- Verifies downloads using provided SHA512 hashes (optional for CMake, required for vcpkg). +- Integrates with GitHub Actions tool cache (`@actions/tool-cache`). +- Optionally uses the `TerrapinRetrievalTool` for accelerated and verified downloads on Windows (when hashes are provided). +- Optionally adds the installed CMake `bin` directory to the `PATH`. +- Sets the `VCPKG_INSTALLATION_ROOT` environment variable. + +## Inputs + +| Input | Description | Required | Default | +| :------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------: | :-------------------------------------------- | +| `cmake-version` | The CMake version to download (e.g., `3.29.0`) or the string `"latest"` to fetch the newest release via GitHub API. | `true` | - | +| `cmake-hash` | **Optional.** The expected SHA512 hash (hex) of the CMake archive for the target platform/architecture. Required for download verification and enabling Terrapin usage for CMake. | `false` | - | +| `add-cmake-to-path` | If `'true'`, adds the `bin` directory of the installed CMake version to the `PATH` environment variable. | `false` | `'true'` | +| `vcpkg-version` | The vcpkg tag version to download (e.g., `2023.10.19`). Find tags on the [vcpkg releases page](https://github.com/microsoft/vcpkg/tags). | `true` | - | +| `vcpkg-hash` | The expected SHA512 hash (hex) for the specified vcpkg tag's `.zip` archive. Required for download verification and enabling Terrapin usage for vcpkg. | `true` | - | +| `terrapin-tool-path` | Path to the `TerrapinRetrievalTool.exe` executable. Used for both CMake and vcpkg downloads on Windows if applicable (hashes provided, Terrapin not disabled). | `false` | `C:/local/Terrapin/TerrapinRetrievalTool.exe` | +| `disable-terrapin` | If set to `'true'`, Terrapin usage will be bypassed for both CMake and vcpkg, forcing direct downloads via `@actions/tool-cache`. | `false` | `'false'` | +| `github-token` | GitHub token used for fetching the `"latest"` CMake version via the GitHub API. Defaults to the workflow's token. | `false` | `${{ github.token }}` | + +**Finding Hashes:** You typically need to download the specific release archive (e.g., CMake `.tar.gz`/`.zip` or vcpkg `.zip` for a tag) and calculate its SHA512 hash locally using tools like `sha512sum` (Linux/macOS) or `Get-FileHash -Algorithm SHA512` (PowerShell). + +**⚠️ SECURITY WARNING:** Omitting the `cmake-hash` significantly increases the risk of supply chain attacks, as the integrity of the downloaded CMake artifact will not be checked, and Terrapin cannot be used for CMake. It is **strongly recommended** to provide `cmake-hash` whenever possible, especially for production workflows. Pinning to a specific `cmake-version` and providing its corresponding hash is the most secure approach. The `vcpkg-hash` is **required**. + +## Outputs + +| Output | Description | +| :----------- | :----------------------------------------------------------------------------------------------------- | +| `cmake-root` | The absolute path to the root directory of the cached CMake installation. | +| `cmake-path` | The absolute path to the directory containing the CMake executables (e.g., `.../cmake-/bin`). | +| `vcpkg-root` | The absolute path to the root directory of the cached and bootstrapped vcpkg instance. | + +## Environment Variables + +This action sets the following environment variable for subsequent steps in the job: + +- **`VCPKG_INSTALLATION_ROOT`**: The absolute path to the cached vcpkg installation directory (same value as the `vcpkg-root` output). This is commonly used by build systems like CMake (`-DCMAKE_TOOLCHAIN_FILE=$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake`). + +Additionally, if `add-cmake-to-path` is `'true'` (the default), the action adds the path specified by the `cmake-path` output to the system `PATH`. + +## Caching + +- The action utilizes `@actions/tool-cache` for caching both CMake and vcpkg installations. +- The **CMake** cache key is based on the tool name (`cmake`), the resolved `cmake-version`, and the runner's platform/architecture (e.g., `linux-x86_64`). +- The **vcpkg** cache key is based on the tool name (`vcpkg`) and the `vcpkg-version`. +- It caches the **bootstrapped** version of vcpkg, meaning the `vcpkg` executable should be present and ready to use within the cached directory. +- Subsequent workflow runs hitting the cache will be significantly faster as they skip download, verification, extraction, and bootstrapping (for vcpkg). diff --git a/package-lock.json b/package-lock.json index 592383e..5b90301 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,13 @@ "version": "0.0.2", "license": "MIT", "dependencies": { + "@actions/artifact": "^2.3.2", "@actions/core": "^1.11.1", "@actions/exec": "^1.1.1", "@actions/github": "^6.0.0", "@actions/glob": "^0.5.0", - "@actions/tool-cache": "^2.0.2" + "@actions/tool-cache": "^2.0.2", + "elfy": "^1.0.0" }, "devDependencies": { "@eslint/eslintrc": "^3.3.1", @@ -33,6 +35,160 @@ "node": ">=20" } }, + "node_modules/@actions/artifact": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@actions/artifact/-/artifact-2.3.2.tgz", + "integrity": "sha512-uX2Mr5KEPcwnzqa0Og9wOTEKIae6C/yx9P/m8bIglzCS5nZDkcQC/zRWjjoEsyVecL6oQpBx5BuqQj/yuVm0gw==", + "license": "MIT", + "dependencies": { + "@actions/core": "^1.10.0", + "@actions/github": "^5.1.1", + "@actions/http-client": "^2.1.0", + "@azure/storage-blob": "^12.15.0", + "@octokit/core": "^3.5.1", + "@octokit/plugin-request-log": "^1.0.4", + "@octokit/plugin-retry": "^3.0.9", + "@octokit/request-error": "^5.0.0", + "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "archiver": "^7.0.1", + "jwt-decode": "^3.1.2", + "unzip-stream": "^0.3.1" + } + }, + "node_modules/@actions/artifact/node_modules/@actions/github": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.1.1.tgz", + "integrity": "sha512-Nk59rMDoJaV+mHCOJPXuvB1zIbomlKS0dmSIqPGxd0enAXBnOfn4VWF+CGtRCwXZG9Epa54tZA7VIRlJDS8A6g==", + "license": "MIT", + "dependencies": { + "@actions/http-client": "^2.0.1", + "@octokit/core": "^3.6.0", + "@octokit/plugin-paginate-rest": "^2.17.0", + "@octokit/plugin-rest-endpoint-methods": "^5.13.0" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/auth-token": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", + "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.0.3" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/core": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", + "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", + "license": "MIT", + "dependencies": { + "@octokit/auth-token": "^2.4.4", + "@octokit/graphql": "^4.5.8", + "@octokit/request": "^5.6.3", + "@octokit/request-error": "^2.0.5", + "@octokit/types": "^6.0.3", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/core/node_modules/@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/endpoint": { + "version": "6.0.12", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", + "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.0.3", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/graphql": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", + "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", + "license": "MIT", + "dependencies": { + "@octokit/request": "^5.6.0", + "@octokit/types": "^6.0.3", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/openapi-types": { + "version": "12.11.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", + "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==", + "license": "MIT" + }, + "node_modules/@actions/artifact/node_modules/@octokit/plugin-paginate-rest": { + "version": "2.21.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", + "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.40.0" + }, + "peerDependencies": { + "@octokit/core": ">=2" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", + "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.39.0", + "deprecation": "^2.3.1" + }, + "peerDependencies": { + "@octokit/core": ">=3" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/request": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", + "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^6.0.1", + "@octokit/request-error": "^2.1.0", + "@octokit/types": "^6.16.1", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.7", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/request/node_modules/@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "node_modules/@actions/artifact/node_modules/@octokit/types": { + "version": "6.41.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", + "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^12.11.0" + } + }, "node_modules/@actions/core": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz", @@ -117,6 +273,184 @@ "node": ">=6.0.0" } }, + "node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz", + "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.3.tgz", + "integrity": "sha512-/wGw8fJ4mdpJ1Cum7s1S+VQyXt1ihwKLzfabS1O/RDADnmzVc01dHn44qD0BvGH6KlZNzOMW95tEpKqhkCChPA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.4.0", + "@azure/core-rest-pipeline": "^1.9.1", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.6.1", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.2.0.tgz", + "integrity": "sha512-1kW8ZhN0CfbNOG6C688z5uh2yrzALE7dDXHiR9dY4vt+EbhGZQSbjDa5bQd2rf3X2pdWMsXbqbArxUyeNdvtmg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-client": "^1.3.0", + "@azure/core-rest-pipeline": "^1.19.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", + "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.19.1.tgz", + "integrity": "sha512-zHeoI3NCs53lLBbWNzQycjnYKsA1CVKlnzSNuSFcUDwBp8HHVObePxrM7HaX+Ha5Ks639H7chNC9HOaIhNS03w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.8.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz", + "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", + "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.5.tgz", + "integrity": "sha512-gT4H8mTaSXRz7eGTuQyq1aIJnJqeXzpOe9Ay7Z3FrCouer14CbV3VzjnJrNrQfbBpGBLO9oy8BmrY75A0p53cA==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.0.7", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.4.tgz", + "integrity": "sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.27.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.27.0.tgz", + "integrity": "sha512-IQjj9RIzAKatmNca3D6bT0qJ+Pkox1WZGOg2esJF2YLHb45pQKOwGPIAV+w3rfgkj7zV3RMxpn/c6iftzSOZJQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.4.0", + "@azure/core-client": "^1.6.2", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-rest-pipeline": "^1.10.1", + "@azure/core-tracing": "^1.1.2", + "@azure/core-util": "^1.6.1", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.0.0", + "events": "^3.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", @@ -856,6 +1190,102 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1465,6 +1895,15 @@ "@octokit/openapi-types": "^20.0.0" } }, + "node_modules/@octokit/plugin-request-log": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz", + "integrity": "sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA==", + "license": "MIT", + "peerDependencies": { + "@octokit/core": ">=3" + } + }, "node_modules/@octokit/plugin-rest-endpoint-methods": { "version": "10.4.1", "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", @@ -1495,6 +1934,31 @@ "@octokit/openapi-types": "^20.0.0" } }, + "node_modules/@octokit/plugin-retry": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz", + "integrity": "sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.0.3", + "bottleneck": "^2.15.3" + } + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/openapi-types": { + "version": "12.11.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", + "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/types": { + "version": "6.41.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", + "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^12.11.0" + } + }, "node_modules/@octokit/request": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", @@ -1533,6 +1997,67 @@ "@octokit/openapi-types": "^24.2.0" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobuf-ts/plugin": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@protobuf-ts/plugin/-/plugin-2.9.6.tgz", + "integrity": "sha512-Wpv5rkXeu6E5Y4r0TjWE0bzRGddiTYl/RM+tLgVlS0r8CqOBqNAmlWv+s8ltf/F75rVrahUal0cpyhFwha9GRA==", + "license": "Apache-2.0", + "dependencies": { + "@protobuf-ts/plugin-framework": "^2.9.6", + "@protobuf-ts/protoc": "^2.9.6", + "@protobuf-ts/runtime": "^2.9.6", + "@protobuf-ts/runtime-rpc": "^2.9.6", + "typescript": "^3.9" + }, + "bin": { + "protoc-gen-dump": "bin/protoc-gen-dump", + "protoc-gen-ts": "bin/protoc-gen-ts" + } + }, + "node_modules/@protobuf-ts/plugin-framework": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@protobuf-ts/plugin-framework/-/plugin-framework-2.9.6.tgz", + "integrity": "sha512-w7A1RXrDCiVzcaRE6YJP7FCARuAFe/Vc4SNQnHAi4CF0V6mEtyjAYEIC5BNYgIRaJEqB26zzsBQjIem3R02SCA==", + "license": "(Apache-2.0 AND BSD-3-Clause)", + "dependencies": { + "@protobuf-ts/runtime": "^2.9.6", + "typescript": "^3.9" + } + }, + "node_modules/@protobuf-ts/protoc": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@protobuf-ts/protoc/-/protoc-2.9.6.tgz", + "integrity": "sha512-c0XvAPDIBAovH9HxV8gBv8gzOTreQIqibcusrB1+DxvFiSvy+2V1tjbUmG5gJEbjk3aAOaoj0a3+QuE+P28xUw==", + "license": "Apache-2.0", + "bin": { + "protoc": "protoc.js" + } + }, + "node_modules/@protobuf-ts/runtime": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@protobuf-ts/runtime/-/runtime-2.9.6.tgz", + "integrity": "sha512-C0CfpKx4n4LBbUrajOdRj2BTbd3qBoK0SiKWLq7RgCoU6xiN4wesBMFHUOBp3fFzKeZwgU8Q2KtzaqzIvPLRXg==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@protobuf-ts/runtime-rpc": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@protobuf-ts/runtime-rpc/-/runtime-rpc-2.9.6.tgz", + "integrity": "sha512-0UeqDRzNxgsh08lY5eWzFJNfD3gZ8Xf+WG1HzbIAbVAigzigwjwsYNNhTeas5H3gco1U5owTzCg177aambKOOw==", + "license": "Apache-2.0", + "dependencies": { + "@protobuf-ts/runtime": "^2.9.6" + } + }, "node_modules/@rtsao/scc": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", @@ -1704,6 +2229,18 @@ "dev": true, "license": "MIT" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.14.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", @@ -1727,6 +2264,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -1777,7 +2323,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -1787,7 +2332,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -1813,6 +2357,86 @@ "node": ">= 8" } }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/archiver-utils/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1940,6 +2564,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, "node_modules/async-function": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", @@ -1966,6 +2596,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -2088,12 +2724,58 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/before-after-hook": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", "license": "Apache-2.0" }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -2160,6 +2842,39 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -2167,6 +2882,14 @@ "dev": true, "license": "MIT" }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -2258,6 +2981,18 @@ ], "license": "CC-BY-4.0" }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -2345,7 +3080,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -2358,9 +3092,24 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2374,6 +3123,37 @@ "dev": true, "license": "MIT" }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/create-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", @@ -2400,7 +3180,6 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -2469,7 +3248,6 @@ "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2592,6 +3370,12 @@ "node": ">= 0.4" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, "node_modules/electron-to-chromium": { "version": "1.5.125", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.125.tgz", @@ -2599,6 +3383,15 @@ "dev": true, "license": "ISC" }, + "node_modules/elfy": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/elfy/-/elfy-1.0.0.tgz", + "integrity": "sha512-4Kp3AA94jC085IJox+qnvrZ3PudqTi4gQNvIoTZfJJ9IqkRuCoqP60vCVYlIg00c5aYusi5Wjh2bf0cHYt+6gQ==", + "license": "MIT", + "dependencies": { + "endian-reader": "^0.3.0" + } + }, "node_modules/emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", @@ -2616,7 +3409,12 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, + "license": "MIT" + }, + "node_modules/endian-reader": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/endian-reader/-/endian-reader-0.3.0.tgz", + "integrity": "sha512-zPlHN59VLEjeJtpEU41ti/i7ZvTbwclvUN2M8anCsI3tOC/3mq6WNTJEKi49A5eLGvDkA0975LZb67Xwp7u4xQ==", "license": "MIT" }, "node_modules/error-ex": { @@ -3237,6 +4035,24 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -3294,6 +4110,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -3308,6 +4130,24 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-xml-parser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.0.tgz", + "integrity": "sha512-Uw9+Mjt4SBRud1IcaYuW/O0lW8SKKdMl5g7g24HiIuyH5fQSD+AVLybSlJtqLYEbytVFjWQa5DMGcNgeksdRBg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", @@ -3398,6 +4238,34 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -3628,7 +4496,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, "license": "ISC" }, "node_modules/has-bigints": { @@ -3725,13 +4592,39 @@ "node": ">= 0.4" } }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true, - "license": "MIT" - }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -3752,6 +4645,26 @@ "node": ">=10.18" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -3825,7 +4738,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, "license": "ISC" }, "node_modules/internal-slot": { @@ -4015,7 +4927,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4103,6 +5014,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -4155,7 +5075,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4272,7 +5191,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, "license": "ISC" }, "node_modules/istanbul-lib-coverage": { @@ -4359,6 +5277,21 @@ "node": ">=8" } }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", @@ -5039,6 +5972,12 @@ "json5": "lib/cli.js" } }, + "node_modules/jwt-decode": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-3.1.2.tgz", + "integrity": "sha512-UfpWE/VZn0iP50d8cz9NrZLM9lSWhcJ+0Gt/nm4by88UL+J1SiKN8/5dkjMmbEzwL2CAe+67GsegCbIKtbp75A==", + "license": "MIT" + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -5059,6 +5998,54 @@ "node": ">=6" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -5106,6 +6093,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -5239,17 +6232,36 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/natural-compare": { @@ -5259,6 +6271,26 @@ "dev": true, "license": "MIT" }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -5277,7 +6309,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5496,6 +6527,12 @@ "node": ">=6" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -5552,7 +6589,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5565,6 +6601,28 @@ "dev": true, "license": "MIT" }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -5728,6 +6786,21 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -5776,6 +6849,52 @@ "dev": true, "license": "MIT" }, + "node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -5927,6 +7046,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safe-push-apply": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz", @@ -6024,7 +7163,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -6037,7 +7175,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -6194,6 +7331,28 @@ "node": ">=8" } }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", @@ -6212,7 +7371,21 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -6286,7 +7459,19 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -6328,6 +7513,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz", + "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -6354,6 +7551,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -6369,6 +7577,15 @@ "node": ">=8" } }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/thingies": { "version": "1.21.0", "resolved": "https://registry.npmjs.org/thingies/-/thingies-1.21.0.tgz", @@ -6402,6 +7619,21 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/tree-dump": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.0.2.tgz", @@ -6436,7 +7668,6 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, "license": "0BSD" }, "node_modules/tunnel": { @@ -6549,6 +7780,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typescript": { + "version": "3.9.10", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", + "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, "node_modules/unbox-primitive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz", @@ -6593,6 +7837,16 @@ "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==", "license": "ISC" }, + "node_modules/unzip-stream": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.4.tgz", + "integrity": "sha512-PyofABPVv+d7fL7GOpusx7eRT9YETY2X04PhwbSipdj6bMxVCFJrr+nm0Mxqbf9hUiTin/UsnuFWBXlDZFy0Cw==", + "license": "MIT", + "dependencies": { + "binary": "^0.3.0", + "mkdirp": "^0.5.1" + } + }, "node_modules/update-browserslist-db": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", @@ -6634,6 +7888,12 @@ "punycode": "^2.1.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", @@ -6659,11 +7919,26 @@ "makeerror": "1.0.12" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -6792,6 +8067,24 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -6870,6 +8163,20 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } } } } diff --git a/package.json b/package.json index e55b212..b7e6223 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "author": "", "license": "MIT", "dependencies": { + "@actions/artifact": "^2.3.2", "@actions/core": "^1.11.1", "@actions/exec": "^1.1.1", "@actions/github": "^6.0.0", diff --git a/src/build-and-prep-ort-files/index.js b/src/build-and-prep-ort-files/index.js new file mode 100644 index 0000000..4cb3628 --- /dev/null +++ b/src/build-and-prep-ort-files/index.js @@ -0,0 +1,228 @@ +const core = require('@actions/core'); +const exec = require('@actions/exec'); +const fs = require('fs').promises; +const path = require('path'); +const os = require('os'); + +/** + * Executes a command and logs its output. Throws an error if the command fails. + * @param {string} command The command to execute. + * @param {string[]} [args] Command arguments. + * @param {exec.ExecOptions} [options] Execution options, including 'cwd' for working directory. + * @returns {Promise<{exitCode: number, stdout: string, stderr: string}>} Command output. + */ +async function runCommand(command, args = [], options = {}) { + // Default working directory is the workspace root unless overridden + const defaultCwd = process.env.GITHUB_WORKSPACE; + const effectiveOptions = { + cwd: defaultCwd, // Apply default + ignoreReturnCode: false, // Throw error on failure by default + silent: false, // Show command output by default + ...options // Apply overrides from caller (like cwd) + }; + + const cwdString = effectiveOptions.cwd === defaultCwd ? 'default workspace' : effectiveOptions.cwd; + core.info(`Executing in ${cwdString}: ${command} ${args.map(arg => arg.includes(' ') ? `"${arg}"` : arg).join(' ')}`); // Basic quoting for display + + try { + const { exitCode, stdout, stderr } = await exec.getExecOutput(command, args, effectiveOptions); + + if (exitCode !== 0 && !effectiveOptions.ignoreReturnCode) { + // Log stderr specifically as error if command failed + core.error(`Stderr: ${stderr}`); + throw new Error(`Command exited with code ${exitCode}: ${command} ${args.join(' ')}`); + } + core.info(`Finished: ${command} ${args.join(' ')}`); + return { exitCode, stdout, stderr }; + } catch (error) { + // exec.getExecOutput throws on non-zero exit codes if ignoreReturnCode is false + core.error(`Error executing command: ${command} ${args.join(' ')} in ${cwdString}`); + core.error(error); // Log the full error object from exec + // Rethrow with a clearer message if possible + throw new Error(`Command execution failed: ${error.message || error}`); + } +} + +/** + * Main function for the GitHub Action. + */ +async function main() { + core.info('Starting ORT Full Build and Test File Preparation Action...'); + + // --- Define Paths using Environment Variables --- + const buildDir = process.env.RUNNER_TEMP; // Hardcoded to runner temp dir + const workspaceDir = process.env.GITHUB_WORKSPACE; + if (!buildDir || !workspaceDir) { + throw new Error("Required environment variables RUNNER_TEMP or GITHUB_WORKSPACE not set."); + } + + const testDataDir = path.join(buildDir, 'minimal_build_test_data'); + const customOpsTestDataDir = path.join(testDataDir, 'custom_ops_model'); + const debugOutputDir = path.join(buildDir, 'Debug'); + const wheelDir = path.join(debugOutputDir, 'dist'); // Directory containing the built wheel + + core.info(`Using Build Directory (RUNNER_TEMP): ${buildDir}`); + core.info(`Workspace Directory (GITHUB_WORKSPACE): ${workspaceDir}`); + core.info(`Derived Test Data Directory: ${testDataDir}`); + core.info(`Wheel Directory: ${wheelDir}`); + + try { // Wrap main logic in try/catch for core.setFailed + // Ensure necessary directories exist + await fs.mkdir(testDataDir, { recursive: true }); + await fs.mkdir(debugOutputDir, { recursive: true }); + await fs.mkdir(wheelDir, { recursive: true }); + core.info(`Ensured directories exist: ${testDataDir}, ${debugOutputDir}, ${wheelDir}`); + + + // --- Step 1: Install Python Requirements --- + core.startGroup('Install Python Requirements'); + const requirementsPath = path.join(workspaceDir, 'tools/ci_build/github/linux/python/requirements.txt'); + // Run this in the workspace directory as it reads a file from there + await runCommand('python3', ['-m', 'pip', 'install', '--user', '-r', requirementsPath], { cwd: workspaceDir }); + core.endGroup(); + + // --- Step 2: Validate Operator Registrations --- + core.startGroup('Validate Operator Registrations'); + const validatorScript = path.join(workspaceDir, 'tools/ci_build/op_registration_validator.py'); + // Run this in the workspace directory as it likely operates on source files + await runCommand('python3', [validatorScript], { cwd: workspaceDir }); + core.endGroup(); + + // --- Step 3: Run Full ORT Build --- + core.startGroup('Run Full ORT Build'); + const buildScript = path.join(workspaceDir, 'tools/ci_build/build.py'); + const buildArgs = [ + buildScript, + '--build_dir', buildDir, + '--cmake_generator', 'Ninja', + '--config', 'Debug', + '--skip_submodule_sync', + '--parallel', + '--use_vcpkg', + '--use_vcpkg_ms_internal_asset_cache', + '--use_binskim_compliant_compile_flags', + '--build_wheel', + '--skip_tests', + '--enable_training_ops', + '--use_nnapi', + '--use_coreml' + ]; + // Run build from the workspace directory + await runCommand('python3', buildArgs, { cwd: workspaceDir }); + core.endGroup(); + + // --- Step 4: Install the ORT Python Wheel --- + core.startGroup('Install ORT Python Wheel'); + // We can run pip from anywhere, it doesn't need a specific CWD here + await runCommand('python3', [ + '-m', 'pip', 'install', + '--user', // Keep installing as user + '--find-links', wheelDir, + 'onnxruntime' // Specify the package name + ]); + core.endGroup(); + + // --- Set CWD for subsequent python scripts --- + // Reason: After installing the 'onnxruntime' wheel, running python scripts + // from the workspace root might cause Python to import the local 'onnxruntime' + // source directory instead of the newly installed package. Changing the CWD + // to a neutral location like '/tmp' prevents this import conflict. + const pythonScriptCwd = '/tmp'; + await fs.mkdir(pythonScriptCwd, { recursive: true }); // Ensure /tmp exists + core.info(`Setting CWD for subsequent Python scripts to: ${pythonScriptCwd}`); + + + // --- Step 5: Convert E2E ONNX models to ORT format --- + core.startGroup('Convert E2E ONNX models to ORT format (Tool)'); + const e2eTestDataPath = path.join(workspaceDir, 'onnxruntime/test/testdata/ort_minimal_e2e_test_data'); + const convertScript = path.join(workspaceDir, 'tools/python/convert_onnx_models_to_ort.py'); + // Pass absolute paths and change CWD + await runCommand('python3', [convertScript, e2eTestDataPath], { cwd: pythonScriptCwd }); + core.endGroup(); + + // --- Step 6: Convert again using installed package tool --- + core.startGroup('Convert E2E ONNX models to ORT format (Installed Package)'); + // Pass absolute paths and change CWD + await runCommand('python3', ['-m', 'onnxruntime.tools.convert_onnx_models_to_ort', e2eTestDataPath], { cwd: pythonScriptCwd }); + core.endGroup(); + + // --- Step 7: Create required ops config files --- + core.startGroup('Create required ops config files'); + const createConfigScript = path.join(workspaceDir, 'tools/python/create_reduced_build_config.py'); + const testDataRoot = path.join(workspaceDir, 'onnxruntime/test/testdata'); + const requiredOpsConfig = path.join(testDataDir, 'required_ops.ort_models.config'); + const requiredOpsTypesConfig = path.join(testDataDir, 'required_ops_and_types.ort_models.config'); + + // Pass absolute paths and change CWD + // Config without type reduction + await runCommand('python3', [createConfigScript, '--format', 'ORT', testDataRoot, requiredOpsConfig], { cwd: pythonScriptCwd }); + + // Config with type reduction + await runCommand('python3', [createConfigScript, '--format', 'ORT', '--enable_type_reduction', testDataRoot, requiredOpsTypesConfig], { cwd: pythonScriptCwd }); + core.endGroup(); + + // --- Step 8: Append standalone invoker ops --- + core.startGroup('Append standalone invoker ops'); + const standaloneInvokerConfig = path.join(e2eTestDataPath, 'required_ops.standalone_invoker.config'); + core.info(`Appending ${standaloneInvokerConfig} to config files...`); + const standaloneOpsContent = await fs.readFile(standaloneInvokerConfig, 'utf8'); + await fs.appendFile(requiredOpsConfig, os.EOL + standaloneOpsContent); + await fs.appendFile(requiredOpsTypesConfig, os.EOL + standaloneOpsContent); + core.info(`Successfully appended standalone invoker ops.`); + core.endGroup(); + + // --- Step 9: Test conversion with custom ops --- + core.startGroup('Test conversion with custom ops'); + await fs.mkdir(customOpsTestDataDir, { recursive: true }); + + const customOpSrcDir = path.join(workspaceDir, 'onnxruntime/test/testdata/custom_op_library'); + const customOpLibrary = path.join(debugOutputDir, 'libcustom_op_library.so'); // Assuming Linux .so extension + + // Copy relevant .onnx files + core.info(`Copying custom op models from ${customOpSrcDir} to ${customOpsTestDataDir}`); + let filesCopied = 0; + try { + const customOpOnnxFiles = (await fs.readdir(customOpSrcDir)).filter(f => f.endsWith('.onnx')); + if (customOpOnnxFiles.length === 0) { + core.warning(`No .onnx files found in ${customOpSrcDir} to copy.`); + } else { + for (const file of customOpOnnxFiles) { + const src = path.join(customOpSrcDir, file); + const dest = path.join(customOpsTestDataDir, file); + await fs.copyFile(src, dest); + core.info(`Copied ${file}`); + filesCopied++; + } + } + core.info(`Copied ${filesCopied} custom op model files.`); + } catch(copyError) { + core.warning(`Could not copy custom op models from ${customOpSrcDir}: ${copyError.message}`); + } + + + // Run conversion, checking if library exists first + // Pass absolute paths and change CWD + try { + await fs.access(customOpLibrary, fs.constants.R_OK); + await runCommand('python3', [convertScript, '--custom_op_library', customOpLibrary, customOpsTestDataDir], { cwd: pythonScriptCwd }); + } catch (err) { + // Log as warning if library doesn't exist or isn't readable + core.warning(`Custom op library ${customOpLibrary} not found or not readable. Skipping custom op conversion test. Error: ${err.message}`); + } + core.endGroup(); + + // --- Step 10: Clean up custom ops test dir --- + core.startGroup('Clean up custom ops test directory'); + core.info(`Cleaning up ${customOpsTestDataDir}`); + await fs.rm(customOpsTestDataDir, { recursive: true, force: true }); + core.endGroup(); + + core.info('Action finished successfully.'); + + } catch (error) { + core.setFailed(`Action failed: ${error.message}`); + } +} + +// Execute the main function +main(); \ No newline at end of file diff --git a/src/build-minimal-ort-and-run-tests/index.js b/src/build-minimal-ort-and-run-tests/index.js new file mode 100644 index 0000000..b549291 --- /dev/null +++ b/src/build-minimal-ort-and-run-tests/index.js @@ -0,0 +1,232 @@ +const core = require('@actions/core'); +const exec = require('@actions/exec'); +// Import the default client class +const { DefaultArtifactClient } = require('@actions/artifact'); +const fs = require('fs').promises; +const path = require('path'); +const os = require('os'); + +/** + * Executes a command and logs its output. Throws an error if the command fails. + * @param {string} command The command to execute. + * @param {string[]} [args] Command arguments. + * @param {exec.ExecOptions} [options] Execution options. + * @returns {Promise<{exitCode: number, stdout: string, stderr: string}>} Command output. + */ +async function runCommand(command, args = [], options = {}) { + const effectiveOptions = { + cwd: process.env.GITHUB_WORKSPACE, // Default working directory + ignoreReturnCode: false, // Throw error on failure by default + silent: false, // Show command output by default + listeners: { + stdout: (data) => { core.info(data.toString().trim()); }, + stderr: (data) => { core.warning(data.toString().trim()); } // Log stderr as warning by default + }, + ...options + }; + const cwdString = effectiveOptions.cwd === process.env.GITHUB_WORKSPACE ? 'default workspace' : effectiveOptions.cwd; + core.info(`Executing in ${cwdString}: ${command} ${args.map(arg => arg.includes(' ') ? `"${arg}"` : arg).join(' ')}`); // Basic quoting for display + try { + const { exitCode, stdout, stderr } = await exec.getExecOutput(command, args, effectiveOptions); + + if (exitCode !== 0 && !effectiveOptions.ignoreReturnCode) { + core.error(`Stderr: ${stderr}`); + throw new Error(`Command exited with code ${exitCode}: ${command} ${args.join(' ')}`); + } + core.info(`Finished: ${command} ${args.join(' ')}`); + return { exitCode, stdout, stderr }; + } catch (error) { + core.error(`Error executing command: ${command} ${args.join(' ')} in ${cwdString}`); + core.error(error); + throw new Error(`Command execution failed: ${error.message || error}`); + } +} + +/** + * Checks if a path exists. + * @param {string} pathToCheck The path to check. + * @returns {Promise} True if the path exists, false otherwise. + */ +async function checkPathExists(pathToCheck) { + try { + await fs.access(pathToCheck); + core.info(`Path exists: ${pathToCheck}`); + return true; + } catch (error) { + if (error.code === 'ENOENT') { + core.info(`Path does not exist: ${pathToCheck}`); + return false; + } + core.warning(`Error checking path ${pathToCheck}: ${error.message}`); + return false; // Assume not accessible on other errors + } +} + + +/** + * Main function for the GitHub Action. + */ +async function main() { + core.info('Starting Minimal ORT Build Action...'); + + // --- Get Inputs --- + const reducedOpsConfigFileBase = core.getInput('reduced-ops-config-file', { required: true }); + const enableTypeReduction = core.getBooleanInput('enable-type-reduction'); + const enableCustomOps = core.getBooleanInput('enable-custom-ops'); + const skipModelTests = core.getBooleanInput('skip-model-tests'); + const binarySizeReportNamePrefix = core.getInput('binary-size-report-name-prefix'); + + // --- Define Paths --- + const buildDir = process.env.RUNNER_TEMP; + const workspaceDir = process.env.GITHUB_WORKSPACE; + if (!buildDir || !workspaceDir) { + throw new Error("Required environment variables RUNNER_TEMP or GITHUB_WORKSPACE not set."); + } + + const testDataDownloadDir = path.join(buildDir, '.test_data'); + const fullReducedOpsConfigFile = path.join(testDataDownloadDir, reducedOpsConfigFileBase); + const debugOutputDir = path.join(buildDir, 'Debug'); // Consistent with build.py default + const testRunnerPath = path.join(debugOutputDir, 'onnx_test_runner'); + const libraryPath = path.join(debugOutputDir, 'libonnxruntime.so'); // Assuming Linux build + const binarySizeReportPath = path.join(debugOutputDir, 'binary_size_data.txt'); + const minimalE2eTestDataDir = path.join(workspaceDir, 'onnxruntime/test/testdata/ort_minimal_e2e_test_data'); + const standardOnnxTestDataDir = '/data/onnx'; + + core.info(`Using Build Directory (RUNNER_TEMP): ${buildDir}`); + core.info(`Workspace Directory (GITHUB_WORKSPACE): ${workspaceDir}`); + core.info(`Test Data Download Directory: ${testDataDownloadDir}`); + core.info(`Reduced Ops Config File Path: ${fullReducedOpsConfigFile}`); + + const artifactClient = new DefaultArtifactClient(); + + try { + // --- Download Test Data --- + core.startGroup('Download Test Data Artifact'); + const artifactName = 'test_data'; + let artifactIdToDownload; + try { + core.info(`Attempting to find artifact named '${artifactName}'...`); + // Use getArtifact to find the artifact by name and get its ID + const getArtifactResponse = await artifactClient.getArtifact(artifactName); + if (!getArtifactResponse || !getArtifactResponse.artifact) { + throw new Error(`Artifact '${artifactName}' not found.`); + } + artifactIdToDownload = getArtifactResponse.artifact.id; + core.info(`Found artifact '${artifactName}' with ID: ${artifactIdToDownload}`); + + core.info(`Downloading artifact ID ${artifactIdToDownload} to ${testDataDownloadDir}...`); + const downloadResponse = await artifactClient.downloadArtifact(artifactIdToDownload, { + path: testDataDownloadDir + }); + core.info(`Artifact download finished. Path: ${downloadResponse.downloadPath}`); + + // Verify the specific config file exists after download + await fs.access(fullReducedOpsConfigFile, fs.constants.R_OK); + core.info(`Verified reduced ops config file exists: ${fullReducedOpsConfigFile}`); + } catch (error) { + core.error(`Failed to find or download required file in artifact '${artifactName}': ${error.message}`); + // Fail the action if artifact/config is missing, as it's required for the build + throw new Error(`Failed to get required test data artifact '${artifactName}' or config file '${reducedOpsConfigFileBase}'.`); + } + core.endGroup(); + + // --- Install Python Requirements --- + core.startGroup('Install Python Requirements'); + const requirementsPath = path.join(workspaceDir, 'tools/ci_build/github/linux/python/requirements.txt'); + await runCommand('python3', ['-m', 'pip', 'install', '--user','-r', requirementsPath], { cwd: workspaceDir }); + core.endGroup(); + + // --- Build Minimal ORT --- + core.startGroup('Build Minimal ORT'); + const buildScript = path.join(workspaceDir, 'tools/ci_build/build.py'); + const minimalBuildArgs = enableCustomOps ? 'custom_ops' : ''; // build.py handles empty string ok + const buildArgs = [ + buildScript, + '--build_dir', buildDir, + '--cmake_generator', 'Ninja', + '--config', 'Debug', + '--skip_submodule_sync', + '--build_shared_lib', + '--use_vcpkg', + '--use_vcpkg_ms_internal_asset_cache', + '--parallel', + '--use_binskim_compliant_compile_flags', + '--minimal_build', minimalBuildArgs, // Add custom_ops if enabled + '--disable_ml_ops', + '--include_ops_by_config', fullReducedOpsConfigFile, + ]; + if (enableTypeReduction) { + buildArgs.push('--enable_reduced_operator_type_support'); + } + await runCommand('python3', buildArgs.filter(arg => arg !== ''), { cwd: workspaceDir }); // Filter empty string from minimalBuildArgs + core.endGroup(); + + // --- Run E2E Model Tests --- + if (!skipModelTests) { + core.startGroup('Run E2E Model Tests'); + core.info(`Running tests against minimal E2E data: ${minimalE2eTestDataDir}`); + await runCommand(testRunnerPath, [minimalE2eTestDataDir]); + + core.info(`Running tests against standard ONNX test data: ${standardOnnxTestDataDir}`); + // Note: This assumes the path /data/onnx exists and is accessible on the runner where this action executes. + const standardTestDataExists = await checkPathExists(standardOnnxTestDataDir); + if (standardTestDataExists) { + await runCommand(testRunnerPath, [standardOnnxTestDataDir]); + } else { + core.warning(`Directory ${standardOnnxTestDataDir} not found or accessible on the runner. Skipping these tests.`); + } + core.endGroup(); + } else { + core.info('Skipping E2E model tests as requested.'); + } + + // --- Check Binary Size --- + core.startGroup('Check Binary Size'); + const checkSizeScript = path.join(workspaceDir, 'tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py'); + const arch = os.machine().toLowerCase(); // e.g., x64, arm64 + const platform = os.platform().toLowerCase(); // e.g., linux, darwin, win32 + const osName = platform === 'linux' ? 'Linux' : platform; // Map 'linux' to 'Linux' as used in script + + // Ensure the library exists before checking its size + try { + await fs.access(libraryPath, fs.constants.R_OK); + await runCommand('python3', [ + checkSizeScript, + '--arch', arch, + '--os', osName, + '--build_config', 'minimal-reduced', // As per original script + libraryPath + ]); + } catch (error) { + core.warning(`Could not access library file ${libraryPath} or run size check script: ${error.message}`); + // Decide if this should be a failure or warning + // throw new Error(`Failed during binary size check: ${error.message}`); + } + core.endGroup(); + + // --- Upload Binary Size Report --- + core.startGroup('Upload Binary Size Report'); + const reportArtifactName = `${binarySizeReportNamePrefix || 'minimal_build_'}${arch}_${osName}_binary_size_report`; + core.info(`Uploading ${binarySizeReportPath} as artifact: ${reportArtifactName}`); + try { + await fs.access(binarySizeReportPath, fs.constants.R_OK); // Check if report exists + const uploadResponse = await artifactClient.uploadArtifact(reportArtifactName, [binarySizeReportPath], debugOutputDir, { + continueOnError: false // Fail the workflow if upload fails + }); + core.info(`Artifact uploaded successfully: ${uploadResponse.artifactName}`); + } catch (err) { + core.warning(`Could not find or upload binary size report ${binarySizeReportPath}: ${err.message}`); + // Fail the job if upload is critical, otherwise just warn + // throw new Error(`Failed to upload artifact: ${err.message}`); + } + core.endGroup(); + + core.info('Action finished successfully.'); + + } catch (error) { + core.setFailed(`Action failed: ${error.message}`); + } +} + +// Execute the main function +main(); From 324b205bc99a75f1db6198d6f1f8dadb2bc4cc0a Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 4 Apr 2025 16:27:43 -0700 Subject: [PATCH 03/14] update --- package-lock.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5b90301..8ae1e5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,8 +14,7 @@ "@actions/exec": "^1.1.1", "@actions/github": "^6.0.0", "@actions/glob": "^0.5.0", - "@actions/tool-cache": "^2.0.2", - "elfy": "^1.0.0" + "@actions/tool-cache": "^2.0.2" }, "devDependencies": { "@eslint/eslintrc": "^3.3.1", From 0bc97d29b952222ffe3d7ec5d183cff7e433b01b Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 4 Apr 2025 16:29:24 -0700 Subject: [PATCH 04/14] update --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed6fcdf..1b5d7eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,7 +78,7 @@ jobs: # Add any other essential files here echo "Copying built artifacts..." - ACTIONS=("build-docker-image" "run-build-script-in-docker" "setup-build-tools") + ACTIONS=("build-and-prep-ort-files" "build-docker-image" "build-minimal-ort-and-run-tests" "run-build-script-in-docker" "setup-build-tools") BUILD_DIR_RELATIVE="build" # Relative to MAIN_REPO_PATH for action_name in "${ACTIONS[@]}"; do From 94a8d2af8546e332d0e88ec492f03b7b3eb4b6d0 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 7 Apr 2025 11:17:45 -0700 Subject: [PATCH 05/14] Update src/build-minimal-ort-and-run-tests/index.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/build-minimal-ort-and-run-tests/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-minimal-ort-and-run-tests/index.js b/src/build-minimal-ort-and-run-tests/index.js index b549291..7dd75c3 100644 --- a/src/build-minimal-ort-and-run-tests/index.js +++ b/src/build-minimal-ort-and-run-tests/index.js @@ -183,7 +183,7 @@ async function main() { // --- Check Binary Size --- core.startGroup('Check Binary Size'); const checkSizeScript = path.join(workspaceDir, 'tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py'); - const arch = os.machine().toLowerCase(); // e.g., x64, arm64 + const arch = os.arch().toLowerCase(); // e.g., x64, arm64 const platform = os.platform().toLowerCase(); // e.g., linux, darwin, win32 const osName = platform === 'linux' ? 'Linux' : platform; // Map 'linux' to 'Linux' as used in script From bc24797e843b945a197a1096043d1f9c20867b08 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 7 Apr 2025 11:17:51 -0700 Subject: [PATCH 06/14] Update src/build-and-prep-ort-files/index.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/build-and-prep-ort-files/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build-and-prep-ort-files/index.js b/src/build-and-prep-ort-files/index.js index 4cb3628..b82b232 100644 --- a/src/build-and-prep-ort-files/index.js +++ b/src/build-and-prep-ort-files/index.js @@ -127,8 +127,8 @@ async function main() { // from the workspace root might cause Python to import the local 'onnxruntime' // source directory instead of the newly installed package. Changing the CWD // to a neutral location like '/tmp' prevents this import conflict. - const pythonScriptCwd = '/tmp'; - await fs.mkdir(pythonScriptCwd, { recursive: true }); // Ensure /tmp exists + const pythonScriptCwd = os.tmpdir(); + await fs.mkdir(pythonScriptCwd, { recursive: true }); // Ensure temporary directory exists core.info(`Setting CWD for subsequent Python scripts to: ${pythonScriptCwd}`); From 5ce127e914800a9eddcf70e6c16abb46b493a64d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 7 Apr 2025 14:44:03 -0700 Subject: [PATCH 07/14] update --- .../action.yml | 19 +- src/build-minimal-ort-and-run-tests/index.js | 217 ++++--- src/run-build-script-in-docker/index.js | 551 ++++++++++++------ 3 files changed, 502 insertions(+), 285 deletions(-) diff --git a/actions/build-minimal-ort-and-run-tests/action.yml b/actions/build-minimal-ort-and-run-tests/action.yml index 908d528..df36816 100644 --- a/actions/build-minimal-ort-and-run-tests/action.yml +++ b/actions/build-minimal-ort-and-run-tests/action.yml @@ -1,21 +1,26 @@ name: 'Build Minimal ORT and Run Tests' -description: 'Downloads test_data artifact, builds minimal ORT based on config, runs tests, and uploads binary size report.' +description: 'Builds minimal ORT using either a reduced Ops config file (downloaded from test_data artifact) or globally allowed types. Runs tests and uploads a binary size report.' inputs: reduced-ops-config-file: - description: 'Path (relative to downloaded test_data artifact) to the reduced Ops config file (e.g., required_ops.ort_models.config).' - required: true + description: 'Path (relative to downloaded test_data artifact) to the reduced Ops config file. Mutually exclusive with `globally_allowed_types`.' + required: false + default: '' + globally_allowed_types: + description: 'Comma-separated list of globally allowed types (e.g., "float,int32_t"). If specified, a config file is generated, and artifact download is skipped. Mutually exclusive with `reduced-ops-config-file`.' + required: false + default: '' enable-type-reduction: - description: 'Build with type reduction enabled.' + description: 'Build with type reduction enabled. Note: Type reduction happens based on the config file content (`--enable_type_reduction` during config creation), not this flag directly during the minimal build.' required: false - default: 'false' # String 'false' for getBooleanInput + default: 'false' enable-custom-ops: description: 'Build with custom op support enabled.' required: false - default: 'false' # String 'false' for getBooleanInput + default: 'false' skip-model-tests: description: 'Skip running the E2E model tests with onnx_test_runner.' required: false - default: 'false' # String 'false' for getBooleanInput + default: 'false' binary-size-report-name-prefix: description: 'Optional prefix for the uploaded binary size artifact name.' required: false diff --git a/src/build-minimal-ort-and-run-tests/index.js b/src/build-minimal-ort-and-run-tests/index.js index 7dd75c3..ae5d4d0 100644 --- a/src/build-minimal-ort-and-run-tests/index.js +++ b/src/build-minimal-ort-and-run-tests/index.js @@ -2,7 +2,7 @@ const core = require('@actions/core'); const exec = require('@actions/exec'); // Import the default client class const { DefaultArtifactClient } = require('@actions/artifact'); -const fs = require('fs').promises; +const fs = require('fs').promises; // Ensure fs promises is imported const path = require('path'); const os = require('os'); @@ -14,52 +14,55 @@ const os = require('os'); * @returns {Promise<{exitCode: number, stdout: string, stderr: string}>} Command output. */ async function runCommand(command, args = [], options = {}) { - const effectiveOptions = { - cwd: process.env.GITHUB_WORKSPACE, // Default working directory - ignoreReturnCode: false, // Throw error on failure by default - silent: false, // Show command output by default - listeners: { - stdout: (data) => { core.info(data.toString().trim()); }, - stderr: (data) => { core.warning(data.toString().trim()); } // Log stderr as warning by default - }, - ...options - }; - const cwdString = effectiveOptions.cwd === process.env.GITHUB_WORKSPACE ? 'default workspace' : effectiveOptions.cwd; - core.info(`Executing in ${cwdString}: ${command} ${args.map(arg => arg.includes(' ') ? `"${arg}"` : arg).join(' ')}`); // Basic quoting for display - try { - const { exitCode, stdout, stderr } = await exec.getExecOutput(command, args, effectiveOptions); + // ... (runCommand function remains the same) ... +    const effectiveOptions = { +        cwd: process.env.GITHUB_WORKSPACE, // Default working directory +        ignoreReturnCode: false, // Throw error on failure by default +        silent: false, // Show command output by default +        listeners: { +            stdout: (data) => { core.info(data.toString().trim()); }, +            stderr: (data) => { core.warning(data.toString().trim()); } // Log stderr as warning by default +        }, +        ...options +    }; +    const cwdString = effectiveOptions.cwd === process.env.GITHUB_WORKSPACE ? 'default workspace' : effectiveOptions.cwd; +    core.info(`Executing in ${cwdString}: ${command} ${args.map(arg => arg.includes(' ') ? `"${arg}"` : arg).join(' ')}`); // Basic quoting for display +    try { +        const { exitCode, stdout, stderr } = await exec.getExecOutput(command, args, effectiveOptions); - if (exitCode !== 0 && !effectiveOptions.ignoreReturnCode) { - core.error(`Stderr: ${stderr}`); - throw new Error(`Command exited with code ${exitCode}: ${command} ${args.join(' ')}`); - } - core.info(`Finished: ${command} ${args.join(' ')}`); - return { exitCode, stdout, stderr }; - } catch (error) { - core.error(`Error executing command: ${command} ${args.join(' ')} in ${cwdString}`); - core.error(error); - throw new Error(`Command execution failed: ${error.message || error}`); - } +        if (exitCode !== 0 && !effectiveOptions.ignoreReturnCode) { +             core.error(`Stderr: ${stderr}`); +             throw new Error(`Command exited with code ${exitCode}: ${command} ${args.join(' ')}`); +        } +        core.info(`Finished: ${command} ${args.join(' ')}`); +        return { exitCode, stdout, stderr }; +    } catch (error) { +        core.error(`Error executing command: ${command} ${args.join(' ')} in ${cwdString}`); +        core.error(error); +        throw new Error(`Command execution failed: ${error.message || error}`); +    } } + /** * Checks if a path exists. * @param {string} pathToCheck The path to check. * @returns {Promise} True if the path exists, false otherwise. */ async function checkPathExists(pathToCheck) { - try { - await fs.access(pathToCheck); - core.info(`Path exists: ${pathToCheck}`); - return true; - } catch (error) { - if (error.code === 'ENOENT') { - core.info(`Path does not exist: ${pathToCheck}`); - return false; - } - core.warning(`Error checking path ${pathToCheck}: ${error.message}`); - return false; // Assume not accessible on other errors - } + // ... (checkPathExists function remains the same) ... +    try { +        await fs.access(pathToCheck); +        core.info(`Path exists: ${pathToCheck}`); +        return true; +    } catch (error) { +        if (error.code === 'ENOENT') { +            core.info(`Path does not exist: ${pathToCheck}`); +            return false; +        } +        core.warning(`Error checking path ${pathToCheck}: ${error.message}`); +        return false; // Assume not accessible on other errors +    } } @@ -70,12 +73,21 @@ async function main() { core.info('Starting Minimal ORT Build Action...'); // --- Get Inputs --- - const reducedOpsConfigFileBase = core.getInput('reduced-ops-config-file', { required: true }); - const enableTypeReduction = core.getBooleanInput('enable-type-reduction'); + const reducedOpsConfigFileBase = core.getInput('reduced-ops-config-file'); // Not required anymore + const globallyAllowedTypes = core.getInput('globally_allowed_types'); // New input + const enableTypeReduction = core.getBooleanInput('enable-type-reduction'); // Note: this doesn't directly control build.py flag const enableCustomOps = core.getBooleanInput('enable-custom-ops'); const skipModelTests = core.getBooleanInput('skip-model-tests'); const binarySizeReportNamePrefix = core.getInput('binary-size-report-name-prefix'); + // --- Input Validation --- + if (reducedOpsConfigFileBase && globallyAllowedTypes) { + throw new Error("Inputs 'reduced-ops-config-file' and 'globally_allowed_types' are mutually exclusive. Provide only one."); + } + if (!reducedOpsConfigFileBase && !globallyAllowedTypes) { + throw new Error("One of 'reduced-ops-config-file' or 'globally_allowed_types' must be provided."); + } + // --- Define Paths --- const buildDir = process.env.RUNNER_TEMP; const workspaceDir = process.env.GITHUB_WORKSPACE; @@ -83,8 +95,7 @@ async function main() { throw new Error("Required environment variables RUNNER_TEMP or GITHUB_WORKSPACE not set."); } - const testDataDownloadDir = path.join(buildDir, '.test_data'); - const fullReducedOpsConfigFile = path.join(testDataDownloadDir, reducedOpsConfigFileBase); + const testDataDir = path.join(buildDir, '.test_data'); // Directory for downloaded artifact OR generated config const debugOutputDir = path.join(buildDir, 'Debug'); // Consistent with build.py default const testRunnerPath = path.join(debugOutputDir, 'onnx_test_runner'); const libraryPath = path.join(debugOutputDir, 'libonnxruntime.so'); // Assuming Linux build @@ -94,41 +105,64 @@ async function main() { core.info(`Using Build Directory (RUNNER_TEMP): ${buildDir}`); core.info(`Workspace Directory (GITHUB_WORKSPACE): ${workspaceDir}`); - core.info(`Test Data Download Directory: ${testDataDownloadDir}`); - core.info(`Reduced Ops Config File Path: ${fullReducedOpsConfigFile}`); + core.info(`Derived Test Data/Config Directory: ${testDataDir}`); const artifactClient = new DefaultArtifactClient(); + let opsConfigFileToUse = ''; // Path to the config file that will be used by build.py try { - // --- Download Test Data --- - core.startGroup('Download Test Data Artifact'); - const artifactName = 'test_data'; - let artifactIdToDownload; - try { - core.info(`Attempting to find artifact named '${artifactName}'...`); - // Use getArtifact to find the artifact by name and get its ID - const getArtifactResponse = await artifactClient.getArtifact(artifactName); - if (!getArtifactResponse || !getArtifactResponse.artifact) { - throw new Error(`Artifact '${artifactName}' not found.`); - } - artifactIdToDownload = getArtifactResponse.artifact.id; - core.info(`Found artifact '${artifactName}' with ID: ${artifactIdToDownload}`); - - core.info(`Downloading artifact ID ${artifactIdToDownload} to ${testDataDownloadDir}...`); - const downloadResponse = await artifactClient.downloadArtifact(artifactIdToDownload, { - path: testDataDownloadDir - }); - core.info(`Artifact download finished. Path: ${downloadResponse.downloadPath}`); - - // Verify the specific config file exists after download - await fs.access(fullReducedOpsConfigFile, fs.constants.R_OK); - core.info(`Verified reduced ops config file exists: ${fullReducedOpsConfigFile}`); - } catch (error) { - core.error(`Failed to find or download required file in artifact '${artifactName}': ${error.message}`); - // Fail the action if artifact/config is missing, as it's required for the build - throw new Error(`Failed to get required test data artifact '${artifactName}' or config file '${reducedOpsConfigFileBase}'.`); + // Ensure the directory for config/test data exists + await fs.mkdir(testDataDir, { recursive: true }); + core.info(`Ensured directory exists: ${testDataDir}`); + + if (globallyAllowedTypes) { + // --- Generate Config File from globally_allowed_types --- + core.startGroup('Generate Ops Config from globally_allowed_types'); + core.info(`Input types: ${globallyAllowedTypes}`); + core.info('Skipping test data artifact download.'); + + const generatedConfigFileName = 'globally_allowed_types.config'; + opsConfigFileToUse = path.join(testDataDir, generatedConfigFileName); + + const configContent = `!globally_allowed_types;${globallyAllowedTypes}\n!no_ops_specified_means_all_ops_are_required\n`; + + await fs.writeFile(opsConfigFileToUse, configContent); + core.info(`Generated config file at: ${opsConfigFileToUse}`); + core.debug(`Config file content:\n${configContent}`); + core.endGroup(); + + } else { + // --- Download Test Data Artifact --- + core.startGroup('Download Test Data Artifact'); + opsConfigFileToUse = path.join(testDataDir, reducedOpsConfigFileBase); + core.info(`Expecting reduced ops config file at: ${opsConfigFileToUse}`); + + const artifactName = 'test_data'; + let artifactIdToDownload; + try { + core.info(`Attempting to find artifact named '${artifactName}'...`); + const getArtifactResponse = await artifactClient.getArtifact(artifactName); + if (!getArtifactResponse || !getArtifactResponse.artifact) { + throw new Error(`Artifact '${artifactName}' not found.`); + } + artifactIdToDownload = getArtifactResponse.artifact.id; + core.info(`Found artifact '${artifactName}' with ID: ${artifactIdToDownload}`); + + core.info(`Downloading artifact ID ${artifactIdToDownload} to ${testDataDir}...`); + const downloadResponse = await artifactClient.downloadArtifact(artifactIdToDownload, { + path: testDataDir + }); + core.info(`Artifact download finished. Path: ${downloadResponse.downloadPath}`); + + // Verify the specific config file exists after download + await fs.access(opsConfigFileToUse, fs.constants.R_OK); + core.info(`Verified reduced ops config file exists: ${opsConfigFileToUse}`); + } catch (error) { + core.error(`Failed to find or download required file '${reducedOpsConfigFileBase}' in artifact '${artifactName}': ${error.message}`); + throw new Error(`Failed to get required test data artifact '${artifactName}' or config file '${reducedOpsConfigFileBase}'.`); + } + core.endGroup(); } - core.endGroup(); // --- Install Python Requirements --- core.startGroup('Install Python Requirements'); @@ -139,7 +173,7 @@ async function main() { // --- Build Minimal ORT --- core.startGroup('Build Minimal ORT'); const buildScript = path.join(workspaceDir, 'tools/ci_build/build.py'); - const minimalBuildArgs = enableCustomOps ? 'custom_ops' : ''; // build.py handles empty string ok + const minimalBuildArgsValue = enableCustomOps ? 'custom_ops' : ''; // build.py handles empty string ok const buildArgs = [ buildScript, '--build_dir', buildDir, @@ -151,13 +185,20 @@ async function main() { '--use_vcpkg_ms_internal_asset_cache', '--parallel', '--use_binskim_compliant_compile_flags', - '--minimal_build', minimalBuildArgs, // Add custom_ops if enabled + '--minimal_build', minimalBuildArgsValue, // Add custom_ops if enabled '--disable_ml_ops', - '--include_ops_by_config', fullReducedOpsConfigFile, + '--include_ops_by_config', opsConfigFileToUse, // Use the determined config file path ]; - if (enableTypeReduction) { - buildArgs.push('--enable_reduced_operator_type_support'); + // Note: enable_type_reduction is handled when *creating* the config file, + // not as a direct build.py flag during minimal build based on that config. + // If the *config file itself* was generated with type reduction, the build will reflect that. + // The enable-type-reduction input here is mostly informative now. + if (enableTypeReduction && globallyAllowedTypes) { + core.warning('Input `enable-type-reduction` is set to true, but type reduction relies on how the config file was generated. When using `globally_allowed_types`, type reduction is NOT automatically applied by this action.'); + } else if (enableTypeReduction) { + core.info('Input `enable-type-reduction` is true. Assuming the provided `reduced-ops-config-file` was generated with type reduction enabled.'); } + await runCommand('python3', buildArgs.filter(arg => arg !== ''), { cwd: workspaceDir }); // Filter empty string from minimalBuildArgs core.endGroup(); @@ -190,16 +231,22 @@ async function main() { // Ensure the library exists before checking its size try { await fs.access(libraryPath, fs.constants.R_OK); - await runCommand('python3', [ + const sizeCheckArgs = [ checkSizeScript, '--arch', arch, '--os', osName, '--build_config', 'minimal-reduced', // As per original script libraryPath - ]); + ]; + // Add threshold if provided (assuming the script takes it as an optional arg) + const sizeThreshold = core.getInput('size-threshold'); + if (sizeThreshold) { + sizeCheckArgs.push('--threshold', sizeThreshold); // Adjust arg name if needed + } + await runCommand('python3', sizeCheckArgs); } catch (error) { - core.warning(`Could not access library file ${libraryPath} or run size check script: ${error.message}`); - // Decide if this should be a failure or warning + core.warning(`Could not access library file ${libraryPath} or run size check script: ${error.message}`); + // Decide if this should be a failure or warning - currently warning // throw new Error(`Failed during binary size check: ${error.message}`); } core.endGroup(); @@ -211,13 +258,13 @@ async function main() { try { await fs.access(binarySizeReportPath, fs.constants.R_OK); // Check if report exists const uploadResponse = await artifactClient.uploadArtifact(reportArtifactName, [binarySizeReportPath], debugOutputDir, { - continueOnError: false // Fail the workflow if upload fails + continueOnError: false // Fail the workflow if upload fails }); core.info(`Artifact uploaded successfully: ${uploadResponse.artifactName}`); } catch (err) { core.warning(`Could not find or upload binary size report ${binarySizeReportPath}: ${err.message}`); - // Fail the job if upload is critical, otherwise just warn - // throw new Error(`Failed to upload artifact: ${err.message}`); + // Fail the job if upload is critical, otherwise just warn - currently warning + // throw new Error(`Failed to upload artifact: ${err.message}`); } core.endGroup(); @@ -229,4 +276,4 @@ async function main() { } // Execute the main function -main(); +main(); \ No newline at end of file diff --git a/src/run-build-script-in-docker/index.js b/src/run-build-script-in-docker/index.js index 4c2cb9b..3490a41 100644 --- a/src/run-build-script-in-docker/index.js +++ b/src/run-build-script-in-docker/index.js @@ -2,226 +2,391 @@ const core = require('@actions/core'); const path = require('node:path'); const os = require('node:os'); const fs = require('node:fs/promises'); +const glob = require('@actions/glob'); // Ensure glob is required // Import shared utilities const { executeCommand, checkPathExists } = require('../common/utils'); // Import helpers // List of known EP names (keep local or move to common if used elsewhere?) - Keep local for now const KNOWN_EPS = new Set([ - 'acl', - 'armnn', - 'azure', - 'cann', - 'coreml', - 'cuda', - 'dml', - 'dnnl', - 'migraphx', - 'nnapi', - 'openvino', - 'qnn', - 'rknpu', - 'rocm', - 'snpe', - 'tensorrt', - 'vitisai', - 'vsinpu', - 'webgpu', - 'webnn', - 'xnnpack', +  'acl', +  'armnn', +  'azure', +  'cann', +  'coreml', +  'cuda', +  'dml', +  'dnnl', +  'migraphx', +  'nnapi', +  'openvino', +  'qnn', +  'rknpu', +  'rocm', +  'snpe', +  'tensorrt', +  'vitisai', +  'vsinpu', +  'webgpu', +  'webnn', +  'xnnpack', ]); const knownEpsString = Array.from(KNOWN_EPS).sort().join(', '); // checkGpu uses exec directly for failure tolerance/specific handling, keep as is or adapt helper? // Keep direct exec here as error handling is specific (warning, not throwing) async function checkGpu() { - let hasGpu = false; - try { - const options = { ignoreReturnCode: true, silent: true }; // Don't fail action if nvidia-smi not found - core.info('Checking for NVIDIA GPU using nvidia-smi...'); - // Use direct exec here because we want to check the exit code and *not* throw - const exitCode = await require('@actions/exec').exec('nvidia-smi', [], options); - if (exitCode === 0) { - core.info('NVIDIA GPU detected via nvidia-smi.'); - hasGpu = true; - } else { - core.warning(`nvidia-smi command failed or not found (exit code: ${exitCode}). Assuming no GPU.`); +  let hasGpu = false; +  try { +    const options = { ignoreReturnCode: true, silent: true }; // Don't fail action if nvidia-smi not found +    core.info('Checking for NVIDIA GPU using nvidia-smi...'); +    // Use direct exec here because we want to check the exit code and *not* throw +    const exitCode = await require('@actions/exec').exec('nvidia-smi', [], options); +    if (exitCode === 0) { +      core.info('NVIDIA GPU detected via nvidia-smi.'); +      hasGpu = true; +    } else { +      core.warning(`nvidia-smi command failed or not found (exit code: ${exitCode}). Assuming no GPU.`); +    } +  } catch (error) { +    core.warning(`Error trying to execute nvidia-smi: ${error.message}. Assuming no GPU.`); +  } +  return hasGpu; +} + +// --- Helper Function to Generate Test Summary --- +async function generateTestSummary(baseDir) { + core.startGroup('Generate Test Result Summary'); + const xmlPattern = path.join(baseDir, '**/*.results.xml').replace(/\\/g, '/'); // Normalize for glob + let totalTests = 0; + let totalFailures = 0; + let totalErrors = 0; + let totalSkipped = 0; + let filesProcessed = 0; + const failedFiles = []; + const processedFiles = []; + + core.info(`Searching for test result files matching: ${xmlPattern}`); + + try { + // Check if base directory exists first + await fs.access(baseDir); + core.info(`Base directory ${baseDir} exists.`); + + const globber = await glob.create(xmlPattern, { followSymbolicLinks: false }); + for await (const file of globber.globGenerator()) { + filesProcessed++; + const fileName = path.relative(baseDir, file) || path.basename(file); // Get relative path for display + processedFiles.push(fileName); + core.debug(`Processing file: ${file}`); + try { + const content = await fs.readFile(file, 'utf8'); + // Simple Regex to find the first or tag and extract counts + // It assumes attributes are present and well-formed. Handles optional skipped/disabled. + const testsuiteRegex = /]*?tests="(\d+)"[^>]*?failures="(\d+)"[^>]*?errors="(\d+)"(?:[^>]*?(?:skipped|disabled)="(\d+)")?/; + const match = content.match(testsuiteRegex); + + if (match) { + totalTests += parseInt(match[1] || '0', 10); + totalFailures += parseInt(match[2] || '0', 10); + totalErrors += parseInt(match[3] || '0', 10); + totalSkipped += parseInt(match[4] || '0', 10); // Group 4 is skipped/disabled + core.debug(` -> Found Tests: ${match[1]}, Failures: ${match[2]}, Errors: ${match[3]}, Skipped/Disabled: ${match[4] || '0'}`); + } else { + core.warning(`Could not find parsable tag with counts in ${fileName}`); + failedFiles.push(`${fileName} (parse error)`); + } + } catch (readError) { + core.warning(`Error reading test result file ${fileName}: ${readError.message}`); + failedFiles.push(`${fileName} (read error)`); + } + } + + if (filesProcessed === 0) { + core.info('No test result XML files found.'); + } else { + core.info(`Processed ${filesProcessed} test result XML file(s).`); + // --- Generate Job Summary --- + let summaryMarkdown = `## Test Results Summary\n\n`; + summaryMarkdown += `Processed **${filesProcessed}** \`*.results.xml\` file(s) from \`${path.basename(baseDir)}\`.\n\n`; // Use basename for clarity + + const totalProblems = totalFailures + totalErrors; + const overallStatus = totalProblems === 0 ? '✅ Passed' : '❌ Failed'; + + summaryMarkdown += `| Metric | Count |\n`; + summaryMarkdown += `| --------------- | ----: |\n`; + summaryMarkdown += `| **Total Tests** | ${totalTests} |\n`; + summaryMarkdown += `| Failures | ${totalFailures > 0 ? `**${totalFailures}** ❌` : totalFailures} |\n`; + summaryMarkdown += `| Errors | ${totalErrors > 0 ? `**${totalErrors}** ❌` : totalErrors} |\n`; + summaryMarkdown += `| Skipped | ${totalSkipped} |\n`; + summaryMarkdown += `| **Overall** | **${overallStatus}** |\n\n`; + + + if (failedFiles.length > 0) { + summaryMarkdown += `⚠️ **Issues processing some files:**\n`; + summaryMarkdown += failedFiles.map(f => `- \`${f}\``).join('\n') + '\n\n'; + } + + summaryMarkdown += `
Processed Files (${processedFiles.length})\n\n`; + summaryMarkdown += processedFiles.map(f => `- \`${f}\``).join('\n') + '\n'; + summaryMarkdown += `
\n`; + + // Add to GitHub Job Summary (append=true ensures it adds even if step failed) + await core.summary.addRaw(summaryMarkdown, true).write(); + core.info("Test result summary added to GitHub Job Summary."); + } + + } catch (error) { + if (error.code === 'ENOENT') { + core.info(`Test result base directory ${baseDir} not found. Skipping summary generation.`); + } else { + core.warning(`Error accessing test result directory ${baseDir} or globbing files: ${error.message}`); + } + } finally { + core.endGroup(); } - } catch (error) { - core.warning(`Error trying to execute nvidia-smi: ${error.message}. Assuming no GPU.`); - } - return hasGpu; } + async function run() { - try { - // --- Get All Inputs (unchanged) --- - const dockerImage = core.getInput('docker_image', { required: true }); + // Define buildDir early for use in finally block const buildConfig = core.getInput('build_config', { required: true }); - const runMode = core.getInput('mode', { required: true }); - const containerUser = core.getInput('container_user'); - const epInputString = core.getInput('execution_providers'); - const extraBuildFlags = core.getInput('extra_build_flags'); - const pythonPathPrefix = core.getInput('python_path_prefix'); - const allowOpset = core.getInput('allow_released_opset_only'); - const nightlyBuild = core.getInput('nightly_build'); - - // --- Validate Mode (unchanged) --- - let buildPyArg; - let shouldPassCacheVars = false; - const lowerCaseRunMode = runMode.toLowerCase(); - switch (lowerCaseRunMode) { - case 'update': - buildPyArg = '--update'; - shouldPassCacheVars = true; - break; - case 'build': - buildPyArg = '--build'; - break; - case 'test': - buildPyArg = '--test'; - break; - default: - core.setFailed(`Invalid mode: '${runMode}'.`); + const runnerTempDir = process.env.RUNNER_TEMP; + if (!runnerTempDir) { + core.setFailed('RUNNER_TEMP environment variable not set.'); // Fail early if this is missing return; } - core.info( - `Running mode: ${runMode} (build.py arg: ${buildPyArg}), Pass Cache Vars: ${shouldPassCacheVars}, Container User: ${containerUser || 'Default'}` - ); - - // --- Process Execution Providers Input (unchanged) --- - const epFlags = []; - const requestedEps = epInputString - .toLowerCase() - .split(' ') - .filter((ep) => ep.trim()); - if (requestedEps.length > 0) { - /* ... EP flag logic ... */ core.info(`Requested EPs: ${requestedEps.join(', ')}`); - for (const ep of requestedEps) { - if (KNOWN_EPS.has(ep)) { - const flag = `--use_${ep}`; - core.info(` Adding flag: ${flag}`); - epFlags.push(flag); + const buildOutputPath = path.join(runnerTempDir, 'build', buildConfig); + + try { + // --- Get All Inputs (unchanged) --- + const dockerImage = core.getInput('docker_image', { required: true }); + // buildConfig already obtained + const runMode = core.getInput('mode', { required: true }); + const containerUser = core.getInput('container_user'); + const epInputString = core.getInput('execution_providers'); + const extraBuildFlags = core.getInput('extra_build_flags'); + const pythonPathPrefix = core.getInput('python_path_prefix'); + const allowOpset = core.getInput('allow_released_opset_only'); + const nightlyBuild = core.getInput('nightly_build'); + + // --- Validate Mode (unchanged) --- + let buildPyArg; + let shouldPassCacheVars = false; + const lowerCaseRunMode = runMode.toLowerCase(); + switch (lowerCaseRunMode) { + case 'update': + buildPyArg = '--update'; + shouldPassCacheVars = true; + break; + case 'build': + buildPyArg = '--build'; + break; + case 'test': + buildPyArg = '--test'; + break; + default: + core.setFailed(`Invalid mode: '${runMode}'.`); + return; + } + core.info( + `Running mode: ${runMode} (build.py arg: ${buildPyArg}), Pass Cache Vars: ${shouldPassCacheVars}, Container User: ${containerUser || 'Default'}` + ); + + // --- Check if Python bindings are being built --- + const isPythonBuild = extraBuildFlags.includes('--build_wheel') || extraBuildFlags.includes('--enable_pybind'); + const isWheelBuild = extraBuildFlags.includes('--build_wheel'); // Specifically check for wheel build + if (isPythonBuild) { + core.info('Detected Python binding build based on extra_build_flags.'); + } + + // --- Process Execution Providers Input (unchanged) --- + const epFlags = []; + const requestedEps = epInputString + .toLowerCase() + .split(' ') + .filter((ep) => ep.trim()); + if (requestedEps.length > 0) { + /* ... EP flag logic ... */ core.info(`Requested EPs: ${requestedEps.join(', ')}`); + for (const ep of requestedEps) { + if (KNOWN_EPS.has(ep)) { + const flag = `--use_${ep}`; + core.info(`  Adding flag: ${flag}`); + epFlags.push(flag); + } else { + core.setFailed(`Unknown EP: '${ep}'. Allowed: ${knownEpsString}`); + return; + } + } + } + + // --- Get Runner Context/Defaults (unchanged) --- + const workspaceDir = process.env.GITHUB_WORKSPACE; + // runnerTempDir already obtained + const homeDir = os.homedir(); + const homeOnnxDir = path.join(homeDir, '.onnx'); + const hostCacheDir = path.join(homeDir, '.cache'); + const containerHomeDir = containerUser ? `/home/${containerUser}` : '/root'; // Adjust based on user? Defaulting home might be tricky. + if (!workspaceDir) throw new Error('GITHUB_WORKSPACE not set.'); + // runnerTempDir checked earlier + + // --- Check Host Paths (uses checkPathExists util) --- + const hostDataOnnxPath = '/data/onnx'; + const hostDataModelsPath = '/data/models'; + const dataOnnxExists = await checkPathExists(hostDataOnnxPath); // USE UTIL + const dataModelsExists = await checkPathExists(hostDataModelsPath); // USE UTIL + const enableOnnxTestsFlag = dataOnnxExists && dataModelsExists; + core.info(`--enable_onnx_tests will be ${enableOnnxTestsFlag ? 'added' : 'skipped'}.`); + + // --- Check for GPU (uses local checkGpu) --- + const gpuAvailable = await checkGpu(); + + // --- Construct build.py Command (unchanged) --- + const buildPyBaseArgs = [ + pythonPathPrefix, + 'python3', + 'tools/ci_build/build.py', + `--build_dir build/${buildConfig}`, // Relative path inside container + `--config ${buildConfig}`, + '--cmake_generator Ninja', + '--skip_submodule_sync', + '--build_shared_lib', + '--parallel', + '--use_vcpkg', + '--use_vcpkg_ms_internal_asset_cache', + ...(enableOnnxTestsFlag ? ['--enable_onnx_tests'] : []), + ...epFlags, + extraBuildFlags, + ]; + const buildPyBase = buildPyBaseArgs.filter((part) => part).join(' '); + const fullBuildPyCommand = `${buildPyBase} ${buildPyArg}`; + core.debug(`Constructed build.py command: ${fullBuildPyCommand}`); + + // --- Construct the full command to run inside Docker --- + let fullDockerCommand = `set -ex; ${fullBuildPyCommand}`; + if (isPythonBuild) { + const pythonRequirementsPath = 'tools/ci_build/github/linux/python/requirements.txt'; + const installReqsCommand = `python3 -m pip install --user -r ${pythonRequirementsPath}`; + fullDockerCommand = `set -ex; ${installReqsCommand} && ${fullBuildPyCommand}`; + core.info(`Prepending python requirements installation for Python build.`); + core.debug(`Full command inside Docker: ${fullDockerCommand}`); } else { - core.setFailed(`Unknown EP: '${ep}'. Allowed: ${knownEpsString}`); - return; + core.debug(`Full command inside Docker: ${fullDockerCommand}`); } - } - } - // --- Get Runner Context/Defaults (unchanged) --- - const workspaceDir = process.env.GITHUB_WORKSPACE; - const runnerTempDir = process.env.RUNNER_TEMP; - const homeDir = os.homedir(); - const homeOnnxDir = path.join(homeDir, '.onnx'); - const hostCacheDir = path.join(homeDir, '.cache'); - const containerHomeDir = containerUser ? `/home/${containerUser}` : '/root'; // Adjust based on user? Defaulting home might be tricky. - if (!workspaceDir) throw new Error('GITHUB_WORKSPACE not set.'); - if (!runnerTempDir) throw new Error('RUNNER_TEMP not set.'); - - // --- Check Host Paths (uses checkPathExists util) --- - const hostDataOnnxPath = '/data/onnx'; - const hostDataModelsPath = '/data/models'; - const dataOnnxExists = await checkPathExists(hostDataOnnxPath); // USE UTIL - const dataModelsExists = await checkPathExists(hostDataModelsPath); // USE UTIL - const enableOnnxTestsFlag = dataOnnxExists && dataModelsExists; - core.info(`--enable_onnx_tests will be ${enableOnnxTestsFlag ? 'added' : 'skipped'}.`); - - // --- Check for GPU (uses local checkGpu) --- - const gpuAvailable = await checkGpu(); - - // --- Construct build.py Command (unchanged) --- - const buildPyBaseArgs = [ - pythonPathPrefix, - 'python3', - 'tools/ci_build/build.py', - `--build_dir build/${buildConfig}`, - `--config ${buildConfig}`, - '--cmake_generator Ninja', - '--skip_submodule_sync', - '--build_shared_lib', - '--parallel', - '--use_vcpkg', - '--use_vcpkg_ms_internal_asset_cache', - ...(enableOnnxTestsFlag ? ['--enable_onnx_tests'] : []), - ...epFlags, - extraBuildFlags, - ]; - const buildPyBase = buildPyBaseArgs.filter((part) => part).join(' '); - const fullBuildPyCommand = `${buildPyBase} ${buildPyArg}`; - core.debug(`Constructed build.py command: ${fullBuildPyCommand}`); - - // --- Ensure Host Cache Directory Exists (unchanged) --- - core.info(`Ensuring host cache directory exists: ${hostCacheDir}`); - try { - await fs.mkdir(hostCacheDir, { recursive: true }); - core.info(`Host directory ${hostCacheDir} ensured.`); - } catch (error) { - core.warning(`Could not ensure host directory ${hostCacheDir} exists: ${error.message}.`); - } + // --- Ensure Host Cache Directory Exists (unchanged) --- + core.info(`Ensuring host cache directory exists: ${hostCacheDir}`); + try { + await fs.mkdir(hostCacheDir, { recursive: true }); + core.info(`Host directory ${hostCacheDir} ensured.`); + } catch (error) { + core.warning(`Could not ensure host directory ${hostCacheDir} exists: ${error.message}.`); + } - // --- Construct Docker Run Arguments (unchanged) --- - const dockerArgs = ['run', '--rm']; - if (gpuAvailable) dockerArgs.push('--gpus', 'all'); - core.info('Adding standard volume mounts: workspace, runner temp, host cache.'); - dockerArgs.push('--volume', `${workspaceDir}:/onnxruntime_src`); - dockerArgs.push('--volume', `${runnerTempDir}:/onnxruntime_src/build`); - dockerArgs.push('--volume', `${hostCacheDir}:${containerHomeDir}/.cache`); // Use determined container home - if (lowerCaseRunMode === 'test') { - /* ... test volume mount logic ... */ core.info('Mode is "test", checking test data mounts.'); - if (dataOnnxExists) { - dockerArgs.push('--volume', `${hostDataOnnxPath}:/data/onnx:ro`); - } else { - core.info(`Skipping ${hostDataOnnxPath} mount.`); - } - if (dataModelsExists) { - dockerArgs.push('--volume', `${hostDataModelsPath}:/data/models:ro`); - } else { - core.info(`Skipping ${hostDataModelsPath} mount.`); - } - core.info(`Ensuring host directory exists: ${homeOnnxDir}`); - try { - await fs.mkdir(homeOnnxDir, { recursive: true }); - core.info(`Host directory ${homeOnnxDir} ensured.`); - dockerArgs.push('--volume', `${homeOnnxDir}:${containerHomeDir}/.onnx`); - } catch (error) { - core.warning(`Could not ensure ${homeOnnxDir}: ${error.message}. Skipping mount.`); - } - } else { - core.info(`Mode is "${runMode}", skipping test data mounts.`); - } - dockerArgs.push('-w', '/onnxruntime_src'); - dockerArgs.push('-e', `ALLOW_RELEASED_ONNX_OPSET_ONLY=${allowOpset}`); - dockerArgs.push('-e', `NIGHTLY_BUILD=${nightlyBuild}`); - if (shouldPassCacheVars) { - /* ... cache var passing logic ... */ core.info('Passing cache env vars into container.'); - const cacheUrl = process.env.ACTIONS_CACHE_URL || ''; - const runtimeToken = process.env.ACTIONS_RUNTIME_TOKEN || ''; - if (cacheUrl) core.setSecret(cacheUrl); - if (runtimeToken) core.setSecret(runtimeToken); - if (cacheUrl) dockerArgs.push('-e', `ACTIONS_CACHE_URL=${cacheUrl}`); - else core.info('ACTIONS_CACHE_URL not found.'); - if (runtimeToken) dockerArgs.push('-e', `ACTIONS_RUNTIME_TOKEN=${runtimeToken}`); - else core.info('ACTIONS_RUNTIME_TOKEN not found.'); - dockerArgs.push('-e', 'RUNNER_TEMP=/onnxruntime_src/build'); - } else { - core.info('Skipping passing cache env vars.'); + // --- Construct Docker Run Arguments (unchanged logic, uses modified command) --- + const dockerArgs = ['run', '--rm']; + if (gpuAvailable) dockerArgs.push('--gpus', 'all'); + core.info('Adding standard volume mounts: workspace, runner temp, host cache.'); + dockerArgs.push('--volume', `${workspaceDir}:/onnxruntime_src`); + // Mount the parent of the build output dir for simplicity + dockerArgs.push('--volume', `${runnerTempDir}/build:/onnxruntime_src/build`); // Mount $RUNNER_TEMP/build -> /onnxruntime_src/build + dockerArgs.push('--volume', `${hostCacheDir}:${containerHomeDir}/.cache`); // Use determined container home + if (lowerCaseRunMode === 'test') { + /* ... test volume mount logic ... */ core.info('Mode is "test", checking test data mounts.'); + if (dataOnnxExists) { + dockerArgs.push('--volume', `${hostDataOnnxPath}:/data/onnx:ro`); + } else { + core.info(`Skipping ${hostDataOnnxPath} mount.`); + } + if (dataModelsExists) { + dockerArgs.push('--volume', `${hostDataModelsPath}:/data/models:ro`); + } else { + core.info(`Skipping ${hostDataModelsPath} mount.`); + } + core.info(`Ensuring host directory exists: ${homeOnnxDir}`); + try { + await fs.mkdir(homeOnnxDir, { recursive: true }); + core.info(`Host directory ${homeOnnxDir} ensured.`); + dockerArgs.push('--volume', `${homeOnnxDir}:${containerHomeDir}/.onnx`); + } catch (error) { + core.warning(`Could not ensure ${homeOnnxDir}: ${error.message}. Skipping mount.`); + } + } else { + core.info(`Mode is "${runMode}", skipping test data mounts.`); + } + dockerArgs.push('-w', '/onnxruntime_src'); + dockerArgs.push('-e', `ALLOW_RELEASED_ONNX_OPSET_ONLY=${allowOpset}`); + dockerArgs.push('-e', `NIGHTLY_BUILD=${nightlyBuild}`); + if (shouldPassCacheVars) { + /* ... cache var passing logic ... */ core.info('Passing cache env vars into container.'); + const cacheUrl = process.env.ACTIONS_CACHE_URL || ''; + const runtimeToken = process.env.ACTIONS_RUNTIME_TOKEN || ''; + if (cacheUrl) core.setSecret(cacheUrl); + if (runtimeToken) core.setSecret(runtimeToken); + if (cacheUrl) dockerArgs.push('-e', `ACTIONS_CACHE_URL=${cacheUrl}`); + else core.info('ACTIONS_CACHE_URL not found.'); + if (runtimeToken) dockerArgs.push('-e', `ACTIONS_RUNTIME_TOKEN=${runtimeToken}`); + else core.info('ACTIONS_RUNTIME_TOKEN not found.'); + // Map RUNNER_TEMP inside container correctly based on volume mount + dockerArgs.push('-e', 'RUNNER_TEMP=/onnxruntime_src/build'); + } else { + core.info('Skipping passing cache env vars.'); + } + dockerArgs.push(dockerImage); + // Pass the potentially modified command + dockerArgs.push('/bin/bash', '-c', fullDockerCommand); + + // --- Execute Docker Command (uses executeCommand util) --- + core.info('Executing docker command...'); + await executeCommand('docker', dockerArgs); // USE UTIL + core.info('Docker command executed successfully.'); + + // --- Verify Wheel Existence if --build_wheel was specified --- + if (isWheelBuild) { + core.startGroup('Verify Python Wheel Output'); + const wheelDir = path.join(buildOutputPath, 'dist'); // Use buildOutputPath + core.info(`Checking for wheel file in: ${wheelDir}`); + try { + await fs.access(wheelDir); // Check if directory exists first + core.info(`Directory ${wheelDir} exists. Searching for .whl file...`); + // Use glob to find files ending with .whl + const wheelGlobber = await glob.create(`${wheelDir}/*.whl`, { followSymbolicLinks: false }); + let wheelFound = false; + for await (const file of wheelGlobber.globGenerator()) { + core.info(`Found wheel file: ${path.basename(file)}`); + wheelFound = true; + // Usually only one wheel is expected, break after finding the first one + break; + } + + if (!wheelFound) { + core.warning(`Wheel directory ${wheelDir} exists, but no .whl file was found inside.`); + // Consider setting failed if wheel is critical? core.setFailed(...) + } + } catch (error) { + if (error.code === 'ENOENT') { + core.warning(`Wheel output directory ${wheelDir} does not exist.`); + // Consider setting failed if wheel is critical? core.setFailed(...) + } else { + core.warning(`Error checking for wheel file in ${wheelDir}: ${error.message}`); + } + } + core.endGroup(); + } + + } catch (error) { + core.setFailed(`Action failed with error: ${error.message}`); + // Error is caught, but finally block will still execute + } finally { + // --- Generate Test Summary from XML Files --- + // This runs regardless of whether the try block succeeded or failed + await generateTestSummary(buildOutputPath); // Pass the specific build output path } - dockerArgs.push(dockerImage); - dockerArgs.push('/bin/bash', '-c', `set -ex; ${fullBuildPyCommand}`); - - // --- Execute Docker Command (uses executeCommand util) --- - core.info('Executing docker command...'); - await executeCommand('docker', dockerArgs); // USE UTIL - core.info('Docker command executed successfully.'); - } catch (error) { - core.setFailed(`Action failed with error: ${error.message}`); - } } // Run main if executed directly if (require.main === module) { - run(); + run(); } // Export run for testing -module.exports = { run }; +module.exports = { run }; \ No newline at end of file From 24bdc290d017a8fa2290b8106ab6e3ce940f9255 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 7 Apr 2025 15:06:34 -0700 Subject: [PATCH 08/14] update --- src/build-minimal-ort-and-run-tests/index.js | 76 +++++++++---------- src/run-build-script-in-docker/index.js | 79 ++++++++++---------- 2 files changed, 77 insertions(+), 78 deletions(-) diff --git a/src/build-minimal-ort-and-run-tests/index.js b/src/build-minimal-ort-and-run-tests/index.js index ae5d4d0..9e5ee16 100644 --- a/src/build-minimal-ort-and-run-tests/index.js +++ b/src/build-minimal-ort-and-run-tests/index.js @@ -15,32 +15,32 @@ const os = require('os'); */ async function runCommand(command, args = [], options = {}) { // ... (runCommand function remains the same) ... -    const effectiveOptions = { -        cwd: process.env.GITHUB_WORKSPACE, // Default working directory -        ignoreReturnCode: false, // Throw error on failure by default -        silent: false, // Show command output by default -        listeners: { -            stdout: (data) => { core.info(data.toString().trim()); }, -            stderr: (data) => { core.warning(data.toString().trim()); } // Log stderr as warning by default -        }, -        ...options -    }; -    const cwdString = effectiveOptions.cwd === process.env.GITHUB_WORKSPACE ? 'default workspace' : effectiveOptions.cwd; -    core.info(`Executing in ${cwdString}: ${command} ${args.map(arg => arg.includes(' ') ? `"${arg}"` : arg).join(' ')}`); // Basic quoting for display -    try { -        const { exitCode, stdout, stderr } = await exec.getExecOutput(command, args, effectiveOptions); - -        if (exitCode !== 0 && !effectiveOptions.ignoreReturnCode) { -             core.error(`Stderr: ${stderr}`); -             throw new Error(`Command exited with code ${exitCode}: ${command} ${args.join(' ')}`); -        } -        core.info(`Finished: ${command} ${args.join(' ')}`); -        return { exitCode, stdout, stderr }; -    } catch (error) { -        core.error(`Error executing command: ${command} ${args.join(' ')} in ${cwdString}`); -        core.error(error); -        throw new Error(`Command execution failed: ${error.message || error}`); -    } + const effectiveOptions = { + cwd: process.env.GITHUB_WORKSPACE, // Default working directory + ignoreReturnCode: false, // Throw error on failure by default + silent: false, // Show command output by default + listeners: { + stdout: (data) => { core.info(data.toString().trim()); }, + stderr: (data) => { core.warning(data.toString().trim()); } // Log stderr as warning by default + }, + ...options + }; + const cwdString = effectiveOptions.cwd === process.env.GITHUB_WORKSPACE ? 'default workspace' : effectiveOptions.cwd; + core.info(`Executing in ${cwdString}: ${command} ${args.map(arg => arg.includes(' ') ? `"${arg}"` : arg).join(' ')}`); // Basic quoting for display + try { + const { exitCode, stdout, stderr } = await exec.getExecOutput(command, args, effectiveOptions); + + if (exitCode !== 0 && !effectiveOptions.ignoreReturnCode) { + core.error(`Stderr: ${stderr}`); + throw new Error(`Command exited with code ${exitCode}: ${command} ${args.join(' ')}`); + } + core.info(`Finished: ${command} ${args.join(' ')}`); + return { exitCode, stdout, stderr }; + } catch (error) { + core.error(`Error executing command: ${command} ${args.join(' ')} in ${cwdString}`); + core.error(error); + throw new Error(`Command execution failed: ${error.message || error}`); + } } @@ -51,18 +51,18 @@ async function runCommand(command, args = [], options = {}) { */ async function checkPathExists(pathToCheck) { // ... (checkPathExists function remains the same) ... -    try { -        await fs.access(pathToCheck); -        core.info(`Path exists: ${pathToCheck}`); -        return true; -    } catch (error) { -        if (error.code === 'ENOENT') { -            core.info(`Path does not exist: ${pathToCheck}`); -            return false; -        } -        core.warning(`Error checking path ${pathToCheck}: ${error.message}`); -        return false; // Assume not accessible on other errors -    } + try { + await fs.access(pathToCheck); + core.info(`Path exists: ${pathToCheck}`); + return true; + } catch (error) { + if (error.code === 'ENOENT') { + core.info(`Path does not exist: ${pathToCheck}`); + return false; + } + core.warning(`Error checking path ${pathToCheck}: ${error.message}`); + return false; // Assume not accessible on other errors + } } diff --git a/src/run-build-script-in-docker/index.js b/src/run-build-script-in-docker/index.js index 3490a41..c1eaf67 100644 --- a/src/run-build-script-in-docker/index.js +++ b/src/run-build-script-in-docker/index.js @@ -9,49 +9,49 @@ const { executeCommand, checkPathExists } = require('../common/utils'); // Impor // List of known EP names (keep local or move to common if used elsewhere?) - Keep local for now const KNOWN_EPS = new Set([ -  'acl', -  'armnn', -  'azure', -  'cann', -  'coreml', -  'cuda', -  'dml', -  'dnnl', -  'migraphx', -  'nnapi', -  'openvino', -  'qnn', -  'rknpu', -  'rocm', -  'snpe', -  'tensorrt', -  'vitisai', -  'vsinpu', -  'webgpu', -  'webnn', -  'xnnpack', + 'acl', + 'armnn', + 'azure', + 'cann', + 'coreml', + 'cuda', + 'dml', + 'dnnl', + 'migraphx', + 'nnapi', + 'openvino', + 'qnn', + 'rknpu', + 'rocm', + 'snpe', + 'tensorrt', + 'vitisai', + 'vsinpu', + 'webgpu', + 'webnn', + 'xnnpack', ]); const knownEpsString = Array.from(KNOWN_EPS).sort().join(', '); // checkGpu uses exec directly for failure tolerance/specific handling, keep as is or adapt helper? // Keep direct exec here as error handling is specific (warning, not throwing) async function checkGpu() { -  let hasGpu = false; -  try { -    const options = { ignoreReturnCode: true, silent: true }; // Don't fail action if nvidia-smi not found -    core.info('Checking for NVIDIA GPU using nvidia-smi...'); -    // Use direct exec here because we want to check the exit code and *not* throw -    const exitCode = await require('@actions/exec').exec('nvidia-smi', [], options); -    if (exitCode === 0) { -      core.info('NVIDIA GPU detected via nvidia-smi.'); -      hasGpu = true; -    } else { -      core.warning(`nvidia-smi command failed or not found (exit code: ${exitCode}). Assuming no GPU.`); -    } -  } catch (error) { -    core.warning(`Error trying to execute nvidia-smi: ${error.message}. Assuming no GPU.`); -  } -  return hasGpu; + let hasGpu = false; + try { + const options = { ignoreReturnCode: true, silent: true }; // Don't fail action if nvidia-smi not found + core.info('Checking for NVIDIA GPU using nvidia-smi...'); + // Use direct exec here because we want to check the exit code and *not* throw + const exitCode = await require('@actions/exec').exec('nvidia-smi', [], options); + if (exitCode === 0) { + core.info('NVIDIA GPU detected via nvidia-smi.'); + hasGpu = true; + } else { + core.warning(`nvidia-smi command failed or not found (exit code: ${exitCode}). Assuming no GPU.`); + } + } catch (error) { + core.warning(`Error trying to execute nvidia-smi: ${error.message}. Assuming no GPU.`); + } + return hasGpu; } // --- Helper Function to Generate Test Summary --- @@ -211,7 +211,7 @@ async function run() { for (const ep of requestedEps) { if (KNOWN_EPS.has(ep)) { const flag = `--use_${ep}`; - core.info(`  Adding flag: ${flag}`); + core.info(` Adding flag: ${flag}`); epFlags.push(flag); } else { core.setFailed(`Unknown EP: '${ep}'. Allowed: ${knownEpsString}`); @@ -288,8 +288,7 @@ async function run() { if (gpuAvailable) dockerArgs.push('--gpus', 'all'); core.info('Adding standard volume mounts: workspace, runner temp, host cache.'); dockerArgs.push('--volume', `${workspaceDir}:/onnxruntime_src`); - // Mount the parent of the build output dir for simplicity - dockerArgs.push('--volume', `${runnerTempDir}/build:/onnxruntime_src/build`); // Mount $RUNNER_TEMP/build -> /onnxruntime_src/build + dockerArgs.push('--volume', `${runnerTempDir}:/onnxruntime_src/build`); dockerArgs.push('--volume', `${hostCacheDir}:${containerHomeDir}/.cache`); // Use determined container home if (lowerCaseRunMode === 'test') { /* ... test volume mount logic ... */ core.info('Mode is "test", checking test data mounts.'); From d0e1599a42e9dc5438b1f1f92d81ea17abdd6c19 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 22 Apr 2025 14:37:26 -0700 Subject: [PATCH 09/14] update --- package-lock.json | 18 +-- package.json | 5 +- src/common/utils.js | 166 +++++++++++++++++++++ src/run-build-script-in-docker/index.js | 186 +++++++----------------- test/common/utils.test.js | 2 +- 5 files changed, 225 insertions(+), 152 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8ae1e5f..8c0bc2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,8 @@ "@actions/exec": "^1.1.1", "@actions/github": "^6.0.0", "@actions/glob": "^0.5.0", - "@actions/tool-cache": "^2.0.2" + "@actions/tool-cache": "^2.0.2", + "fast-xml-parser": "^5.2.0" }, "devDependencies": { "@eslint/eslintrc": "^3.3.1", @@ -3382,15 +3383,6 @@ "dev": true, "license": "ISC" }, - "node_modules/elfy": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/elfy/-/elfy-1.0.0.tgz", - "integrity": "sha512-4Kp3AA94jC085IJox+qnvrZ3PudqTi4gQNvIoTZfJJ9IqkRuCoqP60vCVYlIg00c5aYusi5Wjh2bf0cHYt+6gQ==", - "license": "MIT", - "dependencies": { - "endian-reader": "^0.3.0" - } - }, "node_modules/emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", @@ -3410,12 +3402,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, - "node_modules/endian-reader": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/endian-reader/-/endian-reader-0.3.0.tgz", - "integrity": "sha512-zPlHN59VLEjeJtpEU41ti/i7ZvTbwclvUN2M8anCsI3tOC/3mq6WNTJEKi49A5eLGvDkA0975LZb67Xwp7u4xQ==", - "license": "MIT" - }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", diff --git a/package.json b/package.json index b7e6223..79ef8ca 100644 --- a/package.json +++ b/package.json @@ -19,12 +19,13 @@ "author": "", "license": "MIT", "dependencies": { - "@actions/artifact": "^2.3.2", + "@actions/artifact": "^2.3.2", "@actions/core": "^1.11.1", "@actions/exec": "^1.1.1", "@actions/github": "^6.0.0", "@actions/glob": "^0.5.0", - "@actions/tool-cache": "^2.0.2" + "@actions/tool-cache": "^2.0.2", + "fast-xml-parser": "^5.2.0" }, "devDependencies": { "@eslint/eslintrc": "^3.3.1", diff --git a/src/common/utils.js b/src/common/utils.js index 925c86e..5097bfd 100644 --- a/src/common/utils.js +++ b/src/common/utils.js @@ -5,6 +5,8 @@ const exec = require('@actions/exec'); const crypto = require('node:crypto'); const fs = require('node:fs/promises'); const path = require('node:path'); +const glob = require('@actions/glob'); +const { XMLParser, XMLValidator } = require('fast-xml-parser'); // --- Execution Helper --- @@ -193,6 +195,169 @@ async function checkPathExists(filePath) { } } +// --- GitHub Actions Specific Helpers --- + +async function generateTestSummary(baseDir) { + core.startGroup('Generate Test Result Summary'); + const xmlPattern = path.join(baseDir, '**/*.results.xml').replace(/\\/g, '/'); // Normalize for glob + let totalTests = 0; + let totalFailures = 0; + let totalErrors = 0; + let totalSkipped = 0; + let filesProcessed = 0; + const failedFiles = []; + const processedFiles = []; + + // Configure the XML parser + const parserOptions = { + ignoreAttributes: false, // We need attributes + attributeNamePrefix: "", // Use attribute names directly (e.g., tests, not @​_tests) + parseAttributeValue: true, // Attempt to convert attribute values to numbers/booleans + allowBooleanAttributes: true, // If boolean attributes are used (e.g., disabled="true") + trimValues: true, // Trim whitespace from values + stopNodes: ["*.testcase"] // Optimization: Stop parsing deeper inside testcase tags + }; + const parser = new XMLParser(parserOptions); + + core.info(`Searching for test result files matching: ${xmlPattern}`); + + try { + // Check if base directory exists first + await fs.access(baseDir); // Throws if doesn't exist or no permissions + core.info(`Base directory ${baseDir} exists.`); + + const globber = await glob.create(xmlPattern, { followSymbolicLinks: false }); + for await (const file of globber.globGenerator()) { + filesProcessed++; + const fileName = path.relative(baseDir, file) || path.basename(file); // Get relative path for display + processedFiles.push(fileName); + core.debug(`Processing file: ${file}`); + let fileContent; + + try { + // 1. Read File + fileContent = await fs.readFile(file, 'utf8'); + + // 2. Validate XML Syntax (optional but recommended) + const validationResult = XMLValidator.validate(fileContent); + if (validationResult !== true) { + // Log detailed validation error + const err = validationResult.err; + core.warning(`Invalid XML syntax in ${fileName}: ${err.msg} (Line: ${err.line}, Col: ${err.col})`); + failedFiles.push(`${fileName} (syntax error)`); + continue; // Skip to next file + } + + // 3. Parse XML + const result = parser.parse(fileContent); + + // 4. Find top-level tag ( or ) and its attributes + // fast-xml-parser creates an object where the root tag is the key + const rootKey = Object.keys(result)[0]; // Get the first key (should be the root tag) + const topLevelTag = result[rootKey]; + + if (!rootKey || !topLevelTag) { + core.warning(`Could not find expected root tag (e.g., ) in ${fileName}.`); + failedFiles.push(`${fileName} (structure error - no root)`); + continue; // Skip to next file + } + + // Check if it's an array (multiple testsuite tags under testsuites) or single object + // We only care about the *summary* attributes on the top-level tag itself + const attrs = topLevelTag; // Attributes are properties with parseAttributeValue=true + + if (typeof attrs !== 'object' || attrs === null) { + core.warning(`Expected attributes object not found on root tag <${rootKey}> in ${fileName}.`); + failedFiles.push(`${fileName} (structure error - no attributes)`); + continue; + } + + + // 5. Extract counts safely (handle missing attributes, rely on parseAttributeValue) + const tests = Number(attrs.tests || 0); + const failures = Number(attrs.failures || 0); + const errors = Number(attrs.errors || 0); + // JUnit often uses 'disabled' but 'skipped' is also common. Prioritize 'skipped'. + const skipped = Number(attrs.skipped || attrs.disabled || 0); + + // Validate that extracted values are numbers + if (isNaN(tests) || isNaN(failures) || isNaN(errors) || isNaN(skipped)) { + core.warning(`Non-numeric test counts found in attributes of root tag <${rootKey}> in ${fileName}. Found: tests=${attrs.tests}, failures=${attrs.failures}, errors=${attrs.errors}, skipped/disabled=${attrs.skipped || attrs.disabled}`); + failedFiles.push(`${fileName} (attribute format error)`); + continue; // Skip counts from this file + } + + // 6. Add to totals + totalTests += tests; + totalFailures += failures; + totalErrors += errors; + totalSkipped += skipped; + core.debug( + ` -> Parsed Counts - Tests: ${tests}, Failures: ${failures}, Errors: ${errors}, Skipped/Disabled: ${skipped}` + ); + + } catch (error) { + // Catch errors from readFile, XMLValidator.validate, or parser.parse + // Log specific error codes if available + const errorCode = error.code ? ` (${error.code})` : ''; + core.warning(`Error processing file ${fileName}${errorCode}: ${error.message}`); + failedFiles.push(`${fileName} (processing error)`); + } + } // End for loop + + // --- Generate Summary Markdown --- + if (filesProcessed === 0) { + core.info('No test result XML files found.'); + } else { + core.info(`Processed ${filesProcessed} test result XML file(s).`); + let summaryMarkdown = `## Test Results Summary\n\n`; + summaryMarkdown += `Processed **${filesProcessed}** \`*.results.xml\` file(s) from \`${path.basename( + baseDir + )}\`.\n\n`; + + const totalProblems = totalFailures + totalErrors; + const overallStatus = totalProblems === 0 ? '✅ Passed' : '❌ Failed'; + + summaryMarkdown += `| Metric | Count |\n`; + summaryMarkdown += `| --------------- | ----: |\n`; + summaryMarkdown += `| **Total Tests** | ${totalTests} |\n`; + summaryMarkdown += `| Failures | ${ + totalFailures > 0 ? `**${totalFailures}** ❌` : totalFailures + } |\n`; + summaryMarkdown += `| Errors | ${totalErrors > 0 ? `**${totalErrors}** ❌` : totalErrors} |\n`; + summaryMarkdown += `| Skipped | ${totalSkipped} |\n`; + summaryMarkdown += `| **Overall** | **${overallStatus}** |\n\n`; + + if (failedFiles.length > 0) { + summaryMarkdown += `⚠️ **Issues processing some files:**\n`; + summaryMarkdown += failedFiles.map((f) => `- \`${f}\``).join('\n') + '\n\n'; + } + + summaryMarkdown += `
Processed Files (${processedFiles.length})\n\n`; + summaryMarkdown += processedFiles.map((f) => `- \`${f}\``).join('\n') + '\n'; + summaryMarkdown += `
\n`; + + // Add to GitHub Job Summary + try { + await core.summary.addRaw(summaryMarkdown, true).write(); + core.info("Test result summary added to GitHub Job Summary."); + } catch (summaryError) { + core.error(`Failed to write job summary: ${summaryError.message}`); + } + } + + } catch (error) { + // Handle errors accessing baseDir or creating globber + if (error.code === 'ENOENT') { + core.info(`Test result base directory ${baseDir} not found. Skipping summary generation.`); + } else { + core.error(`Error reading test result directory ${baseDir} or globbing files: ${error.message}`); + } + } finally { + core.endGroup(); + } +} + // Export all utility functions module.exports = { executeCommand, @@ -203,4 +368,5 @@ module.exports = { getPlatformIdentifier, getArchIdentifier, checkPathExists, + generateTestSummary, }; diff --git a/src/run-build-script-in-docker/index.js b/src/run-build-script-in-docker/index.js index c1eaf67..ec3ec28 100644 --- a/src/run-build-script-in-docker/index.js +++ b/src/run-build-script-in-docker/index.js @@ -5,7 +5,7 @@ const fs = require('node:fs/promises'); const glob = require('@actions/glob'); // Ensure glob is required // Import shared utilities -const { executeCommand, checkPathExists } = require('../common/utils'); // Import helpers +const { executeCommand, checkPathExists, generateTestSummary } = require('../common/utils'); // List of known EP names (keep local or move to common if used elsewhere?) - Keep local for now const KNOWN_EPS = new Set([ @@ -54,98 +54,7 @@ async function checkGpu() { return hasGpu; } -// --- Helper Function to Generate Test Summary --- -async function generateTestSummary(baseDir) { - core.startGroup('Generate Test Result Summary'); - const xmlPattern = path.join(baseDir, '**/*.results.xml').replace(/\\/g, '/'); // Normalize for glob - let totalTests = 0; - let totalFailures = 0; - let totalErrors = 0; - let totalSkipped = 0; - let filesProcessed = 0; - const failedFiles = []; - const processedFiles = []; - core.info(`Searching for test result files matching: ${xmlPattern}`); - - try { - // Check if base directory exists first - await fs.access(baseDir); - core.info(`Base directory ${baseDir} exists.`); - - const globber = await glob.create(xmlPattern, { followSymbolicLinks: false }); - for await (const file of globber.globGenerator()) { - filesProcessed++; - const fileName = path.relative(baseDir, file) || path.basename(file); // Get relative path for display - processedFiles.push(fileName); - core.debug(`Processing file: ${file}`); - try { - const content = await fs.readFile(file, 'utf8'); - // Simple Regex to find the first or tag and extract counts - // It assumes attributes are present and well-formed. Handles optional skipped/disabled. - const testsuiteRegex = /]*?tests="(\d+)"[^>]*?failures="(\d+)"[^>]*?errors="(\d+)"(?:[^>]*?(?:skipped|disabled)="(\d+)")?/; - const match = content.match(testsuiteRegex); - - if (match) { - totalTests += parseInt(match[1] || '0', 10); - totalFailures += parseInt(match[2] || '0', 10); - totalErrors += parseInt(match[3] || '0', 10); - totalSkipped += parseInt(match[4] || '0', 10); // Group 4 is skipped/disabled - core.debug(` -> Found Tests: ${match[1]}, Failures: ${match[2]}, Errors: ${match[3]}, Skipped/Disabled: ${match[4] || '0'}`); - } else { - core.warning(`Could not find parsable tag with counts in ${fileName}`); - failedFiles.push(`${fileName} (parse error)`); - } - } catch (readError) { - core.warning(`Error reading test result file ${fileName}: ${readError.message}`); - failedFiles.push(`${fileName} (read error)`); - } - } - - if (filesProcessed === 0) { - core.info('No test result XML files found.'); - } else { - core.info(`Processed ${filesProcessed} test result XML file(s).`); - // --- Generate Job Summary --- - let summaryMarkdown = `## Test Results Summary\n\n`; - summaryMarkdown += `Processed **${filesProcessed}** \`*.results.xml\` file(s) from \`${path.basename(baseDir)}\`.\n\n`; // Use basename for clarity - - const totalProblems = totalFailures + totalErrors; - const overallStatus = totalProblems === 0 ? '✅ Passed' : '❌ Failed'; - - summaryMarkdown += `| Metric | Count |\n`; - summaryMarkdown += `| --------------- | ----: |\n`; - summaryMarkdown += `| **Total Tests** | ${totalTests} |\n`; - summaryMarkdown += `| Failures | ${totalFailures > 0 ? `**${totalFailures}** ❌` : totalFailures} |\n`; - summaryMarkdown += `| Errors | ${totalErrors > 0 ? `**${totalErrors}** ❌` : totalErrors} |\n`; - summaryMarkdown += `| Skipped | ${totalSkipped} |\n`; - summaryMarkdown += `| **Overall** | **${overallStatus}** |\n\n`; - - - if (failedFiles.length > 0) { - summaryMarkdown += `⚠️ **Issues processing some files:**\n`; - summaryMarkdown += failedFiles.map(f => `- \`${f}\``).join('\n') + '\n\n'; - } - - summaryMarkdown += `
Processed Files (${processedFiles.length})\n\n`; - summaryMarkdown += processedFiles.map(f => `- \`${f}\``).join('\n') + '\n'; - summaryMarkdown += `
\n`; - - // Add to GitHub Job Summary (append=true ensures it adds even if step failed) - await core.summary.addRaw(summaryMarkdown, true).write(); - core.info("Test result summary added to GitHub Job Summary."); - } - - } catch (error) { - if (error.code === 'ENOENT') { - core.info(`Test result base directory ${baseDir} not found. Skipping summary generation.`); - } else { - core.warning(`Error accessing test result directory ${baseDir} or globbing files: ${error.message}`); - } - } finally { - core.endGroup(); - } -} async function run() { @@ -156,21 +65,21 @@ async function run() { core.setFailed('RUNNER_TEMP environment variable not set.'); // Fail early if this is missing return; } - const buildOutputPath = path.join(runnerTempDir, 'build', buildConfig); + const buildOutputPath = path.join(runnerTempDir, buildConfig); try { - // --- Get All Inputs (unchanged) --- + // --- Get All Inputs --- const dockerImage = core.getInput('docker_image', { required: true }); // buildConfig already obtained const runMode = core.getInput('mode', { required: true }); const containerUser = core.getInput('container_user'); const epInputString = core.getInput('execution_providers'); const extraBuildFlags = core.getInput('extra_build_flags'); - const pythonPathPrefix = core.getInput('python_path_prefix'); + const pythonPathPrefix = core.getInput('python_path_prefix'); // Get prefix input const allowOpset = core.getInput('allow_released_opset_only'); const nightlyBuild = core.getInput('nightly_build'); - // --- Validate Mode (unchanged) --- + // --- Validate Mode --- let buildPyArg; let shouldPassCacheVars = false; const lowerCaseRunMode = runMode.toLowerCase(); @@ -200,14 +109,14 @@ async function run() { core.info('Detected Python binding build based on extra_build_flags.'); } - // --- Process Execution Providers Input (unchanged) --- + // --- Process Execution Providers Input --- const epFlags = []; const requestedEps = epInputString .toLowerCase() .split(' ') .filter((ep) => ep.trim()); if (requestedEps.length > 0) { - /* ... EP flag logic ... */ core.info(`Requested EPs: ${requestedEps.join(', ')}`); + core.info(`Requested EPs: ${requestedEps.join(', ')}`); for (const ep of requestedEps) { if (KNOWN_EPS.has(ep)) { const flag = `--use_${ep}`; @@ -220,31 +129,28 @@ async function run() { } } - // --- Get Runner Context/Defaults (unchanged) --- + // --- Get Runner Context/Defaults --- const workspaceDir = process.env.GITHUB_WORKSPACE; - // runnerTempDir already obtained const homeDir = os.homedir(); const homeOnnxDir = path.join(homeDir, '.onnx'); const hostCacheDir = path.join(homeDir, '.cache'); - const containerHomeDir = containerUser ? `/home/${containerUser}` : '/root'; // Adjust based on user? Defaulting home might be tricky. + const containerHomeDir = containerUser ? `/home/${containerUser}` : '/root'; if (!workspaceDir) throw new Error('GITHUB_WORKSPACE not set.'); - // runnerTempDir checked earlier - // --- Check Host Paths (uses checkPathExists util) --- + // --- Check Host Paths --- const hostDataOnnxPath = '/data/onnx'; const hostDataModelsPath = '/data/models'; - const dataOnnxExists = await checkPathExists(hostDataOnnxPath); // USE UTIL - const dataModelsExists = await checkPathExists(hostDataModelsPath); // USE UTIL + const dataOnnxExists = await checkPathExists(hostDataOnnxPath); + const dataModelsExists = await checkPathExists(hostDataModelsPath); const enableOnnxTestsFlag = dataOnnxExists && dataModelsExists; core.info(`--enable_onnx_tests will be ${enableOnnxTestsFlag ? 'added' : 'skipped'}.`); - // --- Check for GPU (uses local checkGpu) --- + // --- Check for GPU --- const gpuAvailable = await checkGpu(); - // --- Construct build.py Command (unchanged) --- + // --- Construct build.py command part (without prefix) --- const buildPyBaseArgs = [ - pythonPathPrefix, - 'python3', + 'python3', // Prefix removed from here 'tools/ci_build/build.py', `--build_dir build/${buildConfig}`, // Relative path inside container `--config ${buildConfig}`, @@ -259,22 +165,41 @@ async function run() { extraBuildFlags, ]; const buildPyBase = buildPyBaseArgs.filter((part) => part).join(' '); - const fullBuildPyCommand = `${buildPyBase} ${buildPyArg}`; - core.debug(`Constructed build.py command: ${fullBuildPyCommand}`); + const buildPyCommandPart = `${buildPyBase} ${buildPyArg}`; + core.debug(`Build.py command part: ${buildPyCommandPart}`); + + // --- Construct the sequence of commands to run inside Docker --- + let commandSequence = []; - // --- Construct the full command to run inside Docker --- - let fullDockerCommand = `set -ex; ${fullBuildPyCommand}`; + // Add python requirements install if needed if (isPythonBuild) { const pythonRequirementsPath = 'tools/ci_build/github/linux/python/requirements.txt'; - const installReqsCommand = `python3 -m pip install --user -r ${pythonRequirementsPath}`; - fullDockerCommand = `set -ex; ${installReqsCommand} && ${fullBuildPyCommand}`; - core.info(`Prepending python requirements installation for Python build.`); - core.debug(`Full command inside Docker: ${fullDockerCommand}`); + const installReqsCommandPart = `python3 -m pip install --user -r ${pythonRequirementsPath}`; + commandSequence.push(installReqsCommandPart); + core.info(`Adding python requirements installation command.`); + } + + // Add the main build.py command + commandSequence.push(buildPyCommandPart); + + // Join commands with && + let combinedCommands = commandSequence.join(' && '); + + // Prepend the pythonPathPrefix if provided, applying it to the whole sequence + let fullDockerCommand; + const trimmedPrefix = pythonPathPrefix ? pythonPathPrefix.trim() : ''; + if (trimmedPrefix !== '') { + core.info(`Prepending python path prefix/environment setup: ${trimmedPrefix}`); + // Assume pythonPathPrefix is a valid shell command/assignment prefix like 'export PATH=...' or 'VAR=val' + fullDockerCommand = `set -ex; ${trimmedPrefix} && ${combinedCommands}`; } else { - core.debug(`Full command inside Docker: ${fullDockerCommand}`); + fullDockerCommand = `set -ex; ${combinedCommands}`; } - // --- Ensure Host Cache Directory Exists (unchanged) --- + core.debug(`Full command sequence inside Docker: ${fullDockerCommand}`); + + + // --- Ensure Host Cache Directory Exists --- core.info(`Ensuring host cache directory exists: ${hostCacheDir}`); try { await fs.mkdir(hostCacheDir, { recursive: true }); @@ -283,15 +208,15 @@ async function run() { core.warning(`Could not ensure host directory ${hostCacheDir} exists: ${error.message}.`); } - // --- Construct Docker Run Arguments (unchanged logic, uses modified command) --- + // --- Construct Docker Run Arguments --- const dockerArgs = ['run', '--rm']; if (gpuAvailable) dockerArgs.push('--gpus', 'all'); - core.info('Adding standard volume mounts: workspace, runner temp, host cache.'); + core.info('Adding standard volume mounts: workspace, runner temp build, host cache.'); dockerArgs.push('--volume', `${workspaceDir}:/onnxruntime_src`); dockerArgs.push('--volume', `${runnerTempDir}:/onnxruntime_src/build`); dockerArgs.push('--volume', `${hostCacheDir}:${containerHomeDir}/.cache`); // Use determined container home if (lowerCaseRunMode === 'test') { - /* ... test volume mount logic ... */ core.info('Mode is "test", checking test data mounts.'); + core.info('Mode is "test", checking test data mounts.'); if (dataOnnxExists) { dockerArgs.push('--volume', `${hostDataOnnxPath}:/data/onnx:ro`); } else { @@ -317,7 +242,7 @@ async function run() { dockerArgs.push('-e', `ALLOW_RELEASED_ONNX_OPSET_ONLY=${allowOpset}`); dockerArgs.push('-e', `NIGHTLY_BUILD=${nightlyBuild}`); if (shouldPassCacheVars) { - /* ... cache var passing logic ... */ core.info('Passing cache env vars into container.'); + core.info('Passing cache env vars into container.'); const cacheUrl = process.env.ACTIONS_CACHE_URL || ''; const runtimeToken = process.env.ACTIONS_RUNTIME_TOKEN || ''; if (cacheUrl) core.setSecret(cacheUrl); @@ -326,46 +251,41 @@ async function run() { else core.info('ACTIONS_CACHE_URL not found.'); if (runtimeToken) dockerArgs.push('-e', `ACTIONS_RUNTIME_TOKEN=${runtimeToken}`); else core.info('ACTIONS_RUNTIME_TOKEN not found.'); - // Map RUNNER_TEMP inside container correctly based on volume mount dockerArgs.push('-e', 'RUNNER_TEMP=/onnxruntime_src/build'); } else { core.info('Skipping passing cache env vars.'); } dockerArgs.push(dockerImage); - // Pass the potentially modified command + // Pass the full command sequence dockerArgs.push('/bin/bash', '-c', fullDockerCommand); - // --- Execute Docker Command (uses executeCommand util) --- + // --- Execute Docker Command --- core.info('Executing docker command...'); - await executeCommand('docker', dockerArgs); // USE UTIL + await executeCommand('docker', dockerArgs); core.info('Docker command executed successfully.'); // --- Verify Wheel Existence if --build_wheel was specified --- if (isWheelBuild) { core.startGroup('Verify Python Wheel Output'); - const wheelDir = path.join(buildOutputPath, 'dist'); // Use buildOutputPath + const wheelDir = path.join(buildOutputPath, 'dist'); core.info(`Checking for wheel file in: ${wheelDir}`); try { await fs.access(wheelDir); // Check if directory exists first core.info(`Directory ${wheelDir} exists. Searching for .whl file...`); - // Use glob to find files ending with .whl const wheelGlobber = await glob.create(`${wheelDir}/*.whl`, { followSymbolicLinks: false }); let wheelFound = false; for await (const file of wheelGlobber.globGenerator()) { core.info(`Found wheel file: ${path.basename(file)}`); wheelFound = true; - // Usually only one wheel is expected, break after finding the first one - break; + break; // Usually only one wheel is expected } if (!wheelFound) { core.warning(`Wheel directory ${wheelDir} exists, but no .whl file was found inside.`); - // Consider setting failed if wheel is critical? core.setFailed(...) } } catch (error) { if (error.code === 'ENOENT') { core.warning(`Wheel output directory ${wheelDir} does not exist.`); - // Consider setting failed if wheel is critical? core.setFailed(...) } else { core.warning(`Error checking for wheel file in ${wheelDir}: ${error.message}`); } diff --git a/test/common/utils.test.js b/test/common/utils.test.js index 5b9e6c7..4aaeb51 100644 --- a/test/common/utils.test.js +++ b/test/common/utils.test.js @@ -13,7 +13,7 @@ const utils = require('../../src/common/utils'); // Adjust path as needed // Mock dependencies jest.mock('@actions/core'); jest.mock('@actions/exec'); - +jest.mock('@actions/glob'); // Mocking crypto const mockUpdate = jest.fn(); const mockDigest = jest.fn(); From f8e8b8f540e58afe512b1c879038a2ca3fb91e1d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 23 Apr 2025 11:52:51 -0700 Subject: [PATCH 10/14] update --- actions/format-lint-check/README.md | 12 ++ actions/format-lint-check/action.yml | 18 ++ src/common/utils.js | 234 +++++++++++++++++-------- src/format-lint-check/index.js | 248 +++++++++++++++++++++++++++ 4 files changed, 438 insertions(+), 74 deletions(-) create mode 100644 actions/format-lint-check/README.md create mode 100644 actions/format-lint-check/action.yml create mode 100644 src/format-lint-check/index.js diff --git a/actions/format-lint-check/README.md b/actions/format-lint-check/README.md new file mode 100644 index 0000000..aa6ea5f --- /dev/null +++ b/actions/format-lint-check/README.md @@ -0,0 +1,12 @@ +# actions/format-lint-check/README.md + +# Format and Lint Check Action + +## Description + +This GitHub Action runs on Linux runners and performs the following tasks: + +1. **Installs LLVM/Clang:** Downloads a specific version of the official LLVM release binaries (`LLVM--Linux-X64.tar.xz`) from GitHub Releases, verifies its SHA256 hash, extracts it, caches it using `@actions/tool-cache`, and adds the `bin` directory to the `PATH`. +2. **Checks C/C++ Formatting:** Uses the installed `clang-format` to verify if C/C++ source files (`.h`, `.cc`, `.cpp`) within the specified directories adhere to the formatting rules defined by `.clang-format` files (searched upwards from file locations). It fails the action if any file needs formatting. +3. **Checks Shell Script Permissions:** Verifies that all found shell scripts (`.sh`) have at least one execute bit set. +4. **Checks Source/Text File Permissions:** Verifies that common source and text files (`.h`, `.cc`, `.cpp`, `.md`, `.txt`) do **not** have any execute bits set. diff --git a/actions/format-lint-check/action.yml b/actions/format-lint-check/action.yml new file mode 100644 index 0000000..1571a30 --- /dev/null +++ b/actions/format-lint-check/action.yml @@ -0,0 +1,18 @@ +name: 'Format and Lint Check' +description: 'Installs LLVM/Clang, checks C/C++/Shell formatting and permissions on Linux.' + +inputs: + llvm-version: + description: 'Required. The LLVM release tag to download (e.g., llvmorg-18.1.8).' + required: true + llvm-sha256-hash: + description: 'Required. The expected SHA256 hash of the LLVM Linux X64 tar.xz archive for verification.' + required: true + ignore-patterns: + description: 'Optional. Newline-separated list of glob patterns to ignore during file search (e.g., build/**\n**/external/**).' + required: false + default: '' + +runs: + using: 'node20' + main: 'dist/index.js' # This will point to the built file later \ No newline at end of file diff --git a/src/common/utils.js b/src/common/utils.js index 5097bfd..c71407b 100644 --- a/src/common/utils.js +++ b/src/common/utils.js @@ -199,7 +199,7 @@ async function checkPathExists(filePath) { async function generateTestSummary(baseDir) { core.startGroup('Generate Test Result Summary'); - const xmlPattern = path.join(baseDir, '**/*.results.xml').replace(/\\/g, '/'); // Normalize for glob + const xmlPattern = path.join(baseDir, '**/*.results.xml').replace(/\\/g, '/'); let totalTests = 0; let totalFailures = 0; let totalErrors = 0; @@ -207,109 +207,173 @@ async function generateTestSummary(baseDir) { let filesProcessed = 0; const failedFiles = []; const processedFiles = []; + const allTestCases = []; // <--- NEW: Array to store individual test cases {suite, case, time} - // Configure the XML parser const parserOptions = { - ignoreAttributes: false, // We need attributes - attributeNamePrefix: "", // Use attribute names directly (e.g., tests, not @​_tests) - parseAttributeValue: true, // Attempt to convert attribute values to numbers/booleans - allowBooleanAttributes: true, // If boolean attributes are used (e.g., disabled="true") - trimValues: true, // Trim whitespace from values - stopNodes: ["*.testcase"] // Optimization: Stop parsing deeper inside testcase tags + ignoreAttributes: false, + attributeNamePrefix: "", // No prefix for attributes + parseAttributeValue: true, // Convert attribute values to primitive types if possible + allowBooleanAttributes: true, + trimValues: true, + ignoreDeclaration: true }; const parser = new XMLParser(parserOptions); core.info(`Searching for test result files matching: ${xmlPattern}`); try { - // Check if base directory exists first - await fs.access(baseDir); // Throws if doesn't exist or no permissions + await fs.access(baseDir); core.info(`Base directory ${baseDir} exists.`); const globber = await glob.create(xmlPattern, { followSymbolicLinks: false }); for await (const file of globber.globGenerator()) { filesProcessed++; - const fileName = path.relative(baseDir, file) || path.basename(file); // Get relative path for display + const fileName = path.relative(baseDir, file) || path.basename(file); processedFiles.push(fileName); - core.debug(`Processing file: ${file}`); + core.debug(`--- Processing file: ${fileName} ---`); let fileContent; + // Reset counters for the current file before processing + let fileTests = 0, fileFailures = 0, fileErrors = 0, fileSkipped = 0; + try { - // 1. Read File fileContent = await fs.readFile(file, 'utf8'); - - // 2. Validate XML Syntax (optional but recommended) const validationResult = XMLValidator.validate(fileContent); if (validationResult !== true) { - // Log detailed validation error const err = validationResult.err; core.warning(`Invalid XML syntax in ${fileName}: ${err.msg} (Line: ${err.line}, Col: ${err.col})`); failedFiles.push(`${fileName} (syntax error)`); - continue; // Skip to next file + continue; } - // 3. Parse XML const result = parser.parse(fileContent); + core.debug(`Parsed XML structure: ${JSON.stringify(result, null, 2)}`); - // 4. Find top-level tag ( or ) and its attributes - // fast-xml-parser creates an object where the root tag is the key - const rootKey = Object.keys(result)[0]; // Get the first key (should be the root tag) - const topLevelTag = result[rootKey]; - - if (!rootKey || !topLevelTag) { - core.warning(`Could not find expected root tag (e.g., ) in ${fileName}.`); - failedFiles.push(`${fileName} (structure error - no root)`); - continue; // Skip to next file + const rootKey = Object.keys(result)[0]; + if (!rootKey) { + core.warning(`Could not determine root tag in ${fileName}.`); + failedFiles.push(`${fileName} (structure error - no root key)`); + continue; } - - // Check if it's an array (multiple testsuite tags under testsuites) or single object - // We only care about the *summary* attributes on the top-level tag itself - const attrs = topLevelTag; // Attributes are properties with parseAttributeValue=true - - if (typeof attrs !== 'object' || attrs === null) { - core.warning(`Expected attributes object not found on root tag <${rootKey}> in ${fileName}.`); - failedFiles.push(`${fileName} (structure error - no attributes)`); - continue; + const topLevelData = result[rootKey]; // This is the object/array under the root tag + + core.debug(`Root Key: ${rootKey}, Top Level Data Type: ${typeof topLevelData}`); + + let suitesToProcess = []; + + if (rootKey === 'testsuites') { + // Root is , contains potentially multiple + if (topLevelData && topLevelData.testsuite) { + suitesToProcess = Array.isArray(topLevelData.testsuite) + ? topLevelData.testsuite + : [topLevelData.testsuite]; + core.debug(`Found ${suitesToProcess.length} elements under `); + } else { + core.debug(` root found, but no child elements detected.`); + // Check attributes for summary (less common) + if (typeof topLevelData === 'object' && topLevelData !== null) { + const attrs = topLevelData; + const t = Number(attrs.tests || 0); + const f = Number(attrs.failures || 0); + const e = Number(attrs.errors || 0); + const s = Number(attrs.skipped || attrs.disabled || 0); + if (!isNaN(t) && !isNaN(f) && !isNaN(e) && !isNaN(s) && (t > 0 || f > 0 || e > 0 || s > 0)) { + fileTests = t; fileFailures = f; fileErrors = e; fileSkipped = s; + core.debug(`Using summary counts found directly on tag.`); + } + } + } + } else if (rootKey === 'testsuite') { + // Root is a single + if (topLevelData) { + suitesToProcess = [topLevelData]; // Process this single suite + core.debug(`Root is a single element.`); + } + } else { + core.warning(`Unexpected root tag <${rootKey}> found in ${fileName}. Skipping counts.`); + failedFiles.push(`${fileName} (unexpected root tag)`); + continue; // Skip count aggregation for this file } + // Iterate through the identified testsuite objects + for (const suite of suitesToProcess) { + if (typeof suite === 'object' && suite !== null) { + core.debug(`Processing suite object: ${JSON.stringify(suite, null, 2)}`); + const suiteName = suite.name || 'UnknownSuite'; // Get suite name + const attrs = suite; + + // Aggregate counts from suite attributes + const t = Number(attrs.tests || 0); + const f = Number(attrs.failures || 0); + const e = Number(attrs.errors || 0); + const s = Number(attrs.skipped ?? attrs.disabled ?? 0); + + if (isNaN(t) || isNaN(f) || isNaN(e) || isNaN(s)) { + core.warning(`Non-numeric test counts found in attributes of a tag in ${fileName}.`); + } else { + fileTests += t; + fileFailures += f; + fileErrors += e; + fileSkipped += s; + } + + // --- NEW: Process individual test cases --- + if (suite.testcase) { + const testCases = Array.isArray(suite.testcase) ? suite.testcase : [suite.testcase]; + core.debug(` Found ${testCases.length} elements in suite "${suiteName}"`); + for (const tc of testCases) { + if (typeof tc === 'object' && tc !== null) { + const caseName = tc.name; + const caseTime = tc.time; // Should be parsed as number by parseAttributeValue: true + + // Check if data is valid for sorting + if (caseName && typeof caseTime === 'number' && !isNaN(caseTime)) { + allTestCases.push({ + suite: suiteName, + case: caseName, + time: caseTime + }); + core.debug(` Added test case: ${suiteName} / ${caseName} / ${caseTime}`); + } else { + core.debug(` Skipping test case with missing name or invalid time: ${JSON.stringify(tc)}`); + } + } + } + } else { + core.debug(` No elements found in suite "${suiteName}"`); + } + // --- End NEW --- + + } else { + core.warning(`Encountered non-object item in suitesToProcess for ${fileName}. Skipping item.`); + } + } // End loop through suitesToProcess + + // Add file totals to overall totals outside the suite loop + totalTests += fileTests; + totalFailures += fileFailures; + totalErrors += fileErrors; + totalSkipped += fileSkipped; + core.debug(` -> Aggregated Counts for ${fileName} - Tests: ${fileTests}, Failures: ${fileFailures}, Errors: ${fileErrors}, Skipped: ${fileSkipped}`); + core.debug(` -> Running Totals - Tests: ${totalTests}, Failures: ${totalFailures}, Errors: ${totalErrors}, Skipped: ${totalSkipped}`); - // 5. Extract counts safely (handle missing attributes, rely on parseAttributeValue) - const tests = Number(attrs.tests || 0); - const failures = Number(attrs.failures || 0); - const errors = Number(attrs.errors || 0); - // JUnit often uses 'disabled' but 'skipped' is also common. Prioritize 'skipped'. - const skipped = Number(attrs.skipped || attrs.disabled || 0); - - // Validate that extracted values are numbers - if (isNaN(tests) || isNaN(failures) || isNaN(errors) || isNaN(skipped)) { - core.warning(`Non-numeric test counts found in attributes of root tag <${rootKey}> in ${fileName}. Found: tests=${attrs.tests}, failures=${attrs.failures}, errors=${attrs.errors}, skipped/disabled=${attrs.skipped || attrs.disabled}`); - failedFiles.push(`${fileName} (attribute format error)`); - continue; // Skip counts from this file - } - - // 6. Add to totals - totalTests += tests; - totalFailures += failures; - totalErrors += errors; - totalSkipped += skipped; - core.debug( - ` -> Parsed Counts - Tests: ${tests}, Failures: ${failures}, Errors: ${errors}, Skipped/Disabled: ${skipped}` - ); } catch (error) { - // Catch errors from readFile, XMLValidator.validate, or parser.parse - // Log specific error codes if available const errorCode = error.code ? ` (${error.code})` : ''; core.warning(`Error processing file ${fileName}${errorCode}: ${error.message}`); + core.debug(error.stack); // Log stack for debug failedFiles.push(`${fileName} (processing error)`); } - } // End for loop + core.debug(`--- Finished processing file: ${fileName} ---`); + } // End for loop over files // --- Generate Summary Markdown --- if (filesProcessed === 0) { core.info('No test result XML files found.'); } else { core.info(`Processed ${filesProcessed} test result XML file(s).`); + + // --- Generate overall summary table --- let summaryMarkdown = `## Test Results Summary\n\n`; summaryMarkdown += `Processed **${filesProcessed}** \`*.results.xml\` file(s) from \`${path.basename( baseDir @@ -318,16 +382,40 @@ async function generateTestSummary(baseDir) { const totalProblems = totalFailures + totalErrors; const overallStatus = totalProblems === 0 ? '✅ Passed' : '❌ Failed'; - summaryMarkdown += `| Metric | Count |\n`; - summaryMarkdown += `| --------------- | ----: |\n`; - summaryMarkdown += `| **Total Tests** | ${totalTests} |\n`; - summaryMarkdown += `| Failures | ${ + summaryMarkdown += `| Metric | Count |\n`; + summaryMarkdown += `| ------------- | ----: |\n`; + summaryMarkdown += `| Total Tests | ${totalTests} |\n`; + summaryMarkdown += `| Failures | ${ totalFailures > 0 ? `**${totalFailures}** ❌` : totalFailures } |\n`; - summaryMarkdown += `| Errors | ${totalErrors > 0 ? `**${totalErrors}** ❌` : totalErrors} |\n`; - summaryMarkdown += `| Skipped | ${totalSkipped} |\n`; + summaryMarkdown += `| Errors | ${totalErrors > 0 ? `**${totalErrors}** ❌` : totalErrors} |\n`; + summaryMarkdown += `| Skipped | ${totalSkipped} |\n`; summaryMarkdown += `| **Overall** | **${overallStatus}** |\n\n`; + // --- NEW: Generate Top 10 Slowest Tests --- + if (allTestCases.length > 0) { + // Sort by time descending + allTestCases.sort((a, b) => b.time - a.time); + const slowestTests = allTestCases.slice(0, 10); + + summaryMarkdown += `### Top ${slowestTests.length} Slowest Tests\n\n`; + summaryMarkdown += `| Rank | Time (s) | Suite Name | Test Case Name |\n`; + summaryMarkdown += `| ---- | -------- | --------------- | --------------- |\n`; + slowestTests.forEach((test, index) => { + // Format time to 3 decimal places + const timeFormatted = test.time.toFixed(3); + // Escape pipe characters in names to prevent breaking markdown table + const suiteEscaped = test.suite.replace(/\|/g, '\\|'); + const caseEscaped = test.case.replace(/\|/g, '\\|'); + summaryMarkdown += `| ${index + 1} | ${timeFormatted} | ${suiteEscaped} | ${caseEscaped} |\n`; + }); + summaryMarkdown += '\n'; + } else { + summaryMarkdown += `No individual test case times found to determine slowest tests.\n\n`; + } + // --- End NEW --- + + if (failedFiles.length > 0) { summaryMarkdown += `⚠️ **Issues processing some files:**\n`; summaryMarkdown += failedFiles.map((f) => `- \`${f}\``).join('\n') + '\n\n'; @@ -337,7 +425,6 @@ async function generateTestSummary(baseDir) { summaryMarkdown += processedFiles.map((f) => `- \`${f}\``).join('\n') + '\n'; summaryMarkdown += `\n`; - // Add to GitHub Job Summary try { await core.summary.addRaw(summaryMarkdown, true).write(); core.info("Test result summary added to GitHub Job Summary."); @@ -347,12 +434,11 @@ async function generateTestSummary(baseDir) { } } catch (error) { - // Handle errors accessing baseDir or creating globber - if (error.code === 'ENOENT') { - core.info(`Test result base directory ${baseDir} not found. Skipping summary generation.`); - } else { - core.error(`Error reading test result directory ${baseDir} or globbing files: ${error.message}`); - } + if (error.code === 'ENOENT') { + core.info(`Test result base directory ${baseDir} not found. Skipping summary generation.`); + } else { + core.error(`Error reading test result directory ${baseDir} or globbing files: ${error.message}`); + } } finally { core.endGroup(); } diff --git a/src/format-lint-check/index.js b/src/format-lint-check/index.js new file mode 100644 index 0000000..282f1d1 --- /dev/null +++ b/src/format-lint-check/index.js @@ -0,0 +1,248 @@ +const core = require('@actions/core'); +const exec = require('@actions/exec'); +const tc = require('@actions/tool-cache'); +const glob = require('@actions/glob'); +const path = require('node:path'); +const fs = require('node:fs/promises'); +const crypto = require('node:crypto'); +const os = require('node:os'); + +// --- SHA256 Verification Helper --- +/** + * Verifies the SHA256 hash of a file. + * @param {string} filePath - Path to the file to verify. + * @param {string} expectedHash - The expected SHA256 hash in hex format. + * @returns {Promise} - True if the hash matches, false otherwise. Throws on file access error. + */ +async function verifySHA256(filePath, expectedHash) { + core.info(`Calculating SHA256 for file: ${filePath}`); + const hash = crypto.createHash('sha256'); + // Use fs.createReadStream for potentially large files + const stream = require('node:fs').createReadStream(filePath); // Sync require is ok inside async func + + return new Promise((resolve, reject) => { + stream.on('error', (err) => reject(new Error(`Error reading file ${filePath}: ${err.message}`))); + stream.on('data', (chunk) => hash.update(chunk)); + stream.on('end', () => { + const actualHash = hash.digest('hex'); + core.debug(`Actual SHA256: ${actualHash}`); + core.debug(`Expected SHA256: ${expectedHash}`); + const match = actualHash.toLowerCase() === expectedHash.toLowerCase(); + core.info(`SHA256 Verification Result for ${path.basename(filePath)}: ${match ? 'Match' : 'Mismatch'}`); + resolve(match); + }); + }); +} + +// --- File Permission Check Helpers --- +/** + * Checks if a file has execute permissions set for user, group, or others. + * @param {string} filePath - The path to the file. + * @returns {Promise} - True if any execute bit is set, false otherwise. + */ +async function hasExecutePermission(filePath) { + try { + const stats = await fs.stat(filePath); + // Check if any of the execute bits (owner, group, others) are set + // Mode is a number, use bitwise AND with 0o111 (binary 001 001 001) + return (stats.mode & 0o111) !== 0; + } catch (error) { + core.warning(`Could not stat file ${filePath} for permissions check: ${error.message}`); + return false; // Treat inaccessible files as not having execute permission for safety + } +} + +/** + * Finds C/C++ header and source files, filtering out paths matching ignore patterns. + * + * @param {string[]} ignorePatterns An array of glob patterns to ignore. + * @returns {Promise} A promise that resolves with an array of file paths. + */ +async function findCFiles(ignorePatterns) { + // Define the glob patterns for C/C++ files + const patterns = [ + '**/*.c', // C source files + '**/*.cpp', // C++ source files + '**/*.cc', // C++ source files (alternative extension) + '**/*.h', // C/C++ header files + '**/*.hpp' // C++ header files (alternative extension) + ]; + + // Combine the include patterns and ignore patterns + // The globber will automatically handle ignoring files matching ignorePatterns + const allPatterns = patterns.concat( + ignorePatterns.map(pattern => `!${pattern}`) // Prepend '!' to ignore patterns + ); + + try { + // Create a globber instance with the combined patterns + // followSymbolicLinks: false is generally recommended for safety/performance + // in actions, unless specifically needed. + const globber = await glob.create(allPatterns.join('\n'), { + followSymbolicLinks: false, + implicitDescendants: false, + matchDirectories: false, + excludeHiddenFiles: true, + }); + + // Execute the glob search + console.log(`Searching for C/C++ files, ignoring: ${ignorePatterns.join(', ')}`); + const files = await globber.glob(); + console.log(`Found ${files.length} files.`); + + // Return the array of found file paths + return files; + + } catch (error) { + console.error("Error finding files:", error); + // Re-throw the error or handle it as appropriate for your action + throw error; + } +} + +// --- Main Orchestration Function --- +async function run() { + core.info('Starting Format and Lint Check Action...'); + if (process.platform !== 'linux') { + core.setFailed(`This action only runs on Linux. Detected platform: ${process.platform}`); + return; + } + + let llvmToolPath = ''; + let clangFormatPath = ''; // Will hold the absolute path or just 'clang-format' if in PATH + const filesNeedingFormatting = []; + const filesWithIncorrectPermissions = []; + + try { + // --- Get Inputs (remains the same) --- + const llvmVersion = core.getInput('llvm-version', { required: true }); + const llvmHash = core.getInput('llvm-sha256-hash', { required: true }); + const ignorePatternsInput = core.getInput('ignore-patterns'); + const ignorePatterns = ignorePatternsInput ? ignorePatternsInput.split('\n').filter(p => p.trim()) : []; + + // --- Setup LLVM/Clang (remains the same) --- + core.startGroup(`Setup LLVM/Clang (${llvmVersion})`); + const toolName = 'llvm'; + llvmToolPath = tc.find(toolName, llvmVersion); + if (llvmToolPath) { + core.info(`Found cached LLVM ${llvmVersion} at: ${llvmToolPath}`); + } else { + // ... (Download, Verify Hash, Extract, Cache logic remains the same) ... + core.info(`LLVM ${llvmVersion} not found in cache. Downloading...`); + const archiveFileName = `LLVM-${llvmVersion}-Linux-X64.tar.xz`; + const downloadUrl = `https://github.com/llvm/llvm-project/releases/download/llvmorg-${llvmVersion}/${archiveFileName}`; + core.info(`Download URL: ${downloadUrl}`); + let downloadedArchivePath; + try { + downloadedArchivePath = await tc.downloadTool(downloadUrl); + } catch (downloadError) { + core.error(`Failed to download LLVM archive: ${downloadError.message}`); + if (downloadError.message.includes('404')) { + core.error(`Please double-check the llvm-version ('${llvmVersion}') and that a 'Linux-X64.tar.xz' asset exists for it.`); + } + throw downloadError; + } + core.info(`Downloaded archive to: ${downloadedArchivePath}`); + core.info('Verifying SHA256 hash...'); + const hashMatch = await verifySHA256(downloadedArchivePath, llvmHash); + if (!hashMatch) throw new Error('LLVM archive SHA256 hash verification failed!'); + core.info('SHA256 hash verification successful.'); + core.info(`Extracting ${archiveFileName}...`); + const extractedPath = await tc.extractTar(downloadedArchivePath, undefined, ['-xJ']); + core.info(`Extracted archive to temporary location: ${extractedPath}`); + const dirs = await fs.readdir(extractedPath, { withFileTypes: true }); + const llvmSubDir = dirs.find(d => d.isDirectory() && d.name.startsWith('LLVM-')); + if (!llvmSubDir) throw new Error(`Could not find expected LLVM subdirectory inside ${extractedPath}`); + const actualExtractedRoot = path.join(extractedPath, llvmSubDir.name); + core.info(`Found extracted LLVM root: ${actualExtractedRoot}`); + core.info(`Caching directory: ${actualExtractedRoot}`); + llvmToolPath = await tc.cacheDir(actualExtractedRoot, toolName, llvmVersion); + core.info(`Successfully cached LLVM to: ${llvmToolPath}`); + } + const llvmBinPath = path.join(llvmToolPath, 'bin'); + if (!await fs.stat(llvmBinPath).then(s => s.isDirectory()).catch(() => false)) { + throw new Error(`LLVM 'bin' directory not found at: ${llvmBinPath}`); + } + core.info(`Adding LLVM bin directory to PATH: ${llvmBinPath}`); + core.addPath(llvmBinPath); + try { + // Verify clang-format exists in PATH *and* store the command name/path + // Use 'which' to confirm it's findable and potentially get full path if needed later + let whichOutput = ''; + await exec.exec('which', ['clang-format'], { silent: true, listeners: { stdout: (data) => { whichOutput += data.toString(); } } }); + clangFormatPath = whichOutput.trim(); // Store the path/command found + if (!clangFormatPath) throw new Error("'which clang-format' failed or returned empty."); + core.info(`Using clang-format found at: ${clangFormatPath}`); // Log the path being used + core.info('Running clang-format --version:'); + await exec.exec(clangFormatPath, ['--version']); + } catch (err) { + throw new Error(`clang-format not found in PATH or failed to execute after adding ${llvmBinPath}. Error: ${err.message}`); + } + core.endGroup(); // End LLVM setup + + core.startGroup('Checking C/C++ Formatting with clang-format'); + // Ensure files are absolute paths for clang-format when using --files + const filesToCheckFormatting = await findCFiles(ignorePatterns); + + if (filesToCheckFormatting.length === 0) { + core.info('No C/C++ files (.h, .cc, .cpp) found to format.'); + } else { + core.info(`Checking formatting for ${filesToCheckFormatting.length} C/C++ file(s) using --files argument...`); + + // 1. Create a temporary file path + const tempDir = process.env.RUNNER_TEMP || os.tmpdir(); // Prefer RUNNER_TEMP + const tempFileName = `clang-format-files-${Date.now()}-${crypto.randomBytes(4).toString('hex')}.txt`; + tempFilePath = path.join(tempDir, tempFileName); + core.debug(`Creating temporary file list at: ${tempFilePath}`); + + // 2. Write the absolute file paths to the temporary file (one per line) + const fileListContent = filesToCheckFormatting.join('\n'); + await fs.writeFile(tempFilePath, fileListContent, { encoding: 'utf8' }); + core.debug(`Wrote ${filesToCheckFormatting.length} filenames to ${tempFilePath}`); + + // 3. Prepare clang-format arguments + const clangFormatArgs = ['--dry-run', '-Werror', `--files=${tempFilePath}`]; + + // Log the command structure for debugging + core.debug(`Executing: ${clangFormatPath} ${clangFormatArgs.join(' ')}`); + + let clangFormatOutput = ''; // Capture output + const options = { + ignoreReturnCode: true, // Check exit code manually + silent: false, // Allow clang-format output to show + listeners: { + stdout: (data) => { clangFormatOutput += data.toString(); }, + stderr: (data) => { clangFormatOutput += data.toString(); } + } + }; + + // 4. Execute clang-format + const exitCode = await exec.exec(clangFormatPath, clangFormatArgs, options); + + if (exitCode !== 0) { + core.error(`clang-format check failed (exit code: ${exitCode}). Some files need formatting.`); + core.warning(`clang-format output (potentially noisy):\n${clangFormatOutput}`); + + // Collect filenames for the error message (optional but helpful) + const filesList = filesToCheckFormatting + .map(f => path.relative(process.env.GITHUB_WORKSPACE || '.', f)) + .join(', '); // Join for a concise list, adjust if needed + + core.setFailed(`Clang-format check failed. The following files need formatting: ${filesList}. Please run clang-format ${llvmVersion} locally.`); + } else { + core.info('clang-format check passed. All checked C/C++ files are correctly formatted.'); + } + } + core.endGroup(); + } catch (error) { + core.setFailed(`Action failed: ${error.message}`); + core.debug(error.stack); // Log stack trace on debug + } +} + +// --- Run --- +if (require.main === module) { + run(); +} + +module.exports = { run }; \ No newline at end of file From 1feee7a8d3f82103aca4c523bc09c0bbd760ca0a Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 23 Apr 2025 11:57:32 -0700 Subject: [PATCH 11/14] update --- src/build-minimal-ort-and-run-tests/index.js | 4 +--- src/format-lint-check/index.js | 11 +++-------- src/setup-build-tools/index.js | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/build-minimal-ort-and-run-tests/index.js b/src/build-minimal-ort-and-run-tests/index.js index 9e5ee16..90e0cd4 100644 --- a/src/build-minimal-ort-and-run-tests/index.js +++ b/src/build-minimal-ort-and-run-tests/index.js @@ -14,7 +14,6 @@ const os = require('os'); * @returns {Promise<{exitCode: number, stdout: string, stderr: string}>} Command output. */ async function runCommand(command, args = [], options = {}) { - // ... (runCommand function remains the same) ... const effectiveOptions = { cwd: process.env.GITHUB_WORKSPACE, // Default working directory ignoreReturnCode: false, // Throw error on failure by default @@ -50,7 +49,6 @@ async function runCommand(command, args = [], options = {}) { * @returns {Promise} True if the path exists, false otherwise. */ async function checkPathExists(pathToCheck) { - // ... (checkPathExists function remains the same) ... try { await fs.access(pathToCheck); core.info(`Path exists: ${pathToCheck}`); @@ -241,7 +239,7 @@ async function main() { // Add threshold if provided (assuming the script takes it as an optional arg) const sizeThreshold = core.getInput('size-threshold'); if (sizeThreshold) { - sizeCheckArgs.push('--threshold', sizeThreshold); // Adjust arg name if needed + sizeCheckArgs.push('--threshold', sizeThreshold); } await runCommand('python3', sizeCheckArgs); } catch (error) { diff --git a/src/format-lint-check/index.js b/src/format-lint-check/index.js index 282f1d1..45532cc 100644 --- a/src/format-lint-check/index.js +++ b/src/format-lint-check/index.js @@ -114,20 +114,17 @@ async function run() { const filesWithIncorrectPermissions = []; try { - // --- Get Inputs (remains the same) --- const llvmVersion = core.getInput('llvm-version', { required: true }); const llvmHash = core.getInput('llvm-sha256-hash', { required: true }); const ignorePatternsInput = core.getInput('ignore-patterns'); const ignorePatterns = ignorePatternsInput ? ignorePatternsInput.split('\n').filter(p => p.trim()) : []; - // --- Setup LLVM/Clang (remains the same) --- core.startGroup(`Setup LLVM/Clang (${llvmVersion})`); const toolName = 'llvm'; llvmToolPath = tc.find(toolName, llvmVersion); if (llvmToolPath) { core.info(`Found cached LLVM ${llvmVersion} at: ${llvmToolPath}`); } else { - // ... (Download, Verify Hash, Extract, Cache logic remains the same) ... core.info(`LLVM ${llvmVersion} not found in cache. Downloading...`); const archiveFileName = `LLVM-${llvmVersion}-Linux-X64.tar.xz`; const downloadUrl = `https://github.com/llvm/llvm-project/releases/download/llvmorg-${llvmVersion}/${archiveFileName}`; @@ -222,21 +219,19 @@ async function run() { if (exitCode !== 0) { core.error(`clang-format check failed (exit code: ${exitCode}). Some files need formatting.`); core.warning(`clang-format output (potentially noisy):\n${clangFormatOutput}`); - - // Collect filenames for the error message (optional but helpful) + const filesList = filesToCheckFormatting .map(f => path.relative(process.env.GITHUB_WORKSPACE || '.', f)) - .join(', '); // Join for a concise list, adjust if needed + .join(', '); core.setFailed(`Clang-format check failed. The following files need formatting: ${filesList}. Please run clang-format ${llvmVersion} locally.`); } else { - core.info('clang-format check passed. All checked C/C++ files are correctly formatted.'); + core.info('clang-format check passed. All checked C/C++ files are correctly formatted.'); } } core.endGroup(); } catch (error) { core.setFailed(`Action failed: ${error.message}`); - core.debug(error.stack); // Log stack trace on debug } } diff --git a/src/setup-build-tools/index.js b/src/setup-build-tools/index.js index b90e9e4..a41e53d 100644 --- a/src/setup-build-tools/index.js +++ b/src/setup-build-tools/index.js @@ -7,7 +7,7 @@ const fs = require('node:fs'); const os = require('node:os'); // Import shared utilities -const { executeCommand, verifySHA512, getPlatformIdentifier, getArchIdentifier } = require('../common/utils'); // Adjust path if necessary +const { executeCommand, verifySHA512, getPlatformIdentifier, getArchIdentifier } = require('../common/utils'); // --- CMake Specific Helper --- async function getLatestCMakeVersion(githubToken) { From 8f84913f59e9ca4da1b4b3955a5aa6df2a5bd873 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 23 Apr 2025 11:58:28 -0700 Subject: [PATCH 12/14] Potential fix for code scanning alert no. 3: Incomplete string escaping or encoding Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- src/common/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/utils.js b/src/common/utils.js index c71407b..50d77d5 100644 --- a/src/common/utils.js +++ b/src/common/utils.js @@ -405,8 +405,8 @@ async function generateTestSummary(baseDir) { // Format time to 3 decimal places const timeFormatted = test.time.toFixed(3); // Escape pipe characters in names to prevent breaking markdown table - const suiteEscaped = test.suite.replace(/\|/g, '\\|'); - const caseEscaped = test.case.replace(/\|/g, '\\|'); + const suiteEscaped = test.suite.replace(/([\\|])/g, '\\$1'); + const caseEscaped = test.case.replace(/([\\|])/g, '\\$1'); summaryMarkdown += `| ${index + 1} | ${timeFormatted} | ${suiteEscaped} | ${caseEscaped} |\n`; }); summaryMarkdown += '\n'; From 9a3f038b844910cf55fcbaf34f6e5af0af0441c7 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 23 Apr 2025 11:59:55 -0700 Subject: [PATCH 13/14] update --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b5d7eb..0b1452d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,7 +78,7 @@ jobs: # Add any other essential files here echo "Copying built artifacts..." - ACTIONS=("build-and-prep-ort-files" "build-docker-image" "build-minimal-ort-and-run-tests" "run-build-script-in-docker" "setup-build-tools") + ACTIONS=("build-and-prep-ort-files" "build-docker-image" "build-minimal-ort-and-run-tests" "format-lint-check" "run-build-script-in-docker" "setup-build-tools") BUILD_DIR_RELATIVE="build" # Relative to MAIN_REPO_PATH for action_name in "${ACTIONS[@]}"; do From 1045dee759f3e06a9a2255c4c729eac508acb8ad Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 23 Apr 2025 12:04:48 -0700 Subject: [PATCH 14/14] update --- actions/format-lint-check/README.md | 3 +++ src/format-lint-check/index.js | 21 +-------------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/actions/format-lint-check/README.md b/actions/format-lint-check/README.md index aa6ea5f..4fae93b 100644 --- a/actions/format-lint-check/README.md +++ b/actions/format-lint-check/README.md @@ -8,5 +8,8 @@ This GitHub Action runs on Linux runners and performs the following tasks: 1. **Installs LLVM/Clang:** Downloads a specific version of the official LLVM release binaries (`LLVM--Linux-X64.tar.xz`) from GitHub Releases, verifies its SHA256 hash, extracts it, caches it using `@actions/tool-cache`, and adds the `bin` directory to the `PATH`. 2. **Checks C/C++ Formatting:** Uses the installed `clang-format` to verify if C/C++ source files (`.h`, `.cc`, `.cpp`) within the specified directories adhere to the formatting rules defined by `.clang-format` files (searched upwards from file locations). It fails the action if any file needs formatting. + + +TODO in the future it will also: 3. **Checks Shell Script Permissions:** Verifies that all found shell scripts (`.sh`) have at least one execute bit set. 4. **Checks Source/Text File Permissions:** Verifies that common source and text files (`.h`, `.cc`, `.cpp`, `.md`, `.txt`) do **not** have any execute bits set. diff --git a/src/format-lint-check/index.js b/src/format-lint-check/index.js index 45532cc..ea738d6 100644 --- a/src/format-lint-check/index.js +++ b/src/format-lint-check/index.js @@ -34,23 +34,6 @@ async function verifySHA256(filePath, expectedHash) { }); } -// --- File Permission Check Helpers --- -/** - * Checks if a file has execute permissions set for user, group, or others. - * @param {string} filePath - The path to the file. - * @returns {Promise} - True if any execute bit is set, false otherwise. - */ -async function hasExecutePermission(filePath) { - try { - const stats = await fs.stat(filePath); - // Check if any of the execute bits (owner, group, others) are set - // Mode is a number, use bitwise AND with 0o111 (binary 001 001 001) - return (stats.mode & 0o111) !== 0; - } catch (error) { - core.warning(`Could not stat file ${filePath} for permissions check: ${error.message}`); - return false; // Treat inaccessible files as not having execute permission for safety - } -} /** * Finds C/C++ header and source files, filtering out paths matching ignore patterns. @@ -110,8 +93,6 @@ async function run() { let llvmToolPath = ''; let clangFormatPath = ''; // Will hold the absolute path or just 'clang-format' if in PATH - const filesNeedingFormatting = []; - const filesWithIncorrectPermissions = []; try { const llvmVersion = core.getInput('llvm-version', { required: true }); @@ -189,7 +170,7 @@ async function run() { // 1. Create a temporary file path const tempDir = process.env.RUNNER_TEMP || os.tmpdir(); // Prefer RUNNER_TEMP const tempFileName = `clang-format-files-${Date.now()}-${crypto.randomBytes(4).toString('hex')}.txt`; - tempFilePath = path.join(tempDir, tempFileName); + const tempFilePath = path.join(tempDir, tempFileName); core.debug(`Creating temporary file list at: ${tempFilePath}`); // 2. Write the absolute file paths to the temporary file (one per line)