Skip to content

Publish wheels from a separate workflow in target repo context #381

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aristarkhovNV wants to merge 1 commit into
base: main
Choose a base branch
from
+313 −301
Open

Publish wheels from a separate workflow in target repo context #381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
301 changes: 0 additions & 301 deletions .github/workflows/build-ubuntu.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,304 +369,3 @@ jobs:
exit 1
fi
done

publish-wheel: # to internal Artifactory
runs-on: [self-hosted, linux]
needs:
- build-ubuntu
- test-cloudxr
- test-teleop-ros2
outputs:
wheel_paths: ${{ steps.upload-artifactory.outputs.wheel_paths }}
environment: dev

# Publish only for PRs wihtin the canonical repository after build and CloudXR tests succeed
if: >-
${{
github.repository == 'NVIDIA/IsaacTeleop'
&& needs.build-ubuntu.result == 'success'
&& needs.test-cloudxr.result == 'success'
&& needs.test-teleop-ros2.result == 'success'
}}

steps:
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.11'

- name: Ensure clean wheels directory
run: |
rm -rf wheels

- name: Download wheel artifacts
uses: actions/download-artifact@v7
with:
pattern: isaacteleop-wheels-*
merge-multiple: true
path: wheels

- name: Upload wheel(s) to Artifactory
id: upload-artifactory
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_REPO: ${{ secrets.ARTIFACTORY_REPO }}
ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }}
run: |
set -euo pipefail

if [[ -z "${ARTIFACTORY_URL}" || -z "${ARTIFACTORY_REPO}" || -z "${ARTIFACTORY_USERNAME}" || -z "${ARTIFACTORY_API_KEY}" ]]; then
echo "Missing one or more required secrets: ARTIFACTORY_URL, ARTIFACTORY_REPO, ARTIFACTORY_USERNAME, ARTIFACTORY_API_KEY"
exit 1
fi

if [[ "${ARTIFACTORY_URL}" != https://* ]]; then
echo "ARTIFACTORY_URL must use https://"
exit 1
fi

shopt -s nullglob
wheels=(wheels/*.whl)
if (( ${#wheels[@]} == 0 )); then
echo "No wheels found under wheels/*.whl"
ls -la wheels || true
exit 1
fi

python -m pip install --upgrade pip twine

# Artifactory PyPI repositories use the PyPI API endpoint.
repository_url="${ARTIFACTORY_URL%/}/api/pypi/${ARTIFACTORY_REPO}"
echo "Publishing ${#wheels[@]} wheel(s)"

python -m twine upload \
--non-interactive \
--repository-url "${repository_url}" \
-u "${ARTIFACTORY_USERNAME}" \
-p "${ARTIFACTORY_API_KEY}" \
"${wheels[@]}"

wheel_base="${ARTIFACTORY_URL%/}/${ARTIFACTORY_REPO}"
wheel_prefix="${wheel_base}/"

wheel_paths=()
for wheel in "${wheels[@]}"; do
wheel_name="$(basename "${wheel}")"
echo "Resolving Artifactory URL for ${wheel_name}"

search_json="$(curl --fail-with-body --show-error --silent --location --get --connect-timeout 10 --max-time 60 \
-u "${ARTIFACTORY_USERNAME}:${ARTIFACTORY_API_KEY}" \
--data-urlencode "name=${wheel_name}" \
--data-urlencode "repos=${ARTIFACTORY_REPO}" \
"${ARTIFACTORY_URL%/}/api/search/artifact")"

result_count="$(jq '.results | length' <<< "${search_json}")"
if [[ "${result_count}" -ne 1 ]]; then
echo "Expected exactly 1 Artifactory search result for ${wheel_name}, but got ${result_count}"
echo "${search_json}"
exit 1
fi

storage_uri="$(jq -r '.results[0].uri // empty' <<< "${search_json}")"
if [[ -z "${storage_uri}" ]]; then
echo "Unable to resolve storage URI for ${wheel_name}"
echo "${search_json}"
exit 1
fi

metadata_json="$(curl --fail-with-body --show-error --silent --location --connect-timeout 10 --max-time 60 \
-u "${ARTIFACTORY_USERNAME}:${ARTIFACTORY_API_KEY}" \
"${storage_uri}")"

download_uri="$(jq -r '.downloadUri // empty' <<< "${metadata_json}")"
if [[ -z "${download_uri}" ]]; then
echo "Unable to resolve downloadUri for ${wheel_name}"
echo "${metadata_json}"
exit 1
fi

wheel_path="${download_uri#${wheel_prefix}}"
if [[ "${wheel_path}" == "${download_uri}" || -z "${wheel_path}" ]]; then
echo "Unable to clip Artifactory prefix from downloadUri for ${wheel_name}"
echo "Expected prefix: ${wheel_prefix}"
echo "${metadata_json}"
exit 1
fi

wheel_paths+=("${wheel_path}")
done

{
echo "wheel_paths<<EOF"
printf '%s\n' "${wheel_paths[@]}"
echo "EOF"
} >> "$GITHUB_OUTPUT"

kitmaker:
runs-on: [self-hosted, linux]
needs:
- publish-wheel
outputs:
release_uuid: ${{ steps.submit-kitmaker.outputs.release_uuid }}
environment: release
permissions:
contents: read
if: ${{ github.event_name != 'pull_request' && needs.publish-wheel.result == 'success' }}

steps:
- name: Submit wheel release to Kitmaker
id: submit-kitmaker
env:
KITMAKER_API_ENDPOINT: ${{ secrets.KITMAKER_API_ENDPOINT }}
KITMAKER_PROJECT_ID: ${{ secrets.KITMAKER_PROJECT_ID }}
KITMAKER_TOKEN: ${{ secrets.KITMAKER_TOKEN }}
KITMAKER_PIC_EMAIL: ${{ secrets.KITMAKER_PIC_EMAIL }}
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_REPO: ${{ secrets.ARTIFACTORY_REPO }}
WHEEL_PATHS: ${{ needs.publish-wheel.outputs.wheel_paths }}
KITMAKER_UPLOAD: ${{ startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/tags/v') }}
run: |
set -euo pipefail

if [[ -z "${KITMAKER_API_ENDPOINT}" || -z "${KITMAKER_PROJECT_ID}" || -z "${KITMAKER_TOKEN}" || -z "${KITMAKER_PIC_EMAIL}" || -z "${ARTIFACTORY_URL}" || -z "${ARTIFACTORY_REPO}" ]]; then
echo "Missing one or more required secrets: KITMAKER_API_ENDPOINT, KITMAKER_PROJECT_ID, KITMAKER_TOKEN, KITMAKER_PIC_EMAIL, ARTIFACTORY_URL, ARTIFACTORY_REPO"
exit 1
fi

if [[ "${KITMAKER_API_ENDPOINT}" != https://* ]]; then
echo "KITMAKER_API_ENDPOINT must use https://"
exit 1
fi

if [[ "${ARTIFACTORY_URL}" != https://* ]]; then
echo "ARTIFACTORY_URL must use https://"
exit 1
fi

if [[ -z "${WHEEL_PATHS}" ]]; then
echo "No wheel paths were produced by publish-wheel job"
exit 1
fi

wheel_base="${ARTIFACTORY_URL%/}/${ARTIFACTORY_REPO}"

api_url="${KITMAKER_API_ENDPOINT%/}/v0/projects/${KITMAKER_PROJECT_ID}/releases"
payload_items=()
while IFS= read -r wheel_path; do
[[ -z "${wheel_path}" ]] && continue
wheel_url="${wheel_base}/${wheel_path}"
wheel_name="$(basename "${wheel_path}")"
echo "Queueing wheel for Kitmaker payload: ${wheel_name}"

payload_items+=("$(jq -cn \
--arg pic "${KITMAKER_PIC_EMAIL}" \
--arg url "${wheel_url}" \
--argjson upload "${KITMAKER_UPLOAD}" \
'{pic: $pic, job_type: "wheel-release-job", publish_to: "both_devzone_pypi", url: $url, size: "small", upload: $upload}')")
done <<< "${WHEEL_PATHS}"

if (( ${#payload_items[@]} == 0 )); then
echo "No valid wheel URLs found to submit to Kitmaker"
exit 1
fi

payload_array="$(printf '%s\n' "${payload_items[@]}" | jq -s '.')"
payload="$(jq -cn --arg project_name "isaacteleop" --argjson payload "${payload_array}" '{project_name: $project_name, payload: $payload}')"

echo "Posting ${#payload_items[@]} wheel(s) to Kitmaker in a single request"
response_json="$(curl --fail-with-body --show-error --silent --location --connect-timeout 10 --max-time 120 \
-X POST "${api_url}" \
-H "Authorization: Bearer ${KITMAKER_TOKEN}" \
-H "Content-Type: application/json" \
-d "${payload}")"
echo "${response_json}"

release_uuid="$(jq -r '.release_uuid // empty' <<< "${response_json}")"
if [[ -z "${release_uuid}" ]]; then
echo "Kitmaker response missing release_uuid"
exit 1
fi

echo "release_uuid=${release_uuid}" >> "$GITHUB_OUTPUT"

kitmaker-status:
runs-on: [self-hosted, linux]
needs:
- kitmaker
environment: release
permissions:
contents: read
if: ${{ github.event_name != 'pull_request' && needs.kitmaker.result == 'success' }}

steps:
- name: Monitor Kitmaker release status
env:
KITMAKER_API_ENDPOINT: ${{ secrets.KITMAKER_API_ENDPOINT }}
KITMAKER_PROJECT_ID: ${{ secrets.KITMAKER_PROJECT_ID }}
KITMAKER_TOKEN: ${{ secrets.KITMAKER_TOKEN }}
KITMAKER_RELEASE_UUID: ${{ needs.kitmaker.outputs.release_uuid }}
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_REPO: ${{ secrets.ARTIFACTORY_REPO }}
run: |
set -euo pipefail

# Redact sensitive values that may appear in API responses.
echo "::add-mask::${KITMAKER_API_ENDPOINT}"
echo "::add-mask::${KITMAKER_API_ENDPOINT%/}"
echo "::add-mask::${KITMAKER_RELEASE_UUID}"
echo "::add-mask::${KITMAKER_PROJECT_ID}"
echo "::add-mask::${ARTIFACTORY_URL}"
echo "::add-mask::${ARTIFACTORY_URL%/}"
echo "::add-mask::${ARTIFACTORY_REPO}"

if [[ -z "${KITMAKER_API_ENDPOINT}" || -z "${KITMAKER_TOKEN}" ]]; then
echo "Missing required secrets: KITMAKER_API_ENDPOINT, KITMAKER_TOKEN"
exit 1
fi

if [[ "${KITMAKER_API_ENDPOINT}" != https://* ]]; then
echo "KITMAKER_API_ENDPOINT must use https://"
exit 1
fi

if [[ -z "${KITMAKER_RELEASE_UUID}" ]]; then
echo "No release_uuid was produced by the kitmaker job"
exit 1
fi

status_url="${KITMAKER_API_ENDPOINT%/}/v0/status/${KITMAKER_RELEASE_UUID}"
# Limit total polling time to under an hour with exponential backoff starting at 30s
max_attempts=15
sleep_seconds=30

for ((attempt=1; attempt<=max_attempts; attempt++)); do
echo "Polling Kitmaker status (attempt ${attempt}/${max_attempts})"

response_json="$(curl --fail-with-body --show-error --silent --location --connect-timeout 10 --max-time 60 \
-H "Authorization: Bearer ${KITMAKER_TOKEN}" \
"${status_url}")"

echo "${response_json}"
status="$(jq -r '.status // empty' <<< "${response_json}")"

if [[ "${status}" == "completed" ]]; then
echo "Kitmaker release ${KITMAKER_RELEASE_UUID} completed"
exit 0
fi

if [[ "${status}" == "failed" ]]; then
echo "Kitmaker release ${KITMAKER_RELEASE_UUID} failed"
exit 1
fi

if (( attempt == max_attempts )); then
break
fi

sleep "${sleep_seconds}"
sleep_seconds=$(( sleep_seconds * 125 / 100 ))
done

echo "Timed out waiting for Kitmaker release ${KITMAKER_RELEASE_UUID} to complete"
exit 1
Loading
Loading