diff --git a/tools/devops/automation/scripts/.gitignore b/tools/devops/automation/scripts/.gitignore new file mode 100644 index 000000000000..89923e7760bc --- /dev/null +++ b/tools/devops/automation/scripts/.gitignore @@ -0,0 +1,2 @@ +_provisionator + diff --git a/tools/devops/automation/scripts/provisionator-bootstrap.sh b/tools/devops/automation/scripts/provisionator-bootstrap.sh new file mode 100755 index 000000000000..143cfa4bf75e --- /dev/null +++ b/tools/devops/automation/scripts/provisionator-bootstrap.sh @@ -0,0 +1,131 @@ +#!/bin/bash -e +# shellcheck disable=SC2220,SC2181 +# +# Remove the following GUID if you do not wish for this script self-update: +# D6F71FB5-F2A7-4A62-86D3-10DFE08301CC +# https://github.com/xamarin/provisionator + +while getopts ":t:" opt; do + case "${opt}" in + t) + TOKEN=${OPTARG} + shift 2 + ;; + esac +done + +function selfdir { (cd "$(dirname "$1")"; echo "$PWD"; ) } + +selfdir=$(selfdir "$0") + +channel="${PROVISIONATOR_CHANNEL:-latest}" + +provisionator_sas_token=$AUTH_TOKEN_PROVISIONATOR +if [[ -n $provisionator_sas_token ]]; then + echo "Downloading Provisionator directly from BosStorageMirror Azure blob storage" + base_url="https://bosstoragemirror.blob.core.windows.net/provisionator/664bd334021e3102cdef1af66c4fc9f1b2ecd2a21b47419e80d08da1f6c61c2a/${channel}" +else + # Grab auth token from Environment first + # Otherwise set to provided option, if available + base_url="https://dl.internalx.com/provisionator/664bd334021e3102cdef1af66c4fc9f1b2ecd2a21b47419e80d08da1f6c61c2a/${channel}" + auth_token=$AUTH_TOKEN_GITHUB_COM + if [[ -z $auth_token ]]; then + echo "Setting auth_token to input token and not AUTH_TOKEN_GITHUB_COM" + auth_token=$TOKEN + fi + if [[ -z $auth_token ]]; then + echo "WARNING: Please set AUTH_TOKEN_GITHUB_COM to a GitHub PAT before running provisionator or run provisionator.sh -t ..." + echo "At a future point, Provisionator will not be able to bootstrap without the PAT set properly" + base_url="https://bosstoragemirror.blob.core.windows.net/provisionator/664bd334021e3102cdef1af66c4fc9f1b2ecd2a21b47419e80d08da1f6c61c2a/${channel}" + fi +fi + +latest_version_url="${base_url}/version" + +if [[ -n $provisionator_sas_token ]]; then + echo "latest_version_url: Appending SAS token" + latest_version_url="${latest_version_url}?${provisionator_sas_token}" +fi + +archive_name="provisionator.osx.10.11-x64.zip" +archive_path="${selfdir}/${archive_name}" +archive_extract_path="${selfdir}/_provisionator" +archive_url="${base_url}/${archive_name}" +if [[ -n $provisionator_sas_token ]]; then + echo "archive_url: Appending SAS token" + archive_url="${archive_url}?${provisionator_sas_token}" +fi + +binary_path="${archive_extract_path}/provisionator" +pk_path="${archive_extract_path}/pk" + +set +e + +# Double brackets are required for the following condition to evaulate as expected +if [[ -z $auth_token ]] || [[ -n $provisionator_sas_token ]]; then + if [[ -n $provisionator_sas_token ]]; then + echo "Fetching latest version with SAS token" + else + echo "Fetching latest version without auth token" + fi + latest_version="$(curl -fsL "${latest_version_url}")" +else + echo "Fetching latest version with auth token" + latest_version="$(curl -fsL -H "Authorization: token ${auth_token}" "${latest_version_url}")" +fi +if [[ $? != 0 ]]; then + echo "Unable to determine latest version from ${latest_version_url}" + echo "Please check to make sure a valid GitHub PAT was provided" + exit 1 +fi +set -e + +function update_in_place { + echo "Downloading Provisionator $latest_version..." + local progress_type="-s" + tty -s && progress_type="-#" + if [[ -z $auth_token ]] || [[ -n $provisionator_sas_token ]]; then + curl -fL $progress_type -o "$archive_path" "$archive_url" + else + curl -fLH "Authorization: token ${auth_token}" $progress_type -o "$archive_path" "$archive_url" + fi + rm -rf "$archive_extract_path" + unzip -q -o -d "$archive_extract_path" "$archive_path" + rm -f "$archive_path" +} + +if [[ -f "$binary_path" ]]; then + chmod +x "$binary_path" + current_version="$("$binary_path" -version 2>&1 || true)" + if [[ "$latest_version" != "$current_version" ]]; then + update_in_place + fi +else + update_in_place +fi + +# Check for the pk (PackageKitTool) binary, which should be included in the provisionator macOS ZIP package +if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Checking for PackageKitTool (pk) binary..." + if [[ -f "$pk_path" ]]; then + echo "PackageKitTool (pk) binary found: ${pk_path}" + echo "chmod +x ${pk_path}" + chmod +x "$pk_path" + else + echo "ERROR: pk file not found: ${pk_path}" + exit 5 + fi +fi + +# Opt out of macOS backward compat so correct Big Sur OS versions can be obtained. +# More info https://github.com/ziglang/zig/issues/7569 +export SYSTEM_VERSION_COMPAT=0 + +# TODO: The executable permission should be applied here. However, since this script is reflective of production scripts used by other clients/teams avoid modifying the executable permission here. +# A reason to avoid this is that the executable permission is expected to be preserved in the ZIP file produced by the ClientTools.Provisionator build pipeline +# CONSIDER: A possible follow up item would be to go update all the production scripts to apply the executable permission here to ensure the provisionator binary has the executable permission set +# This may also involve updating the Provisionator Azure DevOps provisionator task to ensure it applies the executable attribute after ZIP extraction +# https://devdiv.visualstudio.com/Engineering/_git/xamarin-azdev-extension?path=/provisionator-task +# echo "Applying executable attribute: ${binary_path}" +# chmod +x "$binary_path" +exec caffeinate "$binary_path" "$@" diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index d3d9951de246..ae746a2d4776 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -85,14 +85,12 @@ steps: make -C $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops provisioning displayName: 'Generate provisionator files.' - - task: xamops.azdevex.provisionator-task.provisionator@2 - displayName: 'Provision Products & Frameworks' - inputs: + - template: ../common/provision.yml + parameters: provisioning_script: $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/build-provisioning.csx - provisioning_extra_args: '-vvvv' - github_token: ${{ parameters.gitHubToken }} - timeoutInMinutes: 250 - retryCountOnTaskFailure: ${{ parameters.retryCount }} # mono does give issues sometimes to download, we will retry + displayName: 'Provision Products & Frameworks' + provisionatorChannel: $(PROVISIONATOR_CHANNEL) + retryCount: ${{ parameters.retryCount }} # mono does give issues sometimes to download, we will retry - bash: | set -x diff --git a/tools/devops/automation/templates/common/provision.yml b/tools/devops/automation/templates/common/provision.yml new file mode 100644 index 000000000000..6424ffd2103d --- /dev/null +++ b/tools/devops/automation/templates/common/provision.yml @@ -0,0 +1,116 @@ +# yamllint disable rule:line-length +# Template to run provisionator via the bootstrap script with BosStorageMirror SAS token auth. +# This replaces the xamops.azdevex.provisionator-task.provisionator@3 task to avoid +# dl.internalx.com timeout issues under concurrent load. + +parameters: + - name: provisioning_script + type: string + + - name: displayName + type: string + default: 'Provisionator' + + - name: provisionatorChannel + type: string + default: 'latest' + + - name: provisioning_extra_args + type: string + default: '-vvvv' + + - name: timeoutInMinutes + type: number + default: 250 + + - name: retryCount + type: number + default: 0 + + - name: enabled + type: boolean + default: true + +steps: + - task: AzureCLI@2 + displayName: 'Generate BosStorageMirror SAS tokens' + condition: and(succeeded(), ${{ parameters.enabled }}) + inputs: + azureSubscription: 'Xamarin - RelEng (BosStorageMirror-Contributor-MI)' + scriptType: 'bash' + scriptLocation: 'inlineScript' + failOnStandardError: true + inlineScript: | + set -euo pipefail + + storage_account="bosstoragemirror" + + # macOS/BSD date syntax for expiry calculation + start_time="$(date -u '+%Y-%m-%dT%H:%MZ')" + prov_expiry="$(date -u -v+20M '+%Y-%m-%dT%H:%MZ')" + files_expiry="$(date -u -v+3H '+%Y-%m-%dT%H:%MZ')" + + echo "Generating provisionator SAS token (expires $prov_expiry)..." + sas_provisionator="$(az storage container generate-sas \ + --account-name "$storage_account" \ + --name provisionator \ + --permissions r \ + --start "$start_time" \ + --expiry "$prov_expiry" \ + --https-only \ + --as-user \ + --auth-mode login \ + -o tsv)" + + if [[ -z "$sas_provisionator" || ${#sas_provisionator} -le 16 ]]; then + echo "ERROR: Failed to generate SAS token for BosStorageMirror provisionator container" + exit 1 + fi + + echo "Generating internal-files SAS token (expires $files_expiry)..." + sas_internal_files="$(az storage container generate-sas \ + --account-name "$storage_account" \ + --name internal-files \ + --permissions r \ + --start "$start_time" \ + --expiry "$files_expiry" \ + --https-only \ + --as-user \ + --auth-mode login \ + -o tsv)" + + if [[ -z "$sas_internal_files" || ${#sas_internal_files} -le 16 ]]; then + echo "ERROR: Failed to generate SAS token for BosStorageMirror internal-files container" + exit 1 + fi + + prov_hint="${sas_provisionator:0:8}...${sas_provisionator: -8}" + files_hint="${sas_internal_files:0:8}...${sas_internal_files: -8}" + + echo "sasTokenProvisionator [hint]: $prov_hint" + echo "sasToken [hint]: $files_hint" + + echo "##vso[task.setvariable variable=Provisionator.SASToken;issecret=true]$sas_provisionator" + echo "##vso[task.setvariable variable=Provisionator.SASTokenHint]$prov_hint" + echo "##vso[task.setvariable variable=BosStorageMirror.SASToken;issecret=true]$sas_internal_files" + echo "##vso[task.setvariable variable=BosStorageMirror.SASTokenHint]$files_hint" + + - bash: | + set -x + set -e + + BOOTSTRAP_SCRIPT="$(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/automation/scripts/provisionator-bootstrap.sh" + chmod +x "$BOOTSTRAP_SCRIPT" + + echo "Provisionator channel: ${PROVISIONATOR_CHANNEL}" + echo "Provisioning script: ${{ parameters.provisioning_script }}" + + "$BOOTSTRAP_SCRIPT" ${{ parameters.provisioning_script }} ${{ parameters.provisioning_extra_args }} || exit 2 + displayName: '${{ parameters.displayName }}' + condition: and(succeeded(), ${{ parameters.enabled }}) + env: + PROVISIONATOR_CHANNEL: ${{ parameters.provisionatorChannel }} + AUTH_TOKEN_PROVISIONATOR: $(Provisionator.SASToken) + AUTH_TOKEN_BOSSTORAGEMIRROR: $(BosStorageMirror.SASToken) + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + retryCountOnTaskFailure: ${{ parameters.retryCount }} diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index 12cd12a3d0b4..fd4bcfacc0c4 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -104,14 +104,13 @@ steps: displayName: 'Set VM Vendor' -- task: xamops.azdevex.provisionator-task.provisionator@2 - displayName: 'Provision Brew components' - inputs: +- template: ../common/provision.yml + parameters: provisioning_script: $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/provision-brew-packages.csx - provisioning_extra_args: '-vvvv' - github_token: $(Github.Token) - timeoutInMinutes: 30 - enabled: false + displayName: 'Provision Brew components' + provisionatorChannel: $(PROVISIONATOR_CHANNEL) + timeoutInMinutes: 30 + enabled: false - bash: | sudo rm -Rf $(Build.SourcesDirectory)/package diff --git a/tools/devops/automation/templates/tests/build.yml b/tools/devops/automation/templates/tests/build.yml index 9d94572b96d0..1eacbac94c70 100644 --- a/tools/devops/automation/templates/tests/build.yml +++ b/tools/devops/automation/templates/tests/build.yml @@ -203,13 +203,11 @@ steps: condition: ${{ parameters.clearProvisionatorCache }} # Use the provisionator to install the test dependencies. Those have been generated in the 'Generate Provisionator csx file' step. -- task: xamops.azdevex.provisionator-task.provisionator@2 - displayName: 'Provisionator dependencies' - inputs: +- template: ../common/provision.yml + parameters: provisioning_script: $(System.DefaultWorkingDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/build-provisioning.csx - provisioning_extra_args: '-vvvv' - github_token: ${{ parameters.gitHubToken }} - timeoutInMinutes: 250 + displayName: 'Provisionator dependencies' + provisionatorChannel: $(PROVISIONATOR_CHANNEL) - bash: | set -x diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index c0e2c9ae0ae8..9e24069e0af6 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -99,13 +99,11 @@ steps: rm -f ~/Library/Preferences/Xamarin/Settings.plist displayName: 'Clear old Xcode settings' -- task: xamops.azdevex.provisionator-task.provisionator@2 - displayName: 'Provision Xcode' - inputs: +- template: ../common/provision.yml + parameters: provisioning_script: $(System.DefaultWorkingDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/provision-xcode.csx - provisioning_extra_args: '-vvvv' - github_token: ${{ parameters.gitHubToken }} - timeoutInMinutes: 250 + displayName: 'Provision Xcode' + provisionatorChannel: $(PROVISIONATOR_CHANNEL) - bash: $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/system-dependencies.sh --ignore-visual-studio --ignore-shellcheck --ignore-yamllint --provision-simulators --ignore-xcode-components displayName: 'Provision simulators'