From 5fd6030686a61caa284be835fe9e94abdcf4a0cd Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 15:40:29 +0200 Subject: [PATCH 01/13] Latest draft --- .../templates/getWorkflowInput/action.yml | 4 +- .../Get-GitHubWorkflowDefaultInput.ps1 | 4 +- utilities/tools/Invoke-PipelinesForBranch.ps1 | 258 ++++++++++++++++++ 3 files changed, 262 insertions(+), 4 deletions(-) rename .github/actions/templates/getWorkflowInput/scripts/Get-WorkflowDefaultInput.ps1 => utilities/pipelines/sharedScripts/Get-GitHubWorkflowDefaultInput.ps1 (97%) create mode 100644 utilities/tools/Invoke-PipelinesForBranch.ps1 diff --git a/.github/actions/templates/getWorkflowInput/action.yml b/.github/actions/templates/getWorkflowInput/action.yml index ba46f0656b..f3951974b1 100644 --- a/.github/actions/templates/getWorkflowInput/action.yml +++ b/.github/actions/templates/getWorkflowInput/action.yml @@ -66,7 +66,7 @@ runs: # Otherwise retrieve default values else { # Load used functions - . "$env:GITHUB_ACTION_PATH/scripts/Get-WorkflowDefaultInput.ps1" + . "$env:GITHUB_ACTION_PATH/scripts/Get-GitHubWorkflowDefaultInput.ps1" $functionInput = @{ workflowPath = '${{ inputs.workflowPath }}' @@ -75,7 +75,7 @@ runs: Write-Verbose "Invoke task with" -Verbose Write-Verbose ($functionInput | ConvertTo-Json | Out-String) -Verbose - $workflowParameters = Get-WorkflowDefaultInput @functionInput -Verbose + $workflowParameters = Get-GitHubWorkflowDefaultInput @functionInput -Verbose $removeDeployment = $workflowParameters.removeDeployment } diff --git a/.github/actions/templates/getWorkflowInput/scripts/Get-WorkflowDefaultInput.ps1 b/utilities/pipelines/sharedScripts/Get-GitHubWorkflowDefaultInput.ps1 similarity index 97% rename from .github/actions/templates/getWorkflowInput/scripts/Get-WorkflowDefaultInput.ps1 rename to utilities/pipelines/sharedScripts/Get-GitHubWorkflowDefaultInput.ps1 index 6db706dcd3..5449674308 100644 --- a/.github/actions/templates/getWorkflowInput/scripts/Get-WorkflowDefaultInput.ps1 +++ b/utilities/pipelines/sharedScripts/Get-GitHubWorkflowDefaultInput.ps1 @@ -9,11 +9,11 @@ Retrieve input parameter default values for a specified workflow. Return hashtab Mandatory. The path to the github workflow file. .EXAMPLE -Get-WorkflowDefaultInput -workflowPath 'path/to/workflow' -verbose +Get-GitHubWorkflowDefaultInput -workflowPath 'path/to/workflow' -verbose Retrieve input parameter default values for the 'path/to/workflow' workflow. #> -function Get-WorkflowDefaultInput { +function Get-GitHubWorkflowDefaultInput { [CmdletBinding()] param ( [Parameter(Mandatory)] diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 new file mode 100644 index 0000000000..c74d427469 --- /dev/null +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -0,0 +1,258 @@ +#region helper functions + +<# +.SYNOPSIS +Invoke a given GitHub workflow + +.DESCRIPTION +Long description + +.PARAMETER PersonalAccessToken +Mandatory. The GitHub PAT to leverage when interacting with the GitHub API. + +.PARAMETER GitHubRepositoryOwner +Mandatory. The repository's organization. + +.PARAMETER GitHubRepositoryName +Mandatory. The name of the repository to trigger the workflows in. + +.PARAMETER WorkflowFileName +Mandatory. The name of the workflow to trigger + +.PARAMETER TargetBranch +Optional. The branch to trigger the pipeline for. + +.EXAMPLE +Invoke-GitHubWorkflow -PersonalAccessToken '' -GitHubRepositoryOwner 'Azure' -GitHubRepositoryName 'ResourceModules' -WorkflowFileName 'ms.analysisservices.servers.yml' -TargetBranch 'main' + +Trigger the workflow 'ms.analysisservices.servers.yml' with branch 'main' in repository 'Azure/ResourceModules'. +#> +function Invoke-GitHubWorkflow { + + [CmdletBinding(SupportsShouldProcess)] + param ( + [Parameter(Mandatory = $true)] + [string] $PersonalAccessToken, + + [Parameter(Mandatory = $true)] + [string] $GitHubRepositoryOwner, + + [Parameter(Mandatory = $true)] + [string] $GitHubRepositoryName, + + [Parameter(Mandatory = $true)] + [string] $WorkflowFilePath, + + [Parameter(Mandatory = $false)] + [string] $TargetBranch = 'main' + ) + + # Load used function + . (Join-Path (Split-Path $PSScriptRoot -Parent) 'pipelines' 'sharedScripts' 'Get-GitHubWorkflowDefaultInput.ps1') + + $workflowFileName = Split-Path $WorkflowFilePath -Leaf + $workflowParameters = Get-GitHubWorkflowDefaultInput -workflowPath $WorkflowFilePath -Verbose + $removeDeploymentFlag = $workflowParameters.removeDeployment + + $requestInputObject = @{ + Method = 'POST' + Uri = "https://api.github.com/repos/$GitHubRepositoryOwner/$GitHubRepositoryName/actions/workflows/$workflowFileName/dispatches" + Headers = @{ + Authorization = "Bearer $PersonalAccessToken" + } + Body = @{ + ref = $TargetBranch + inputs = @{ + prerelease = 'false' + removeDeployment = $removeDeploymentFlag + } + } | ConvertTo-Json + } + if ($PSCmdlet.ShouldProcess("GitHub workflow [$workflowFileName] for branch [$TargetBranch]", 'Invoke')) { + $response = Invoke-RestMethod @requestInputObject + + if ($response) { + Write-Error "Request failed. Reponse: [$response]" + return $false + } + } + + return $true +} + +<# +.SYNOPSIS +Get a list of all GitHub module workflows + +.DESCRIPTION +Get a list of all GitHub module workflows. Does not return all properties but only the relevant ones. + +.PARAMETER PersonalAccessToken +Mandatory. The GitHub PAT to leverage when interacting with the GitHub API. + +.PARAMETER GitHubRepositoryOwner +Mandatory. The repository's organization. + +.PARAMETER GitHubRepositoryName +Mandatory. The name of the repository to fetch the workflows from. + +.PARAMETER Filter +Optional. A filter to apply when fetching the workflows. By default we fetch all module workflows (ms.*). + +.EXAMPLE +Get-GitHubModuleWorkflowList -PersonalAccessToken '' -GitHubRepositoryOwner 'Azure' -GitHubRepositoryName 'ResourceModules' + +Get all module workflows from repository 'Azure/ResourceModules' +#> +function Get-GitHubModuleWorkflowList { + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string] $PersonalAccessToken, + + [Parameter(Mandatory = $true)] + [string] $GitHubRepositoryOwner, + + [Parameter(Mandatory = $true)] + [string] $GitHubRepositoryName, + + [Parameter(Mandatory = $false)] + [string] $Filter = 'ms.*' + ) + + $allWorkflows = @() + $page = 1 + do { + $requestInputObject = @{ + Method = 'GET' + Uri = "https://api.github.com/repos/$GitHubRepositoryOwner/$GitHubRepositoryName/actions/workflows?per_page=100&page=$page" + Headers = @{ + Authorization = "Bearer $PersonalAccessToken" + } + } + $response = Invoke-RestMethod @requestInputObject + + if (-not $response.workflows) { + Write-Error "Request failed. Reponse: [$response]" + } + + $allWorkflows += $response.workflows | Select-Object -Property @('id', 'name', 'path', 'badge_url') | Where-Object { (Split-Path $_.path -Leaf) -like $Filter } + + $expectedPages = [math]::ceiling($response.total_count / 100) + $page++ + } while ($page -le $expectedPages) + + return $allWorkflows +} +#endregion + +function Invoke-PipelinesForBranch { + + [CmdletBinding(SupportsShouldProcess)] + param ( + [Parameter(Mandatory = $true)] + [string] $PersonalAccessToken, + + [Parameter(Mandatory = $true)] + [string] $TargetBranch, + + [Parameter(Mandatory = $false)] + [string] $PipelineFilter = 'ms.*', + + [Parameter(Mandatory = $false)] + [ValidateSet('GitHub', 'AzureDevOps')] + [string] $Environment = 'GitHub', + + [Parameter(Mandatory = $false)] + [bool] $GeneratePipelineBadges = $true, + + [Parameter(Mandatory = $false)] + [string] $RepositoryRoot = (Split-Path (Split-Path $PSScriptRoot -Parent)), + + [Parameter(Mandatory = $false, ParameterSetName = 'GitHub')] + [string] $GitHubRepositoryOwner = 'Azure', + + [Parameter(Mandatory = $false, ParameterSetName = 'GitHub')] + [string] $GitHubRepositoryName = 'ResourceModules', + + [Parameter(Mandatory = $false, ParameterSetName = 'AzureDevOps')] + [string] $AzureDevOpsOrganizationName = 'ResourceModules', + + [Parameter(Mandatory = $false, ParameterSetName = 'AzureDevOps')] + [string] $AzureDevOpsProjectName = 'ResourceModules', + + [Parameter(Mandatory = $false, ParameterSetName = 'AzureDevOps')] + [string] $AzureDevOpsPipelineFolderPath = 'CARML-Modules' + ) + + if ($Environment -eq 'GitHub') { + + $baseInputObject = @{ + PersonalAccessToken = $PersonalAccessToken + GitHubRepositoryOwner = $GitHubRepositoryOwner + GitHubRepositoryName = $GitHubRepositoryName + } + + # List all workflows + $workflows = Get-GitHubModuleWorkflowList @baseInputObject -Filter $PipelineFilter + + $gitHubWorkflowBadges = [System.Collections.ArrayList]@() + + # Invoke workflows for target branch + foreach ($workflow in $workflows) { + + $workflowName = $workflow.name + $workflowFilePath = $workflow.path + $WorkflowFileName = Split-Path $Workflow.path -Leaf + + if ($PSCmdlet.ShouldProcess("GitHub workflow [$WorkflowFileName] for branch [$TargetBranch]", 'Invoke')) { + $null = Invoke-GitHubWorkflow @baseInputObject -TargetBranch $TargetBranch -WorkflowFilePath (Join-Path $RepositoryRoot $workflowFilePath) + } + + # Generate pipeline badges + if ($GeneratePipelineBadges) { + $encodedBranch = [System.Web.HttpUtility]::UrlEncode($TargetBranch) + $workflowUrl = "https://github.com/$GitHubRepositoryOwner/$GitHubRepositoryName/actions/workflows/$workflowFileName" + + $gitHubWorkflowBadges += "[![$workflowName]($workflowUrl/badge.svg?branch=$encodedBranch)]($workflowUrl)" + } + + } + + if ($gitHubWorkflowBadges.Count -gt 0) { + Write-Verbose 'GitHub Workflow Badges' -Verbose + Write-Verbose '======================' -Verbose + Write-Verbose ($gitHubWorkflowBadges | Sort-Object | Out-String) -Verbose + } + } + + if ($Environment -eq 'AzureDevOps') { + + Write-Verbose "Log into Azure DevOps project $OrganizationName/$AzureDevOpsProjectName with a PAT" + $azureDevOpsOrgUrl = "https://dev.azure.com/$AzureDevOpsOrganizationName/" + $PersonalAccessToken | az devops login + + Write-Verbose "Set default Azure DevOps configuration to [$AzureDevOpsOrganizationName/$AzureDevOpsProjectName]" + az devops configure --defaults organization=$orgUazureDevOpsOrgUrlrl project=$AzureDevOpsProjectName --use-git-aliases $true + + Write-Verbose "Get and list all Azure Pipelines in $PipelineTargetPath" + $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json | Sort-Object name + + # List all pipelines + # Filter to only module pipelines + # Invoke pipelines for target branch + + # Generate pipeline badges + if ($GeneratePipelineBadges) { + + $encodedBranch = [System.Web.HttpUtility]::UrlEncode($TargetBranch) + $workflowUrl = "https://dev.azure.com/$AzureDevOpsOrganizationName/$AzureDevOpsProjectName/_apis/build/status/$AzureDevOpsPipelineFolderPath/{0}?branchName=$encodedBranch" + + $gitHubWorkflowBadges += "[![$workflowName]($workflowUrl/badge.svg?branch=$encodedBranch)]($workflowUrl)" + + + # [![Build Status](https://dev.azure.com/servicescode/infra-as-code-source/_apis/build/status/CARML-Modules/AnalysisServices%20-%20Servers?branchName=issue%2F1127)](https://dev.azure.com/servicescode/infra-as-code-source/_build/latest?definitionId=2156&branchName=issue%2F1127) + } + } +} From c2d4c89a21b1d0b006ca6ee33e4f40103c52fc03 Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 17:27:41 +0200 Subject: [PATCH 02/13] Added ADO Logic --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 56 ++++++++++++++----- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index c74d427469..77910e4f82 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -212,7 +212,7 @@ function Invoke-PipelinesForBranch { # Generate pipeline badges if ($GeneratePipelineBadges) { - $encodedBranch = [System.Web.HttpUtility]::UrlEncode($TargetBranch) + $encodedBranch = [uri]::EscapeDataString($TargetBranch) $workflowUrl = "https://github.com/$GitHubRepositoryOwner/$GitHubRepositoryName/actions/workflows/$workflowFileName" $gitHubWorkflowBadges += "[![$workflowName]($workflowUrl/badge.svg?branch=$encodedBranch)]($workflowUrl)" @@ -229,30 +229,60 @@ function Invoke-PipelinesForBranch { if ($Environment -eq 'AzureDevOps') { - Write-Verbose "Log into Azure DevOps project $OrganizationName/$AzureDevOpsProjectName with a PAT" $azureDevOpsOrgUrl = "https://dev.azure.com/$AzureDevOpsOrganizationName/" + + # Login into Azure DevOps project with a PAT $PersonalAccessToken | az devops login - Write-Verbose "Set default Azure DevOps configuration to [$AzureDevOpsOrganizationName/$AzureDevOpsProjectName]" + # Set default Azure DevOps configuration (to not continously specify it on every command) az devops configure --defaults organization=$orgUazureDevOpsOrgUrlrl project=$AzureDevOpsProjectName --use-git-aliases $true - Write-Verbose "Get and list all Azure Pipelines in $PipelineTargetPath" - $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json | Sort-Object name + Write-Verbose "Get and list all [$AzureDevOpsOrganizationName/$AzureDevOpsProjectName] Azure DevOps pipelines in folder [$AzureDevOpsPipelineFolderPath]" + $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json | Sort-Object 'name' + + Write-Verbose 'Fetching details' # Required as we need the original file path for filtering (which is only available when fetching the pipeline directly) + + $detailedAzurePipelines = $azurePipelines | ForEach-Object -ThrottleLimit 10 -Parallel { + Write-Verbose ('Fetching detailed information for pipeline [{0}]' -f $PSItem.name) + az pipelines show --organization $USING:azureDevOpsOrgUrl --project $USING:AzureDevOpsProjectName --id $PSItem.id | ConvertFrom-Json + } + + $modulePipelines = $detailedAzurePipelines | Where-Object { (Split-Path $_.process.yamlFileName -Leaf) -like $PipelineFilter } + + $azureDevOpsPipelineBadges = [System.Collections.ArrayList]@() - # List all pipelines - # Filter to only module pipelines # Invoke pipelines for target branch + foreach ($modulePipeline in $modulePipelines) { - # Generate pipeline badges - if ($GeneratePipelineBadges) { - $encodedBranch = [System.Web.HttpUtility]::UrlEncode($TargetBranch) - $workflowUrl = "https://dev.azure.com/$AzureDevOpsOrganizationName/$AzureDevOpsProjectName/_apis/build/status/$AzureDevOpsPipelineFolderPath/{0}?branchName=$encodedBranch" + if ($PSCmdlet.ShouldProcess("GitHub workflow [$WorkflowFileName] for branch [$TargetBranch]", 'Invoke')) { + $null = az pipelines run --branch $TargetBranch --id $modulePipeline.id + } - $gitHubWorkflowBadges += "[![$workflowName]($workflowUrl/badge.svg?branch=$encodedBranch)]($workflowUrl)" + # Generate pipeline badges + if ($GeneratePipelineBadges) { + + $pipelineDefinitionId = $modulePipeline.id + $encodedPipelineName = [uri]::EscapeDataString($modulePipeline.Name) + $encodedBranch = [uri]::EscapeDataString($TargetBranch) + $primaryUrl = 'https://dev.azure.com/{0}/{1}/_apis/build/status/{2}/{3}?branchName={4}' -f $AzureDevOpsOrganizationName, $AzureDevOpsProjectName, $AzureDevOpsPipelineFolderPath, $encodedPipelineName, $encodedBranch + $secondaryUrl = 'https://dev.azure.com/{0}/{1}/_build/latest?definitionId={2}&branchName={3}' -f $AzureDevOpsOrganizationName, $AzureDevOpsProjectName, $pipelineDefinitionId, $encodedBranch + + $azureDevOpsPipelineBadges += "[![Build Status]($primaryUrl)]($secondaryUrl)" + + # [ + # ![Build Status] + # (https://dev.azure.com/servicescode/infra-as-code-source/_apis/build/status/CARML-Modules/AnalysisServices%20-%20Servers?branchName=issue%2F1127) + # ] + # (https://dev.azure.com/servicescode/infra-as-code-source/_build/latest?definitionId=2156&branchName=issue%2F1127) + } + } - # [![Build Status](https://dev.azure.com/servicescode/infra-as-code-source/_apis/build/status/CARML-Modules/AnalysisServices%20-%20Servers?branchName=issue%2F1127)](https://dev.azure.com/servicescode/infra-as-code-source/_build/latest?definitionId=2156&branchName=issue%2F1127) + if ($azureDevOpsPipelineBadges.Count -gt 0) { + Write-Verbose 'Azure DevOps Pipeline Badges' -Verbose + Write-Verbose '============================' -Verbose + Write-Verbose ($azureDevOpsPipelineBadges | Sort-Object | Out-String) -Verbose } } } From 3246748b399691b338ccd6f088b5d21536051dbb Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 17:28:14 +0200 Subject: [PATCH 03/13] Added ADO Logic --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index 77910e4f82..dbbdd1164b 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -238,7 +238,7 @@ function Invoke-PipelinesForBranch { az devops configure --defaults organization=$orgUazureDevOpsOrgUrlrl project=$AzureDevOpsProjectName --use-git-aliases $true Write-Verbose "Get and list all [$AzureDevOpsOrganizationName/$AzureDevOpsProjectName] Azure DevOps pipelines in folder [$AzureDevOpsPipelineFolderPath]" - $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json | Sort-Object 'name' + $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json Write-Verbose 'Fetching details' # Required as we need the original file path for filtering (which is only available when fetching the pipeline directly) @@ -247,7 +247,7 @@ function Invoke-PipelinesForBranch { az pipelines show --organization $USING:azureDevOpsOrgUrl --project $USING:AzureDevOpsProjectName --id $PSItem.id | ConvertFrom-Json } - $modulePipelines = $detailedAzurePipelines | Where-Object { (Split-Path $_.process.yamlFileName -Leaf) -like $PipelineFilter } + $modulePipelines = $detailedAzurePipelines | Where-Object { (Split-Path $_.process.yamlFileName -Leaf) -like $PipelineFilter } | Sort-Object -Property 'Name' $azureDevOpsPipelineBadges = [System.Collections.ArrayList]@() @@ -259,10 +259,8 @@ function Invoke-PipelinesForBranch { $null = az pipelines run --branch $TargetBranch --id $modulePipeline.id } - # Generate pipeline badges if ($GeneratePipelineBadges) { - $pipelineDefinitionId = $modulePipeline.id $encodedPipelineName = [uri]::EscapeDataString($modulePipeline.Name) $encodedBranch = [uri]::EscapeDataString($TargetBranch) @@ -270,12 +268,6 @@ function Invoke-PipelinesForBranch { $secondaryUrl = 'https://dev.azure.com/{0}/{1}/_build/latest?definitionId={2}&branchName={3}' -f $AzureDevOpsOrganizationName, $AzureDevOpsProjectName, $pipelineDefinitionId, $encodedBranch $azureDevOpsPipelineBadges += "[![Build Status]($primaryUrl)]($secondaryUrl)" - - # [ - # ![Build Status] - # (https://dev.azure.com/servicescode/infra-as-code-source/_apis/build/status/CARML-Modules/AnalysisServices%20-%20Servers?branchName=issue%2F1127) - # ] - # (https://dev.azure.com/servicescode/infra-as-code-source/_build/latest?definitionId=2156&branchName=issue%2F1127) } } From a7a90da38857f979b7f43b237a0214cdb78d69ff Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 17:29:49 +0200 Subject: [PATCH 04/13] Update to latest --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index dbbdd1164b..f79404d2e2 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -194,12 +194,12 @@ function Invoke-PipelinesForBranch { GitHubRepositoryName = $GitHubRepositoryName } - # List all workflows + Write-Verbose 'Fetching current GitHub workflows' -Verbose $workflows = Get-GitHubModuleWorkflowList @baseInputObject -Filter $PipelineFilter $gitHubWorkflowBadges = [System.Collections.ArrayList]@() - # Invoke workflows for target branch + Write-Verbose "Triggering GitHub workflows for branch [$TargetBranch]" -Verbose foreach ($workflow in $workflows) { $workflowName = $workflow.name @@ -241,7 +241,6 @@ function Invoke-PipelinesForBranch { $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json Write-Verbose 'Fetching details' # Required as we need the original file path for filtering (which is only available when fetching the pipeline directly) - $detailedAzurePipelines = $azurePipelines | ForEach-Object -ThrottleLimit 10 -Parallel { Write-Verbose ('Fetching detailed information for pipeline [{0}]' -f $PSItem.name) az pipelines show --organization $USING:azureDevOpsOrgUrl --project $USING:AzureDevOpsProjectName --id $PSItem.id | ConvertFrom-Json @@ -251,7 +250,7 @@ function Invoke-PipelinesForBranch { $azureDevOpsPipelineBadges = [System.Collections.ArrayList]@() - # Invoke pipelines for target branch + Write-Verbose "Triggering Azure DevOps pipelines for branch [$TargetBranch]" -Verbose foreach ($modulePipeline in $modulePipelines) { From 205415ba8a4cb84dabb4e59f3ddb70cf88b658ba Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 17:56:15 +0200 Subject: [PATCH 05/13] Added documentation --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index f79404d2e2..5155c7e71f 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -147,6 +147,57 @@ function Get-GitHubModuleWorkflowList { } #endregion +<# +.SYNOPSIS +Trigger all pipelines for either Azure DevOps or GitHub + +.DESCRIPTION +Trigger all pipelines for either Azure DevOps or GitHub. By default, pipelines are filtered to CARML module pipelines. +Note, for Azure DevOps you'll need the 'azure-devops' extension: `az extension add --upgrade -n azure-devops` + +.PARAMETER PersonalAccessToken +Mandatory. The PAT to use to interact with either GitHub / Azure DevOps. + +.PARAMETER TargetBranch +Mandatory. The branch to run the pipelines for (e.g. `main`). + +.PARAMETER PipelineFilter +Optional. The pipeline files to filter down to. By default only files with a name that starts with 'ms.*' are considered. E.g. 'ms.network*'. + +.PARAMETER Environment +Optional. The environment to run the pipelines for. By default it's GitHub. + +.PARAMETER GeneratePipelineBadges +Optional. Generate pipeline status badges for the given pipeline configuration. + +.PARAMETER RepositoryRoot +Optional. The root of the repository. Used to load related functions in their folder path. + +.PARAMETER GitHubRepositoryOwner +Optional. The GitHub organization to run the workfows in. Required if the chosen environment is `GitHub`. Defaults to 'Azure'. + +.PARAMETER GitHubRepositoryName +Optional. The GitHub repository to run the workfows in. Required if the chosen environment is `GitHub`. Defaults to 'ResourceModules'. + +.PARAMETER AzureDevOpsOrganizationName +Optional. The Azure DevOps organization to run the pipelines in. Required if the chosen environment is `AzureDevOps`. + +.PARAMETER AzureDevOpsProjectName +Optional. The Azure DevOps project to run the pipelines in. Required if the chosen environment is `AzureDevOps`. + +.PARAMETER AzureDevOpsPipelineFolderPath +Optional. The folder in Azure DevOps the pipelines are registerd in. Required if the chosen environment is `AzureDevOps`. Defaults to 'CARML-Modules'. + +.EXAMPLE +Invoke-PipelinesForBranch -PAT '' -TargetBranch 'feature/branch' -Environment 'GitHub' -PipelineFilter 'ms.network.*' + +Run all GitHub workflows that start with 'ms.network.*' using branch 'feature/branch'. Also returns all GitHub status badges. + +.EXAMPLE +Invoke-PipelinesForBranch -PAT '' -TargetBranch 'feature/branch' -Environment 'AzureDevOps' -PipelineFilter 'ms.network.*' -AzureDevOpsOrganizationName 'contoso' -AzureDevOpsProjectName 'Sanchez' -AzureDevOpsPipelineFolderPath 'CARML-Modules' + +Run all Azure DevOps pipelines that start with 'ms.network.*' using branch 'feature/branch'. Also returns all Azure DevOps pipeline status badges. +#> function Invoke-PipelinesForBranch { [CmdletBinding(SupportsShouldProcess)] @@ -177,10 +228,10 @@ function Invoke-PipelinesForBranch { [string] $GitHubRepositoryName = 'ResourceModules', [Parameter(Mandatory = $false, ParameterSetName = 'AzureDevOps')] - [string] $AzureDevOpsOrganizationName = 'ResourceModules', + [string] $AzureDevOpsOrganizationName = '', [Parameter(Mandatory = $false, ParameterSetName = 'AzureDevOps')] - [string] $AzureDevOpsProjectName = 'ResourceModules', + [string] $AzureDevOpsProjectName, [Parameter(Mandatory = $false, ParameterSetName = 'AzureDevOps')] [string] $AzureDevOpsPipelineFolderPath = 'CARML-Modules' From 42dd4c6db28da542ee29504dfedb75612e65f611 Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 17:58:34 +0200 Subject: [PATCH 06/13] Update to latest --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index 5155c7e71f..7211aab570 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -189,12 +189,12 @@ Optional. The Azure DevOps project to run the pipelines in. Required if the chos Optional. The folder in Azure DevOps the pipelines are registerd in. Required if the chosen environment is `AzureDevOps`. Defaults to 'CARML-Modules'. .EXAMPLE -Invoke-PipelinesForBranch -PAT '' -TargetBranch 'feature/branch' -Environment 'GitHub' -PipelineFilter 'ms.network.*' +Invoke-PipelinesForBranch -PersonalAccessToken '' -TargetBranch 'feature/branch' -Environment 'GitHub' -PipelineFilter 'ms.network.*' Run all GitHub workflows that start with 'ms.network.*' using branch 'feature/branch'. Also returns all GitHub status badges. .EXAMPLE -Invoke-PipelinesForBranch -PAT '' -TargetBranch 'feature/branch' -Environment 'AzureDevOps' -PipelineFilter 'ms.network.*' -AzureDevOpsOrganizationName 'contoso' -AzureDevOpsProjectName 'Sanchez' -AzureDevOpsPipelineFolderPath 'CARML-Modules' +Invoke-PipelinesForBranch -PersonalAccessToken '' -TargetBranch 'feature/branch' -Environment 'AzureDevOps' -PipelineFilter 'ms.network.*' -AzureDevOpsOrganizationName 'contoso' -AzureDevOpsProjectName 'Sanchez' -AzureDevOpsPipelineFolderPath 'CARML-Modules' Run all Azure DevOps pipelines that start with 'ms.network.*' using branch 'feature/branch'. Also returns all Azure DevOps pipeline status badges. #> From 1a4d1fafcaac0c4cd31de1e79109ec0b817cf439 Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 18:15:16 +0200 Subject: [PATCH 07/13] Optimizations --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index 7211aab570..d26f6021ed 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -286,7 +286,7 @@ function Invoke-PipelinesForBranch { $PersonalAccessToken | az devops login # Set default Azure DevOps configuration (to not continously specify it on every command) - az devops configure --defaults organization=$orgUazureDevOpsOrgUrlrl project=$AzureDevOpsProjectName --use-git-aliases $true + az devops configure --defaults organization=$azureDevOpsOrgUrl project=$AzureDevOpsProjectName --use-git-aliases $true Write-Verbose "Get and list all [$AzureDevOpsOrganizationName/$AzureDevOpsProjectName] Azure DevOps pipelines in folder [$AzureDevOpsPipelineFolderPath]" $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json From a6b6577380acea03437878336496cc146a545483 Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 18:18:09 +0200 Subject: [PATCH 08/13] Update to latest --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index d26f6021ed..ac4b350113 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -289,7 +289,7 @@ function Invoke-PipelinesForBranch { az devops configure --defaults organization=$azureDevOpsOrgUrl project=$AzureDevOpsProjectName --use-git-aliases $true Write-Verbose "Get and list all [$AzureDevOpsOrganizationName/$AzureDevOpsProjectName] Azure DevOps pipelines in folder [$AzureDevOpsPipelineFolderPath]" - $azurePipelines = az pipelines list --organization $azureDevOpsOrgUrl --project $AzureDevOpsProjectName --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json + $azurePipelines = az pipelines list --folder-path $AzureDevOpsPipelineFolderPath | ConvertFrom-Json Write-Verbose 'Fetching details' # Required as we need the original file path for filtering (which is only available when fetching the pipeline directly) $detailedAzurePipelines = $azurePipelines | ForEach-Object -ThrottleLimit 10 -Parallel { @@ -305,7 +305,7 @@ function Invoke-PipelinesForBranch { foreach ($modulePipeline in $modulePipelines) { - if ($PSCmdlet.ShouldProcess("GitHub workflow [$WorkflowFileName] for branch [$TargetBranch]", 'Invoke')) { + if ($PSCmdlet.ShouldProcess(('GitHub workflow [{0}] for branch [{1}]' -f $modulePipeline.name, $TargetBranch), 'Invoke')) { $null = az pipelines run --branch $TargetBranch --id $modulePipeline.id } From ff593a31165334b561157c3269ab75bac32c177e Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 19:27:15 +0200 Subject: [PATCH 09/13] Added docs --- ...ibution guide - Validate module locally.md | 2 +- ...bution guide - Validate module on scale.md | 28 +++++++++++++++++++ docs/wiki/Contribution guide.md | 1 + 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 docs/wiki/Contribution guide - Validate module on scale.md diff --git a/docs/wiki/Contribution guide - Validate module locally.md b/docs/wiki/Contribution guide - Validate module locally.md index ec11fff779..bd3c70ab06 100644 --- a/docs/wiki/Contribution guide - Validate module locally.md +++ b/docs/wiki/Contribution guide - Validate module locally.md @@ -11,7 +11,7 @@ Use this script to test a module from your PC locally, without a CI environment. --- # Location -You can find the script under [`/utilities/tools/Test-ModuleLocally.ps1`](https://github.com/Azure/ResourceModules/blob/main/utilities/tools//Test-ModuleLocally.ps1) +You can find the script under [`/utilities/tools/Test-ModuleLocally.ps1`](https://github.com/Azure/ResourceModules/blob/main/utilities/tools/Test-ModuleLocally.ps1) # How it works diff --git a/docs/wiki/Contribution guide - Validate module on scale.md b/docs/wiki/Contribution guide - Validate module on scale.md new file mode 100644 index 0000000000..e6730d68b0 --- /dev/null +++ b/docs/wiki/Contribution guide - Validate module on scale.md @@ -0,0 +1,28 @@ +Use this script to test a branch across multiple modules using the CI environment. As the script will trigger the pipelines as you would in the CI environment, both static & deployment tests are executed. + +--- + +### _Navigation_ + +- [Location](#location) +- [How it works](#how-it-works) +- [How to use it](#how-to-use-it) + +--- +# Location + +You can find the script under [`/utilities/tools/Invoke-PipelinesForBranch.ps1`](https://github.com/Azure/ResourceModules/blob/main/utilities/tools/Invoke-PipelinesForBranch.ps1) + +# How it works + +The most important parameter is the 'Environment' you want to run the pipelines for, that is, either GitHub or Azure DevOps. Depending on your choice you'll have to provide a Personal Access Token that grants the permissions to read & trigger pipelines in the desired environment. + +Upon triggering, the utility will: +1. Fetch all pipelines in the target environment & filter them down to, by default, module pipelines. +1. Trigger these pipelines for the provided targeted branch (e.g. `main`) +1. Return the formatted status badges for the just triggered pipelines + +# How to use it + +For details on how to use the function, please refer to the script's local documentation. +> **Note:** The script must be loaded ('*dot-sourced*') before the function can be invoked. diff --git a/docs/wiki/Contribution guide.md b/docs/wiki/Contribution guide.md index 6103b823a0..c7a76ba0dc 100644 --- a/docs/wiki/Contribution guide.md +++ b/docs/wiki/Contribution guide.md @@ -8,4 +8,5 @@ This section provides the step-by-step process we suggest to follow for contribu - [Generate module Readme](./Contribution%20guide%20-%20Generate%20module%20Readme) - [Get formatted RBAC roles](./Contribution%20guide%20-%20Get%20formatted%20RBAC%20roles) - [Validate module locally](./Contribution%20guide%20-%20Validate%20module%20locally) +- [Validate modules on scale](./Contribution%20guide%20-%20Validate%20module%20on%20scale) --- From 4a1662189615060647380581cb343380f6076134 Mon Sep 17 00:00:00 2001 From: Alexander Sehr Date: Sun, 26 Jun 2022 23:07:31 +0200 Subject: [PATCH 10/13] Update docs/wiki/Contribution guide - Validate module on scale.md Co-authored-by: Marius Storhaug --- docs/wiki/Contribution guide - Validate module on scale.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wiki/Contribution guide - Validate module on scale.md b/docs/wiki/Contribution guide - Validate module on scale.md index e6730d68b0..2fa714dc5e 100644 --- a/docs/wiki/Contribution guide - Validate module on scale.md +++ b/docs/wiki/Contribution guide - Validate module on scale.md @@ -20,7 +20,7 @@ The most important parameter is the 'Environment' you want to run the pipelines Upon triggering, the utility will: 1. Fetch all pipelines in the target environment & filter them down to, by default, module pipelines. 1. Trigger these pipelines for the provided targeted branch (e.g. `main`) -1. Return the formatted status badges for the just triggered pipelines +1. Return the formatted status badges for the pipelines that were triggered. # How to use it From 5508432cf442c1fa04204c8a1331c8843e35d319 Mon Sep 17 00:00:00 2001 From: Alexander Sehr Date: Sun, 26 Jun 2022 23:07:36 +0200 Subject: [PATCH 11/13] Update docs/wiki/Contribution guide - Validate module on scale.md Co-authored-by: Marius Storhaug --- docs/wiki/Contribution guide - Validate module on scale.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wiki/Contribution guide - Validate module on scale.md b/docs/wiki/Contribution guide - Validate module on scale.md index 2fa714dc5e..0d19518abb 100644 --- a/docs/wiki/Contribution guide - Validate module on scale.md +++ b/docs/wiki/Contribution guide - Validate module on scale.md @@ -1,4 +1,4 @@ -Use this script to test a branch across multiple modules using the CI environment. As the script will trigger the pipelines as you would in the CI environment, both static & deployment tests are executed. +Use this script to tests multiple modules with a given branch, using the CI environment. The script will start the pipelines in the CI environment causing both static & deployment tests to run. --- From 6395225e5557f122755aeec1120fb3cbb943643d Mon Sep 17 00:00:00 2001 From: Alexander Sehr Date: Sun, 26 Jun 2022 23:09:54 +0200 Subject: [PATCH 12/13] Update docs/wiki/Contribution guide - Validate module on scale.md Co-authored-by: Marius Storhaug --- docs/wiki/Contribution guide - Validate module on scale.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wiki/Contribution guide - Validate module on scale.md b/docs/wiki/Contribution guide - Validate module on scale.md index 0d19518abb..f8e2dcc160 100644 --- a/docs/wiki/Contribution guide - Validate module on scale.md +++ b/docs/wiki/Contribution guide - Validate module on scale.md @@ -18,7 +18,7 @@ You can find the script under [`/utilities/tools/Invoke-PipelinesForBranch.ps1`] The most important parameter is the 'Environment' you want to run the pipelines for, that is, either GitHub or Azure DevOps. Depending on your choice you'll have to provide a Personal Access Token that grants the permissions to read & trigger pipelines in the desired environment. Upon triggering, the utility will: -1. Fetch all pipelines in the target environment & filter them down to, by default, module pipelines. +1. Fetch all pipelines in the target environment and filter them down to module pipelines by default. 1. Trigger these pipelines for the provided targeted branch (e.g. `main`) 1. Return the formatted status badges for the pipelines that were triggered. From d7cf4d13b7a5f6e215ec93e5275642cf1a35373b Mon Sep 17 00:00:00 2001 From: MrMCake Date: Sun, 26 Jun 2022 23:11:08 +0200 Subject: [PATCH 13/13] Improved runtime --- utilities/tools/Invoke-PipelinesForBranch.ps1 | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/utilities/tools/Invoke-PipelinesForBranch.ps1 b/utilities/tools/Invoke-PipelinesForBranch.ps1 index ac4b350113..143c017c68 100644 --- a/utilities/tools/Invoke-PipelinesForBranch.ps1 +++ b/utilities/tools/Invoke-PipelinesForBranch.ps1 @@ -299,32 +299,27 @@ function Invoke-PipelinesForBranch { $modulePipelines = $detailedAzurePipelines | Where-Object { (Split-Path $_.process.yamlFileName -Leaf) -like $PipelineFilter } | Sort-Object -Property 'Name' - $azureDevOpsPipelineBadges = [System.Collections.ArrayList]@() - Write-Verbose "Triggering Azure DevOps pipelines for branch [$TargetBranch]" -Verbose - foreach ($modulePipeline in $modulePipelines) { - - - if ($PSCmdlet.ShouldProcess(('GitHub workflow [{0}] for branch [{1}]' -f $modulePipeline.name, $TargetBranch), 'Invoke')) { - $null = az pipelines run --branch $TargetBranch --id $modulePipeline.id + $modulePipelines | ForEach-Object -ThrottleLimit 10 -Parallel { + if ($Using:WhatIfPreference) { + Write-Verbose ("Would performing the operation `"Invoke`" on target `"GitHub workflow [{0}] for branch [{1}]`"." -f $PSItem.Name, $USING:TargetBranch) -Verbose + } else { + $null = az pipelines run --branch $USING:TargetBranch --id $PSItem.id --organization $USING:azureDevOpsOrgUrl --project $USING:AzureDevOpsProjectName } + } - # Generate pipeline badges - if ($GeneratePipelineBadges) { + if ($GeneratePipelineBadges) { + foreach ($modulePipeline in $modulePipelines) { + + # Generate pipeline badges $pipelineDefinitionId = $modulePipeline.id $encodedPipelineName = [uri]::EscapeDataString($modulePipeline.Name) $encodedBranch = [uri]::EscapeDataString($TargetBranch) $primaryUrl = 'https://dev.azure.com/{0}/{1}/_apis/build/status/{2}/{3}?branchName={4}' -f $AzureDevOpsOrganizationName, $AzureDevOpsProjectName, $AzureDevOpsPipelineFolderPath, $encodedPipelineName, $encodedBranch $secondaryUrl = 'https://dev.azure.com/{0}/{1}/_build/latest?definitionId={2}&branchName={3}' -f $AzureDevOpsOrganizationName, $AzureDevOpsProjectName, $pipelineDefinitionId, $encodedBranch - $azureDevOpsPipelineBadges += "[![Build Status]($primaryUrl)]($secondaryUrl)" + Write-Verbose "[![Build Status]($primaryUrl)]($secondaryUrl)" -Verbose } } - - if ($azureDevOpsPipelineBadges.Count -gt 0) { - Write-Verbose 'Azure DevOps Pipeline Badges' -Verbose - Write-Verbose '============================' -Verbose - Write-Verbose ($azureDevOpsPipelineBadges | Sort-Object | Out-String) -Verbose - } } }