From cb604eb93dba5b7e4eb9ef259c823394b74f44a0 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Wed, 6 Jan 2021 15:45:16 -0800 Subject: [PATCH 01/13] Add weekly release pipeline. --- .ADO/common-templates/checkout.yml | 21 ++ .../download-openapi-docs.yml | 129 +++++++++ .ADO/common-templates/install-tools.yml | 71 +++++ .../generate-auth-module.yml | 191 ++++++++++++ .../generate-meta-module.yml | 148 ++++++++++ .../generate-service-modules.yml | 272 ++++++++++++++++++ .ADO/weekly-generation.yml | 43 +++ tools/GenerateModules.ps1 | 9 +- tools/GenerateProfiles.ps1 | 8 +- tools/SetMetaModuleVersion.ps1 | 15 + ...odules.ps1 => SetServiceModuleVersion.ps1} | 19 +- tools/UpdateOpenApi.ps1 | 1 - 12 files changed, 916 insertions(+), 11 deletions(-) create mode 100644 .ADO/common-templates/checkout.yml create mode 100644 .ADO/common-templates/download-openapi-docs.yml create mode 100644 .ADO/common-templates/install-tools.yml create mode 100644 .ADO/generation-templates/generate-auth-module.yml create mode 100644 .ADO/generation-templates/generate-meta-module.yml create mode 100644 .ADO/generation-templates/generate-service-modules.yml create mode 100644 .ADO/weekly-generation.yml create mode 100644 tools/SetMetaModuleVersion.ps1 rename tools/{SetVersionToAllModules.ps1 => SetServiceModuleVersion.ps1} (54%) diff --git a/.ADO/common-templates/checkout.yml b/.ADO/common-templates/checkout.yml new file mode 100644 index 00000000000..ffaf73ce62d --- /dev/null +++ b/.ADO/common-templates/checkout.yml @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +steps: + - checkout: self + clean: true + fetchDepth: 1 + persistCredentials: true + + - task: PowerShell@2 + displayName: Configure user + inputs: + targetType: inline + script: | + git config --global user.email "GraphTooling@service.microsoft.com" + git config --global user.name "Microsoft Graph DevX Tooling" + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 + displayName: Run CredScan + inputs: + debugMode: false diff --git a/.ADO/common-templates/download-openapi-docs.yml b/.ADO/common-templates/download-openapi-docs.yml new file mode 100644 index 00000000000..dfb2b1d76ae --- /dev/null +++ b/.ADO/common-templates/download-openapi-docs.yml @@ -0,0 +1,129 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: Branch + type: string + default: "weeklyOpenApiDocsDownload" + - name: BaseBranch + type: string + default: "dev" + +jobs: + - job: GetLatestDocs + displayName: Download OpenApiDocs + pool: MsGraphDevXAzureAgents + steps: + - template: ./checkout.yml + + - task: PowerShell@2 + displayName: Download v1.0 OpenApi docs + continueOnError: false + enabled: true + inputs: + filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" + pwsh: true + + - task: PowerShell@2 + displayName: Download beta OpenApi docs + continueOnError: false + enabled: false + inputs: + filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" + arguments: "-BetaGraphVersion" + pwsh: true + + - task: PowerShell@2 + name: OpenAPIDocDiff + displayName: Get OpenAPI diff + inputs: + pwsh: true + targetType: "inline" + script: | + $diff = git diff --name-only + $ModulesWithChanges = @{} + $diff | %{ + if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2)) + { + $ModulesWithChanges.Add($matches.2, $matches.1) + } + } + # TEST DATA (PLS REMOVE ME!!) + $ModulesWithChanges.Add("CloudCommunications", "CloudCommunications") + $ModulesWithChanges.Add("Search", "Search") + $ModuleNames = $ModulesWithChanges.Keys + Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" + + - task: PowerShell@2 + displayName: Generate profiles + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + enabled: false + continueOnError: false + inputs: + targetType: filePath + pwsh: true + filePath: $(System.DefaultWorkingDirectory)/tools/GenerateProfiles.ps1 + + - task: PowerShell@2 + name: CalculateAndBumpServiceModuleVersion + displayName: Calculate and bump service module version + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + inputs: + pwsh: true + targetType: inline + script: | + "$(OpenAPIDocDiff.ModulesWithChanges)" -split " " | ForEach-Object { + # Get PSGallery version + $Module = Find-Module "Microsoft.Graph.$_" -Repository PSGallery + $ModuleVersion = [System.Version]($Module.Version) + $NewModuleVersion = "$($ModuleVersion.Major).$($ModuleVersion.Minor + 1).$($ModuleVersion.Build)" + # Bump minor version + . $(System.DefaultWorkingDirectory)/tools/SetServiceModuleVersion.ps1 -VersionNumber $NewModuleVersion -Modules $_ + } + + - task: PowerShell@2 + name: CalculateAndBumpMetaModuleVersion + displayName: Calculate and bump meta-module version + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + inputs: + pwsh: true + targetType: inline + script: | + # Get PSGallery version + $Module = Find-Module "Microsoft.Graph" -Repository PSGallery + $ModuleVersion = [System.Version]($Module.Version) + $NewModuleVersion = "$($ModuleVersion.Major).$($ModuleVersion.Minor + 1).$($ModuleVersion.Build)" + # Bump minor version + . $(System.DefaultWorkingDirectory)/tools/SetMetaModuleVersion.ps1 -VersionNumber $NewModuleVersion + + - task: Bash@3 + displayName: "Create weekly branch: $(Branch)" + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + inputs: + targetType: inline + script: | + git status + git checkout $(BaseBranch) + git branch $(Branch) + git checkout $(Branch) + git status + + - task: Bash@3 + displayName: "Commit downloaded files" + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + env: + GITHUB_TOKEN: $(GITHUB_TOKEN) + inputs: + targetType: inline + script: | + git status + git add . + git commit -m 'Weekly OpenApiDocs Download' + git status + git push --set-upstream origin $(Branch) + git status + +# References +# [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables +# [1] https://hub.github.com/hub-pull-request.1.html +# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token diff --git a/.ADO/common-templates/install-tools.yml b/.ADO/common-templates/install-tools.yml new file mode 100644 index 00000000000..3fba2060ac5 --- /dev/null +++ b/.ADO/common-templates/install-tools.yml @@ -0,0 +1,71 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +steps: + - task: UseDotNet@2 + displayName: "Use .NET Core SDK 3.x" + inputs: + debugMode: false + version: 3.x + + - task: NuGetToolInstaller@1 + displayName: "Install Nuget 5.7" + inputs: + versionSpec: 5.7.0 + checkLatest: false # Optional + + - task: NuGetAuthenticate@0 + displayName: Authenticate NuGet + + - task: PowerShell@2 + displayName: "Install Powershell Core" + enabled: false + inputs: + targetType: inline + script: | + dotnet tool update --global PowerShell + pwsh + + - task: PowerShell@2 + displayName: "Version Check" + inputs: + targetType: inline + pwsh: true + script: | + Write-Host $PSVersionTable.PSVersion + Write-Host $host.Version + Write-Host (Get-Host).Version + + - task: NodeTool@0 + displayName: "Install NodeJs 14.11.0" + inputs: + versionSpec: "14.11.0" + checkLatest: true # Optional + + - task: Npm@1 + displayName: "Install AutoRest" + inputs: + command: "custom" + customCommand: "install -g autorest@latest" + + - task: PowerShell@2 + displayName: "Register PS Repository" + enabled: false + inputs: + targetType: inline + pwsh: true + errorActionPreference: "continue" + script: | + $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Continue + Get-PSRepository + $patToken = '$(NUGETFEEDKEY)' | ConvertTo-SecureString -AsPlainText -Force + $nugetFeed = '$(NUGETFEED)' + $user = '$(NUGETBUILDUSER)' + $credsAzureDevopsServices = New-Object System.Management.Automation.PSCredential($user, $patToken) + UnRegister-PackageSource -Name 'LocalNugetPackageSource' -ErrorAction Continue + UnRegister-PSRepository -Name 'LocalNugetFeed' -ErrorAction Continue + Register-PackageSource -Name 'LocalNugetPackageSource' -Location $nugetFeed -SkipValidate -Trusted -Verbose -ProviderName 'Nuget' -ErrorAction Continue + Register-PSRepository -Name 'LocalNugetFeed' -SourceLocation $nugetFeed -PublishLocation $nugetFeed -InstallationPolicy Trusted -Credential $credsAzureDevopsServices -PackageManagementProvider 'Nuget' -ErrorAction Continue + Get-PSRepository + Find-Module -Name Microsoft.Graph.Authentication -AllowPrerelease -Credential $credsAzureDevopsServices -AllVersions -Repository 'LocalNugetFeed' + Find-Module -Name Microsoft.Graph.Authentication -AllowPrerelease -Repository 'LocalNugetFeed' diff --git a/.ADO/generation-templates/generate-auth-module.yml b/.ADO/generation-templates/generate-auth-module.yml new file mode 100644 index 00000000000..d7e7b778676 --- /dev/null +++ b/.ADO/generation-templates/generate-auth-module.yml @@ -0,0 +1,191 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: AUTH_MODULE_PATH + displayName: 'Authentication Module Path' + type: string + default: 'src\Authentication\Authentication\' + - name: AUTH_MODULE_NAME + displayName: 'Authentication Module Name' + type: string + default: 'Authentication' + - name: AUTH_MODULE_DLL_PATTERN + displayName: 'Authentication Module DLL Pattern' + type: string + default: 'Microsoft.Graph.Authentication.dll' + - name: Api_Key + displayName: 'Api Key' + type: string + - name: EnableSigning + displayName: 'Enable Signing' + type: boolean + default: false + - name: BUILDNUMBER + displayName: 'Build Number' + type: string + default: $[format('{0:yyMMddHH}', pipeline.startTime)] + +jobs: +- job: GenerateAuthModule + displayName: Auth Module Generation + pool: MsGraphDevXAzureAgents + + steps: + - template: common-templates/install-tools.yml + + - task: PowerShell@2 + displayName: 'Generate and Build Auth Module' + inputs: + targetType: 'inline' + pwsh: true + script: | + pwsh $(System.DefaultWorkingDirectory)/tools/GenerateAuthenticationModule.ps1 -ArtifactsLocation $(Build.ArtifactStagingDirectory) -Build -EnableSigning -ModulePreviewNumber $(BUILDNUMBER) -RepositoryName "LocalNugetFeed" + + - task: DotNetCoreCLI@2 + displayName: 'Run Enabled Tests' + inputs: + command: 'test' + publishTestResults: true + projects: '$(System.DefaultWorkingDirectory)/src/Authentication/Authentication.Test/*.csproj' + testRunTitle: 'Run Enabled Tests' + + - task: RoslynAnalyzers@2 + displayName: 'Run Roslyn Analyzer' + inputs: + userProvideBuildInfo: 'msBuildInfo' + msBuildVersion: '16.0' + msBuildArchitecture: 'x64' + msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" $(System.DefaultWorkingDirectory)//src//Authentication//Authentication.sln /nologo /nr:false /p:VisualStudioVersion="16.0"' + + - task: CodeMetrics@1 + displayName: 'Run Code Metrics' + inputs: + Files: '$(System.DefaultWorkingDirectory)//**//Microsoft.Graph.Authentication.dll;$(System.DefaultWorkingDirectory)//**//Microsoft.Graph.Authentication.exe' + continueOnError: true + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP DLL Strong Name (Auth Module)' + enabled: true + inputs: + ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' + FolderPath: $(AUTH_MODULE_PATH) + Pattern: $(AUTH_MODULE_DLL_PATTERN) + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP DLL Code Signing (Auth Module)' + enabled: true + inputs: + ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' + FolderPath: $(AUTH_MODULE_PATH) + Pattern: $(AUTH_MODULE_DLL_PATTERN) + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: PowerShell@2 + displayName: 'Pack Auth Module' + inputs: + targetType: 'inline' + pwsh: true + script: | + pwsh $(System.DefaultWorkingDirectory)/tools/PackModule.ps1 -Module $(AUTH_MODULE_NAME) -ArtifactsLocation $(Build.ArtifactStagingDirectory) + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP NuGet Code Signing (Auth Module)' + enabled: true + inputs: + ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' + FolderPath: '$(Build.ArtifactStagingDirectory)\$(AUTH_MODULE_NAME)' + Pattern: 'Microsoft.Graph.$(AUTH_MODULE_NAME)*.nupkg' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: NuGetCommand@2 + displayName: 'Publish NuGet To Local Build Feed' + enabled: false + inputs: + command: push + packagesToPush: '$(Build.ArtifactStagingDirectory)\$(AUTH_MODULE_NAME)\Microsoft.Graph.$(AUTH_MODULE_NAME)*.nupkg' + publishVstsFeed: '0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57' + allowPackageConflicts: true + + - task: PublishBuildArtifacts@1 + displayName: Publish Microsoft.Graph.Authentication.nupkg Artifact' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(AUTH_MODULE_NAME)' + ArtifactName: 'drop' + publishLocation: 'Container' \ No newline at end of file diff --git a/.ADO/generation-templates/generate-meta-module.yml b/.ADO/generation-templates/generate-meta-module.yml new file mode 100644 index 00000000000..1dfbaf61534 --- /dev/null +++ b/.ADO/generation-templates/generate-meta-module.yml @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: ROLLUP_MODULE_PATH + displayName: "Module Path" + type: string + default: "src/Graph/Graph" + - name: ROLLUP_MODULE_NAME + displayName: "Module Name" + type: string + default: "Graph" + - name: Api_Key + displayName: "Api Key" + type: string + - name: GRAPH_VERSION + displayName: "Graph Version" + type: string + default: "beta" + - name: MODULE_PREFIX + displayName: "Module Prefix" + type: string + default: "Microsoft.Graph" + - name: EnableSigning + displayName: "Enable Signing" + type: boolean + default: false + - name: BUILDNUMBER + displayName: "Build Number" + type: string + default: $[format('{0:yyMMddHH}', pipeline.startTime)] + +jobs: + - job: GenerateMetaModule + displayName: Meta Module Generation + pool: MsGraphDevXAzureAgents + + steps: + - template: common-templates/install-tools.yml + + - task: PowerShell@2 + displayName: "Generate and Build Meta Module" + inputs: + targetType: "inline" + pwsh: true + script: | + Get-PSRepository + $(System.DefaultWorkingDirectory)/tools/GenerateRollUpModule.ps1 -RepositoryName 'LocalNugetFeed' -ModulePreviewNumber $(BUILDNUMBER) + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: "ESRP DLL Code Signing" + enabled: true + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(ROLLUP_MODULE_PATH) + Pattern: "$(MODULE_PREFIX).psm1, $(MODULE_PREFIX).*.format.ps1xml, *.ps1" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: NuGetCommand@2 + displayName: "Pack Meta Module" + inputs: + command: "pack" + Configuration: Release + packagesToPack: "$(System.DefaultWorkingDirectory)/$(ROLLUP_MODULE_PATH)/$(MODULE_PREFIX).nuspec" + packDestination: "$(Build.ArtifactStagingDirectory)/" + versioningScheme: "off" + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: "ESRP NuGet Code Signing" + enabled: true + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: "$(Build.ArtifactStagingDirectory)/" + Pattern: "Microsoft.Graph*.nupkg" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: NuGetCommand@2 + displayName: "Publish NuGet To Local Build Feed" + enabled: false + inputs: + command: push + packagesToPush: "$(Build.ArtifactStagingDirectory)/Microsoft.Graph*.nupkg" + publishVstsFeed: "0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57" + allowPackageConflicts: true + + - task: PublishBuildArtifacts@1 + displayName: "Publish Microsoft.Graph.nupkg Artifact" + inputs: + PathtoPublish: "$(Build.ArtifactStagingDirectory)/" + ArtifactName: "drop" + publishLocation: "Container" diff --git a/.ADO/generation-templates/generate-service-modules.yml b/.ADO/generation-templates/generate-service-modules.yml new file mode 100644 index 00000000000..e60aa9972e3 --- /dev/null +++ b/.ADO/generation-templates/generate-service-modules.yml @@ -0,0 +1,272 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: AuthModulePath + type: string + default: 'src\Authentication\Authentication\bin\' + - name: AuthModuleDllPattern + type: string + default: "Microsoft.Graph.Authentication.dll" + - name: ServiceModulePath + type: string + default: 'src\' + - name: ModulePrefix + type: string + default: "Microsoft.Graph" + - name: Branch + type: string + default: "" + - name: EnableSigning + type: boolean + default: false +jobs: + - job: GenerateServiceModules + displayName: Service module generation + pool: MsGraphDevXAzureAgents + timeoutInMinutes: 360 + condition: and(succeeded(), ne(stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + steps: + - template: ../common-templates/install-tools.yml + + - task: Bash@3 + displayName: "Switch branch to $(Branch)" + condition: and(succeeded(), ne('${{ parameters.Branch }}', '')) + enabled: true + inputs: + targetType: inline + script: | + git status + git pull + git checkout $(Branch) + git status + + # Consider making generate-auth-module a template with a collection of tasks instead to avoid rebuilding and signing auth module here. + - task: PowerShell@2 + displayName: Build auth module + enabled: false + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateAuthenticationModule.ps1 -ArtifactsLocation $(Build.ArtifactStagingDirectory) -Build -EnableSigning -BuildWhenEqual -RepositoryName "LocalNugetFeed" + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ESRP DLL strong name (Auth Module) + enabled: false + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(AuthModulePath) + Pattern: $(AuthModuleDllPattern) + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: "ESRP DLL code sign (Auth Module)" + enabled: false + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(AuthModulePath) + Pattern: $(AuthModuleDllPattern) + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + # End of suggestion. + + - task: PowerShell@2 + displayName: Generate and build service modules + enabled: true + inputs: + targetType: inline + pwsh: true + script: | + $Modules = "$(ModulesToGenerate)" -split " " + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -Test -UpdateAutoRest -ModulesToGenerate $Modules + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ESRP DLL strong name (Service Modules) + enabled: false + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(ServiceModulePath) + Pattern: "$(ModulePrefix).*.private.dll" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: "ESRP DLL code sign (Service Modules)" + enabled: false + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(ServiceModulePath) + Pattern: "$(ModulePrefix).*.private.dll, $(ModulePrefix).*.psm1, $(ModulePrefix).*.format.ps1xml, ProxyCmdletDefinitions.ps1, load-dependency.ps1" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 160 + + - task: PowerShell@2 + displayName: Pack service modules + enabled: false + inputs: + targetType: inline + pwsh: true + script: | + $ModuleMappingConfigPath = '$(System.DefaultWorkingDirectory)/config/ModulesMapping.jsonc' + [HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable + $ModuleMapping.Keys | ForEach-Object { + $ModuleName = $_ + $ModuleProjectDir = "$(System.DefaultWorkingDirectory)/src/$ModuleName/$ModuleName" + & $(System.DefaultWorkingDirectory)/tools/PackModule.ps1 -Module $ModuleName -ArtifactsLocation $(Build.ArtifactStagingDirectory)\ + } + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ESRP NuGet code sign (Service Modules) + enabled: false + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: '$(Build.ArtifactStagingDirectory)\' + Pattern: "*.nupkg" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: NuGetCommand@2 + displayName: Publish NuGet to local build feed + enabled: false + inputs: + command: push + packagesToPush: "$(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg" + publishVstsFeed: "0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57" + allowPackageConflicts: true + + - task: PublishBuildArtifacts@1 + displayName: Publish service modules build artifacts + enabled: false + inputs: + PathtoPublish: "$(Build.ArtifactStagingDirectory)/" + ArtifactName: "drop" + publishLocation: "Container" diff --git a/.ADO/weekly-generation.yml b/.ADO/weekly-generation.yml new file mode 100644 index 00000000000..bb6297a7c1d --- /dev/null +++ b/.ADO/weekly-generation.yml @@ -0,0 +1,43 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +pool: MsGraphDevXAzureAgents + +variables: + BranchDate: "$[format('{0:yyyyMMdd}', pipeline.startTime)]" + Branch: "testweeklyOpenApiDocsDownload/$(BranchDate)" + BaseBranch: "dev" + +schedules: + - cron: "0 12 * * WED" # Run every wednesday at noon UTC + displayName: "Weekly PS SDK generation" + branches: + include: + - dev + always: true + +stages: + - stage: DownloadOpenAPIDocs + displayName: Download OpenAPI docs + jobs: + - template: common-templates/download-openapi-docs.yml + parameters: + Branch: $(Branch) + BaseBranch: $(BaseBranch) + + - stage: GenerateServiceModules + displayName: Generate service modules + variables: + ModulesToGenerate: $[ stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + jobs: + - template: generation-templates/generate-service-modules.yml + parameters: + Branch: $(Branch) + EnableSigning: true + # Git diff on download OpenAPI docs. Store changed modules in a variable. + # Commit downloaded docs. + # If modules change, bump their minor and meta-module version. + # Generate changed modules + run tests. + # Commit generated files to weekly branch. + # Create PR to dev diff --git a/tools/GenerateModules.ps1 b/tools/GenerateModules.ps1 index 28b45e1f096..d0c8fef1c15 100644 --- a/tools/GenerateModules.ps1 +++ b/tools/GenerateModules.ps1 @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. Param( + $ModulesToGenerate = @(), [string] $RepositoryApiKey, [string] $RepositoryName = "PSGallery", [int] $ModulePreviewNumber = -1, @@ -73,9 +74,13 @@ if ($UpdateAutoRest) { # Update AutoRest. & autorest --reset } -[HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable -$ModuleMapping.Keys | ForEach-Object -ThrottleLimit $ModuleMapping.Keys.Count -Parallel { +if ($ModulesToGenerate.Count -eq 0) { + [HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable + $ModulesToGenerate = $ModuleMapping.Keys +} + +$ModulesToGenerate | ForEach-Object -ThrottleLimit $ModulesToGenerate.Count -Parallel { enum VersionState { Invalid Valid diff --git a/tools/GenerateProfiles.ps1 b/tools/GenerateProfiles.ps1 index 208faf13eb4..fdc79b04b7b 100644 --- a/tools/GenerateProfiles.ps1 +++ b/tools/GenerateProfiles.ps1 @@ -51,8 +51,8 @@ try { } # Get crawl data. Write-Host "Crawling '$moduleName' paths for resources and operations ..." -ForegroundColor Green - $crawlResult = @{resources= @(); operations = @{}} - foreach ($path in $allPaths) { + $crawlResult = @{resources= @(); operations = [ordered]@{}} + foreach ($path in ($allPaths | Sort-Object -Property endpoint)) { $crawlResult.operations[$path.endpoint] = (@{apiVersion = $path.apiVersion; originalLocation = $path.originalLocation}) } $telemetryDir = Join-Path $ModuleProfilesDirectory "crawl-log-$profileName.json" @@ -60,11 +60,11 @@ try { Write-Host "Telemetry written at $telemetryDir" -ForegroundColor Blue # Get profile. - $profile = @{resources = @{}; operations = @{}} + $profile = @{resources = @{}; operations = [ordered]@{}} foreach ($operation in $crawlResult.operations.keys) { $profile.operations[$operation] = $crawlResult.operations[$operation].apiVersion } - $profilesNode = @{profiles = @{ $profileName = $profile}} + $profilesNode = @{profiles = [ordered]@{ $profileName = $profile}} $profilesInYaml = $profilesNode | ConvertTo-Yaml $profileReadMeContent = @" # Microsoft Graph $profileName Profile diff --git a/tools/SetMetaModuleVersion.ps1 b/tools/SetMetaModuleVersion.ps1 new file mode 100644 index 00000000000..ef20547cc32 --- /dev/null +++ b/tools/SetMetaModuleVersion.ps1 @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +Param( + [string] $VersionNumber, + [string] $ReleaseNotes = "See https://aka.ms/GraphPowerShell-Release." +) + +$ModuleMetadataPath = Join-Path $PSScriptRoot "..\config\ModuleMetadata.json" -Resolve +$ModuleContent = (Get-Content $ModuleMetadataPath -Raw | ConvertFrom-Json) + +# Update meta-module +$ModuleContent.Version = $VersionNumber +$ModuleContent.ReleaseNotes = $ReleaseNotes + +Set-Content -Value (ConvertTo-Json $ModuleContent) -Path $ModuleMetadataPath diff --git a/tools/SetVersionToAllModules.ps1 b/tools/SetServiceModuleVersion.ps1 similarity index 54% rename from tools/SetVersionToAllModules.ps1 rename to tools/SetServiceModuleVersion.ps1 index d3ec0609823..ae439a2de9d 100644 --- a/tools/SetVersionToAllModules.ps1 +++ b/tools/SetServiceModuleVersion.ps1 @@ -2,12 +2,23 @@ # Licensed under the MIT License. Param( [string] $VersionNumber, - [string] $ReleaseNotes = "See https://aka.ms/GraphPowerShell-Release." + [string] $ReleaseNotes = "See https://aka.ms/GraphPowerShell-Release.", + [string[]] $Modules = $null ) $WriteToModuleReadMe = Join-Path $PSScriptRoot ".\WriteToModuleReadMe.ps1" -Resolve -$SrcPath = Join-Path $PSScriptRoot "..\src\*\*" -# Get all module readme.md -Get-Item "$SrcPath\readme.md" | ForEach-Object { + +$ModulesReadme = @() +if ($null -ne $Modules) { + $Modules | ForEach-Object { + $ReadMePath = Join-Path $PSScriptRoot "..\src\$_\$_\readme.md" + $ModulesReadme += Get-Item $ReadMePath + } +} else { + $ReadMePath = Join-Path $PSScriptRoot "..\src\*\*\readme.md" + # Get all module readme.md + $ModulesReadme = Get-Item $ReadMePath +} +$ModulesReadme | ForEach-Object { # Set readme values. & $WriteToModuleReadMe -ReadMePath $_.FullName -FieldName "module-version" -NewFieldValue $VersionNumber & $WriteToModuleReadMe -ReadMePath $_.FullName -FieldName "release-notes" -NewFieldValue $ReleaseNotes diff --git a/tools/UpdateOpenApi.ps1 b/tools/UpdateOpenApi.ps1 index 46d66a43a31..9ffe88bf8a0 100644 --- a/tools/UpdateOpenApi.ps1 +++ b/tools/UpdateOpenApi.ps1 @@ -47,5 +47,4 @@ $ModuleMapping.Keys | ForEach-Object -Begin { $RequestCount = 0 } -End { Write-H $RequestCount++ } - Write-Host -ForegroundColor Green "-------------Done-------------" \ No newline at end of file From 574ab1febd674f3589b5f90bee0e0c72ae5efb8f Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Thu, 14 Jan 2021 12:06:45 -0800 Subject: [PATCH 02/13] Update ADO pipelines. --- .../generate-meta-module.yml | 148 ------------------ .../common-templates/checkout.yml | 6 +- .../download-openapi-docs.yml | 62 ++++---- .../common-templates/install-tools.yml | 31 +++- .../download-openapidocs-template.yml | 71 --------- .../downloadopenapidocs-pipeline.yml | 33 ---- .../generate-auth-module.yml | 0 .../generate-meta-module.yml | 148 ++++++++++++++++++ .../generate-service-modules.yml | 23 +-- .azure-pipelines/integrated-pipeline.yml | 7 - .../weekly-generation.yml | 16 +- 11 files changed, 226 insertions(+), 319 deletions(-) delete mode 100644 .ADO/generation-templates/generate-meta-module.yml rename {.ADO => .azure-pipelines}/common-templates/checkout.yml (84%) rename {.ADO => .azure-pipelines}/common-templates/download-openapi-docs.yml (63%) rename {.ADO => .azure-pipelines}/common-templates/install-tools.yml (77%) delete mode 100644 .azure-pipelines/download-openapidocs-template.yml delete mode 100644 .azure-pipelines/downloadopenapidocs-pipeline.yml rename {.ADO => .azure-pipelines}/generation-templates/generate-auth-module.yml (100%) create mode 100644 .azure-pipelines/generation-templates/generate-meta-module.yml rename {.ADO => .azure-pipelines}/generation-templates/generate-service-modules.yml (95%) rename {.ADO => .azure-pipelines}/weekly-generation.yml (68%) diff --git a/.ADO/generation-templates/generate-meta-module.yml b/.ADO/generation-templates/generate-meta-module.yml deleted file mode 100644 index 1dfbaf61534..00000000000 --- a/.ADO/generation-templates/generate-meta-module.yml +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -parameters: - - name: ROLLUP_MODULE_PATH - displayName: "Module Path" - type: string - default: "src/Graph/Graph" - - name: ROLLUP_MODULE_NAME - displayName: "Module Name" - type: string - default: "Graph" - - name: Api_Key - displayName: "Api Key" - type: string - - name: GRAPH_VERSION - displayName: "Graph Version" - type: string - default: "beta" - - name: MODULE_PREFIX - displayName: "Module Prefix" - type: string - default: "Microsoft.Graph" - - name: EnableSigning - displayName: "Enable Signing" - type: boolean - default: false - - name: BUILDNUMBER - displayName: "Build Number" - type: string - default: $[format('{0:yyMMddHH}', pipeline.startTime)] - -jobs: - - job: GenerateMetaModule - displayName: Meta Module Generation - pool: MsGraphDevXAzureAgents - - steps: - - template: common-templates/install-tools.yml - - - task: PowerShell@2 - displayName: "Generate and Build Meta Module" - inputs: - targetType: "inline" - pwsh: true - script: | - Get-PSRepository - $(System.DefaultWorkingDirectory)/tools/GenerateRollUpModule.ps1 -RepositoryName 'LocalNugetFeed' -ModulePreviewNumber $(BUILDNUMBER) - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: "ESRP DLL Code Signing" - enabled: true - inputs: - ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" - FolderPath: $(ROLLUP_MODULE_PATH) - Pattern: "$(MODULE_PREFIX).psm1, $(MODULE_PREFIX).*.format.ps1xml, *.ps1" - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd \"SHA256\"" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolVerify", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 20 - - - task: NuGetCommand@2 - displayName: "Pack Meta Module" - inputs: - command: "pack" - Configuration: Release - packagesToPack: "$(System.DefaultWorkingDirectory)/$(ROLLUP_MODULE_PATH)/$(MODULE_PREFIX).nuspec" - packDestination: "$(Build.ArtifactStagingDirectory)/" - versioningScheme: "off" - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: "ESRP NuGet Code Signing" - enabled: true - inputs: - ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" - FolderPath: "$(Build.ArtifactStagingDirectory)/" - Pattern: "Microsoft.Graph*.nupkg" - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetSign", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetVerify", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 20 - - - task: NuGetCommand@2 - displayName: "Publish NuGet To Local Build Feed" - enabled: false - inputs: - command: push - packagesToPush: "$(Build.ArtifactStagingDirectory)/Microsoft.Graph*.nupkg" - publishVstsFeed: "0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57" - allowPackageConflicts: true - - - task: PublishBuildArtifacts@1 - displayName: "Publish Microsoft.Graph.nupkg Artifact" - inputs: - PathtoPublish: "$(Build.ArtifactStagingDirectory)/" - ArtifactName: "drop" - publishLocation: "Container" diff --git a/.ADO/common-templates/checkout.yml b/.azure-pipelines/common-templates/checkout.yml similarity index 84% rename from .ADO/common-templates/checkout.yml rename to .azure-pipelines/common-templates/checkout.yml index ffaf73ce62d..8c79b44211d 100644 --- a/.ADO/common-templates/checkout.yml +++ b/.azure-pipelines/common-templates/checkout.yml @@ -8,14 +8,14 @@ steps: persistCredentials: true - task: PowerShell@2 - displayName: Configure user + displayName: "Configure user" inputs: - targetType: inline + targetType: "inline" script: | git config --global user.email "GraphTooling@service.microsoft.com" git config --global user.name "Microsoft Graph DevX Tooling" - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 - displayName: Run CredScan + displayName: "Run CredScan" inputs: debugMode: false diff --git a/.ADO/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml similarity index 63% rename from .ADO/common-templates/download-openapi-docs.yml rename to .azure-pipelines/common-templates/download-openapi-docs.yml index dfb2b1d76ae..3526ca36dba 100644 --- a/.ADO/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -16,10 +16,11 @@ jobs: steps: - template: ./checkout.yml + - template: ./install-tools.yml + - task: PowerShell@2 displayName: Download v1.0 OpenApi docs continueOnError: false - enabled: true inputs: filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" pwsh: true @@ -27,7 +28,6 @@ jobs: - task: PowerShell@2 displayName: Download beta OpenApi docs continueOnError: false - enabled: false inputs: filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" arguments: "-BetaGraphVersion" @@ -35,7 +35,7 @@ jobs: - task: PowerShell@2 name: OpenAPIDocDiff - displayName: Get OpenAPI diff + displayName: Get OpenAPI docs diff inputs: pwsh: true targetType: "inline" @@ -48,8 +48,9 @@ jobs: $ModulesWithChanges.Add($matches.2, $matches.1) } } - # TEST DATA (PLS REMOVE ME!!) - $ModulesWithChanges.Add("CloudCommunications", "CloudCommunications") + + # TEST DATA (PLS REMOVE ME!!) + $ModulesWithChanges.Add("CloudCommunications", "CloudCommunications") $ModulesWithChanges.Add("Search", "Search") $ModuleNames = $ModulesWithChanges.Keys Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" @@ -57,7 +58,6 @@ jobs: - task: PowerShell@2 displayName: Generate profiles condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) - enabled: false continueOnError: false inputs: targetType: filePath @@ -65,36 +65,39 @@ jobs: filePath: $(System.DefaultWorkingDirectory)/tools/GenerateProfiles.ps1 - task: PowerShell@2 - name: CalculateAndBumpServiceModuleVersion - displayName: Calculate and bump service module version + name: CalculateAndBumpModuleVersion + displayName: Calculate and bump module version condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) inputs: pwsh: true targetType: inline script: | + # Calculate meta-module version + $MetaModule = Find-Module "Microsoft.Graph" -Repository PSGallery + $MetaModuleVersion = [System.Version]($MetaModule.Version) + $NewMetaModuleVersion = "$($MetaModuleVersion.Major).$($MetaModuleVersion.Minor + 1).$($MetaModuleVersion.Build)" + # Bump meta-module minor version + Write-Host "Bumping Microsoft.Graph to $NewMetaModuleVersion." + & "$(System.DefaultWorkingDirectory)\tools\SetMetaModuleVersion.ps1" -VersionNumber $NewMetaModuleVersion + + # Calculate existing service module version "$(OpenAPIDocDiff.ModulesWithChanges)" -split " " | ForEach-Object { - # Get PSGallery version - $Module = Find-Module "Microsoft.Graph.$_" -Repository PSGallery - $ModuleVersion = [System.Version]($Module.Version) - $NewModuleVersion = "$($ModuleVersion.Major).$($ModuleVersion.Minor + 1).$($ModuleVersion.Build)" - # Bump minor version - . $(System.DefaultWorkingDirectory)/tools/SetServiceModuleVersion.ps1 -VersionNumber $NewModuleVersion -Modules $_ + try { + $Module = Find-Module "Microsoft.Graph.$_" -Repository PSGallery -ErrorAction Stop + $ModuleVersion = [System.Version]($Module.Version) + $NewModuleVersion = "$($ModuleVersion.Major).$($ModuleVersion.Minor + 1).$($ModuleVersion.Build)" + Write-Host "Bumping $_ to $NewModuleVersion." + . "$(System.DefaultWorkingDirectory)\tools\SetServiceModuleVersion.ps1" -VersionNumber $NewModuleVersion -Modules $_ + } catch { + if ($_.Exception.Message -like "No match*") { + Write-Warning "$_. Version will be set to $NewMetaModuleVersion." + } + } } - - task: PowerShell@2 - name: CalculateAndBumpMetaModuleVersion - displayName: Calculate and bump meta-module version - condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) - inputs: - pwsh: true - targetType: inline - script: | - # Get PSGallery version - $Module = Find-Module "Microsoft.Graph" -Repository PSGallery - $ModuleVersion = [System.Version]($Module.Version) - $NewModuleVersion = "$($ModuleVersion.Major).$($ModuleVersion.Minor + 1).$($ModuleVersion.Build)" - # Bump minor version - . $(System.DefaultWorkingDirectory)/tools/SetMetaModuleVersion.ps1 -VersionNumber $NewModuleVersion + # Calculate new service module version + $NewModuleReadMePath = Join-Path $(System.DefaultWorkingDirectory) "/tools/Templates/readme.md" + . "$(System.DefaultWorkingDirectory)\tools\WriteToModuleReadMe.ps1" -ReadMePath $NewModuleReadMePath -FieldName "module-version" -NewFieldValue $NewMetaModuleVersion - task: Bash@3 displayName: "Create weekly branch: $(Branch)" @@ -103,13 +106,14 @@ jobs: targetType: inline script: | git status + git fetch --all git checkout $(BaseBranch) git branch $(Branch) git checkout $(Branch) git status - task: Bash@3 - displayName: "Commit downloaded files" + displayName: Commit downloaded files condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) env: GITHUB_TOKEN: $(GITHUB_TOKEN) diff --git a/.ADO/common-templates/install-tools.yml b/.azure-pipelines/common-templates/install-tools.yml similarity index 77% rename from .ADO/common-templates/install-tools.yml rename to .azure-pipelines/common-templates/install-tools.yml index 3fba2060ac5..be0c18f3112 100644 --- a/.ADO/common-templates/install-tools.yml +++ b/.azure-pipelines/common-templates/install-tools.yml @@ -3,13 +3,19 @@ steps: - task: UseDotNet@2 - displayName: "Use .NET Core SDK 3.x" + displayName: "Use .NET Core SDK 2.x" inputs: debugMode: false - version: 3.x + version: 2.x + + # - task: UseDotNet@2 + # displayName: "Use .NET Core SDK 3.x" + # inputs: + # debugMode: false + # version: 3.x - task: NuGetToolInstaller@1 - displayName: "Install Nuget 5.7" + displayName: Install Nuget 5.7 inputs: versionSpec: 5.7.0 checkLatest: false # Optional @@ -18,7 +24,7 @@ steps: displayName: Authenticate NuGet - task: PowerShell@2 - displayName: "Install Powershell Core" + displayName: Install Powershell core enabled: false inputs: targetType: inline @@ -27,7 +33,7 @@ steps: pwsh - task: PowerShell@2 - displayName: "Version Check" + displayName: Version check inputs: targetType: inline pwsh: true @@ -37,19 +43,28 @@ steps: Write-Host (Get-Host).Version - task: NodeTool@0 - displayName: "Install NodeJs 14.11.0" + displayName: Install NodeJs 14.11.0 inputs: versionSpec: "14.11.0" checkLatest: true # Optional - task: Npm@1 - displayName: "Install AutoRest" + displayName: Install AutoRest inputs: command: "custom" customCommand: "install -g autorest@latest" - task: PowerShell@2 - displayName: "Register PS Repository" + displayName: Install PowerShell dependencies + inputs: + targetType: inline + pwsh: true + errorActionPreference: "continue" + script: | + Install-Module "powershell-yaml" -Repository PSGallery -Force + + - task: PowerShell@2 + displayName: Register PS repository enabled: false inputs: targetType: inline diff --git a/.azure-pipelines/download-openapidocs-template.yml b/.azure-pipelines/download-openapidocs-template.yml deleted file mode 100644 index 451d2666534..00000000000 --- a/.azure-pipelines/download-openapidocs-template.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -parameters: -- name: BUILDNUMBER - displayName: 'Build Number' - type: string - default: $[format('{0:yyMMddHH}', pipeline.startTime)] - -jobs: -- job: MsGraphDownloadOpenApiDocs - displayName: Microsoft Graph PowerShell SDK Download OpenApiDocs Module Generation - pool: MsGraphDevXAzureAgents - timeoutInMinutes: 600 - - steps: - - checkout: self - persistCredentials: true - - - template: ./install-tools-template.yml - - - task: PowerShell@2 - displayName: 'Download v1.0 OpenApiDocs' - continueOnError: true - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - inputs: - targetType: 'filePath' - pwsh: true - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - - - - task: PowerShell@2 - displayName: 'Download beta OpenApiDocs' - continueOnError: false - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - inputs: - targetType: 'filePath' - pwsh: true - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - arguments: > - -BetaGraphVersion - - - task: PowerShell@2 - displayName: "Configure User" - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - inputs: - targetType: 'inline' - pwsh: true - script: | - git config --global user.email 'GraphTooling@service.microsoft.com' - git config --global user.name 'Microsoft Graph DevX Tooling' - - - task: PowerShell@2 - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - inputs: - targetType: 'inline' - script: | - $date = Get-Date -Format yyyyMMddHH - $docsBranch = "{0}{1}" -f $date, "buildDocsDownload" - git status - git checkout '$(Build.SourceBranchName)' - git branch $docsBranch - git checkout $docsBranch - git status - git add . - git commit -m '$(BUILDNUMBER): Build OpenApiDocs Download [skip ci]' - git status - git push --set-upstream origin $docsBranch - git status \ No newline at end of file diff --git a/.azure-pipelines/downloadopenapidocs-pipeline.yml b/.azure-pipelines/downloadopenapidocs-pipeline.yml deleted file mode 100644 index ca568ab115c..00000000000 --- a/.azure-pipelines/downloadopenapidocs-pipeline.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) - -pool: - vmImage: "windows-latest" - -variables: - BRANCH: 'weeklyOpenApiDocsDownload' - GitUserEmail: 'GraphTooling@service.microsoft.com' - GitUserName: 'Microsoft Graph DevX Tooling' - BaseBranch: 'dev' - -schedules: - - cron: "0 0 * * WED" # Run Every Wednesday - displayName: "Weekly OpenApiDocs Download and PR" - branches: - include: - - dev - always: true - -stages: -- stage: DownloadOpenApiDocs - displayName: 'Download Open Api Docs from DevX API' - jobs: - - job: DownloadOpenAPiDocs - steps: - - template: ./download-openapidocs-template.yml - parameters: - GitUserEmail: $(GitUserEmail) - GitUserName: $(GitUserName) - BaseBranch: $(BaseBranch) \ No newline at end of file diff --git a/.ADO/generation-templates/generate-auth-module.yml b/.azure-pipelines/generation-templates/generate-auth-module.yml similarity index 100% rename from .ADO/generation-templates/generate-auth-module.yml rename to .azure-pipelines/generation-templates/generate-auth-module.yml diff --git a/.azure-pipelines/generation-templates/generate-meta-module.yml b/.azure-pipelines/generation-templates/generate-meta-module.yml new file mode 100644 index 00000000000..bf05161baaf --- /dev/null +++ b/.azure-pipelines/generation-templates/generate-meta-module.yml @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: ROLLUP_MODULE_PATH + displayName: 'Module Path' + type: string + default: 'src/Graph/Graph' + - name: ROLLUP_MODULE_NAME + displayName: 'Module Name' + type: string + default: 'Graph' + - name: Api_Key + displayName: 'Api Key' + type: string + - name: GRAPH_VERSION + displayName: 'Graph Version' + type: string + default: 'beta' + - name: MODULE_PREFIX + displayName: 'Module Prefix' + type: string + default: 'Microsoft.Graph' + - name: EnableSigning + displayName: 'Enable Signing' + type: boolean + default: false + - name: BUILDNUMBER + displayName: 'Build Number' + type: string + default: $[format('{0:yyMMddHH}', pipeline.startTime)] + +jobs: +- job: GenerateMetaModule + displayName: Meta Module Generation + pool: MsGraphDevXAzureAgents + + steps: + - template: common-templates/install-tools.yml + + - task: PowerShell@2 + displayName: 'Generate and Build Meta Module' + inputs: + targetType: 'inline' + pwsh: true + script: | + Get-PSRepository + $(System.DefaultWorkingDirectory)/tools/GenerateRollUpModule.ps1 -RepositoryName 'LocalNugetFeed' -ModulePreviewNumber $(BUILDNUMBER) + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP DLL Code Signing' + enabled: true + inputs: + ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' + FolderPath: $(ROLLUP_MODULE_PATH) + Pattern: '$(MODULE_PREFIX).psm1, $(MODULE_PREFIX).*.format.ps1xml, *.ps1' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: NuGetCommand@2 + displayName: 'Pack Meta Module' + inputs: + command: 'pack' + Configuration: Release + packagesToPack: '$(System.DefaultWorkingDirectory)/$(ROLLUP_MODULE_PATH)/$(MODULE_PREFIX).nuspec' + packDestination: '$(Build.ArtifactStagingDirectory)/' + versioningScheme: 'off' + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP NuGet Code Signing' + enabled: true + inputs: + ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' + FolderPath: '$(Build.ArtifactStagingDirectory)/' + Pattern: 'Microsoft.Graph*.nupkg' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: NuGetCommand@2 + displayName: 'Publish NuGet To Local Build Feed' + enabled: false + inputs: + command: push + packagesToPush: '$(Build.ArtifactStagingDirectory)/Microsoft.Graph*.nupkg' + publishVstsFeed: '0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57' + allowPackageConflicts: true + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Microsoft.Graph.nupkg Artifact' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/' + ArtifactName: 'drop' + publishLocation: 'Container' \ No newline at end of file diff --git a/.ADO/generation-templates/generate-service-modules.yml b/.azure-pipelines/generation-templates/generate-service-modules.yml similarity index 95% rename from .ADO/generation-templates/generate-service-modules.yml rename to .azure-pipelines/generation-templates/generate-service-modules.yml index e60aa9972e3..51b297430a8 100644 --- a/.ADO/generation-templates/generate-service-modules.yml +++ b/.azure-pipelines/generation-templates/generate-service-modules.yml @@ -4,19 +4,14 @@ parameters: - name: AuthModulePath type: string - default: 'src\Authentication\Authentication\bin\' - name: AuthModuleDllPattern type: string - default: "Microsoft.Graph.Authentication.dll" - name: ServiceModulePath type: string - default: 'src\' - name: ModulePrefix type: string - default: "Microsoft.Graph" - name: Branch type: string - default: "" - name: EnableSigning type: boolean default: false @@ -32,19 +27,19 @@ jobs: - task: Bash@3 displayName: "Switch branch to $(Branch)" condition: and(succeeded(), ne('${{ parameters.Branch }}', '')) - enabled: true inputs: targetType: inline script: | git status - git pull + git fetch --all git checkout $(Branch) + git pull git status # Consider making generate-auth-module a template with a collection of tasks instead to avoid rebuilding and signing auth module here. - task: PowerShell@2 displayName: Build auth module - enabled: false + enabled: true inputs: targetType: inline pwsh: true @@ -53,7 +48,7 @@ jobs: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: ESRP DLL strong name (Auth Module) - enabled: false + enabled: true inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(AuthModulePath) @@ -80,7 +75,7 @@ jobs: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: "ESRP DLL code sign (Auth Module)" - enabled: false + enabled: true inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(AuthModulePath) @@ -129,17 +124,15 @@ jobs: - task: PowerShell@2 displayName: Generate and build service modules - enabled: true inputs: targetType: inline pwsh: true script: | $Modules = "$(ModulesToGenerate)" -split " " - . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -Test -UpdateAutoRest -ModulesToGenerate $Modules + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -Test -UpdateAutoRest -EnableSigning -ModulesToGenerate $Modules - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: ESRP DLL strong name (Service Modules) - enabled: false inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(ServiceModulePath) @@ -166,7 +159,6 @@ jobs: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: "ESRP DLL code sign (Service Modules)" - enabled: false inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(ServiceModulePath) @@ -214,7 +206,6 @@ jobs: - task: PowerShell@2 displayName: Pack service modules - enabled: false inputs: targetType: inline pwsh: true @@ -229,7 +220,6 @@ jobs: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: ESRP NuGet code sign (Service Modules) - enabled: false inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: '$(Build.ArtifactStagingDirectory)\' @@ -265,7 +255,6 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish service modules build artifacts - enabled: false inputs: PathtoPublish: "$(Build.ArtifactStagingDirectory)/" ArtifactName: "drop" diff --git a/.azure-pipelines/integrated-pipeline.yml b/.azure-pipelines/integrated-pipeline.yml index 6af676623ee..60e5e82eb2f 100644 --- a/.azure-pipelines/integrated-pipeline.yml +++ b/.azure-pipelines/integrated-pipeline.yml @@ -52,13 +52,6 @@ stages: jobs: - template: ./security-prechecks-template.yml -- stage: DownloadOpenApiDocs - displayName: 'Download Open Api Docs from DevX API' - jobs: - - template: ./download-openapidocs-template.yml - parameters: - BUILDNUMBER: $(BUILDNUMBER) - - stage: GenerateAuthModule displayName: 'Generate Authentication Module (Microsoft.Graph.Authentication)' jobs: diff --git a/.ADO/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml similarity index 68% rename from .ADO/weekly-generation.yml rename to .azure-pipelines/weekly-generation.yml index bb6297a7c1d..c5417a07249 100644 --- a/.ADO/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -5,16 +5,18 @@ name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) pool: MsGraphDevXAzureAgents variables: - BranchDate: "$[format('{0:yyyyMMdd}', pipeline.startTime)]" - Branch: "testweeklyOpenApiDocsDownload/$(BranchDate)" + BranchDate: "$[format('{0:yyyyMMddHHmm}', pipeline.startTime)]" + Branch: "test/weeklyOpenApiDocsDownload/$(BranchDate)" #TODO: Remove test data. BaseBranch: "dev" - +trigger: none +pr: none schedules: - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: include: - dev + - po/* #TODO: Remove test data. always: true stages: @@ -30,11 +32,19 @@ stages: displayName: Generate service modules variables: ModulesToGenerate: $[ stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + AuthModulePath: "src/Authentication/Authentication/bin/" + AuthModuleDllPattern: "Microsoft.Graph.Authentication.dll" + ServiceModulePath: "src/" + ModulePrefix: "Microsoft.Graph" jobs: - template: generation-templates/generate-service-modules.yml parameters: Branch: $(Branch) EnableSigning: true + AuthModulePath: $(AuthModulePath) + AuthModuleDllPattern: $(AuthModuleDllPattern) + ServiceModulePath: $(ServiceModulePath) + ModulePrefix: $(ModulePrefix) # Git diff on download OpenAPI docs. Store changed modules in a variable. # Commit downloaded docs. # If modules change, bump their minor and meta-module version. From c568d785ef03ec963e39177eef7155d94646531b Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Thu, 14 Jan 2021 12:12:51 -0800 Subject: [PATCH 03/13] Add retry to OpenAPI docs download. --- tools/DownloadOpenApiDoc.ps1 | 54 ++++++++++++++++++++++++++++++------ tools/GenerateModules.ps1 | 15 +++++----- tools/UpdateOpenApi.ps1 | 2 +- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/tools/DownloadOpenApiDoc.ps1 b/tools/DownloadOpenApiDoc.ps1 index 3b1e7cfd081..55ca298de55 100644 --- a/tools/DownloadOpenApiDoc.ps1 +++ b/tools/DownloadOpenApiDoc.ps1 @@ -17,14 +17,50 @@ if ($ForceRefresh.IsPresent) { $OpenApiServiceUrl = "$OpenApiServiceUrl&forceRefresh=true" } Write-Host -ForegroundColor Green "[$RequestCount] Downloading OpenAPI doc for '$ModuleName' module: $OpenApiServiceUrl" -try { - Invoke-WebRequest $OpenApiServiceUrl -OutFile "$OpenApiDocOutput\$ModuleName.yml" -} -catch { - # Get the Http Error Message from DevX Api, Rethrow Error to be handled Upstream - $ErrorMessage = $_.Exception.Message - Write-Host -ForegroundColor Red "[$RequestCount] Request Failed for $ModuleName Error Message: $ErrorMessage" - if ($GraphVersion -eq "beta") { - throw +$Retries = 3 +$Delay = 3 +$Retrycount = 0 +$Completed = $false +while (-not $Completed) { + try { + Invoke-WebRequest $OpenApiServiceUrl -OutFile "$OpenApiDocOutput\$ModuleName.yml" + $Completed = $true + } + catch { + $Exception = $_.Exception + switch ($Exception.Response.StatusCode.value__) { + { @(429, 503, 504) -contains $_ } { + if ($Retrycount -ge $Retries) { + Write-Warning "Request to $OpenApiServiceUrl failed the maximum number of $Retrycount times." + throw + } + else { + [ref]$RetryAfterHeader = $null + if ($Exception.Response.Headers.TryGetValues("Retry-After", $RetryAfterHeader)) { + # Use Retry-After response header + $DelayInSeconds = $RetryAfterHeader.Value + } + else { + # Use exponential backoff + $mPow = [math]::Pow(2, $Retrycount) + $DelayInSeconds = $mPow * $Delay + } + + Write-Warning "Request to $OpenApiServiceUrl failed. Retrying in $DelayInSeconds seconds." + Start-Sleep $DelayInSeconds + $Retrycount++ + } + } + 404 { + Write-Warning "Request to $OpenApiServiceUrl returned 404. Download will be skipped." + $Completed = $true + } + default { + # Get HTTP error message from DevX api, Re-throw error to be handled Upstream + $ErrorMessage = $Exception.Message + Write-Warning "[$RequestCount] Request for $ModuleName failed with error message: $ErrorMessage" + throw + } + } } } \ No newline at end of file diff --git a/tools/GenerateModules.ps1 b/tools/GenerateModules.ps1 index d0c8fef1c15..8f2871bd6e4 100644 --- a/tools/GenerateModules.ps1 +++ b/tools/GenerateModules.ps1 @@ -51,7 +51,7 @@ if (-not (Test-Path $ModuleMappingConfigPath)) { } $AllowPreRelease = $true -if($ModulePreviewNumber -eq -1) { +if ($ModulePreviewNumber -eq -1) { $AllowPreRelease = $false } # Install module locally in order to specify it as a dependency for other modules down the generation pipeline. @@ -61,7 +61,7 @@ Write-Host -ForegroundColor Green "Auth Module: $($ExistingAuthModule.Name), $($ if (!(Get-Module -Name $ExistingAuthModule.Name -ListAvailable)) { Install-Module $ExistingAuthModule.Name -Repository $RepositoryName -Force -AllowClobber -AllowPrerelease:$AllowPreRelease } -if($ExistingAuthModule.Version -like '*preview*' ) { +if ($ExistingAuthModule.Version -like '*preview*' ) { $version = $ExistingAuthModule.Version.Remove($ExistingAuthModule.Version.IndexOf('-')) Write-Warning "Required Version: $ModulePrefix.$RequiredModule Version: $version" $RequiredGraphModules += @{ ModuleName = $ExistingAuthModule.Name ; ModuleVersion = $version } @@ -92,7 +92,7 @@ $ModulesToGenerate | ForEach-Object -ThrottleLimit $ModulesToGenerate.Count -Par $FullyQualifiedModuleName = "$using:ModulePrefix.$ModuleName" Write-Host -ForegroundColor Green "Generating '$FullyQualifiedModuleName' module..." $ModuleProjectDir = Join-Path $Using:ModulesOutputDir "$ModuleName\$ModuleName" - + # Copy AutoRest readme.md config is none exists. if (-not (Test-Path "$ModuleProjectDir\readme.md")) { New-Item -Path $ModuleProjectDir -Type Directory -Force @@ -149,7 +149,7 @@ $ModulesToGenerate | ForEach-Object -ThrottleLimit $ModulesToGenerate.Count -Par # Get profiles for generated modules. $ModuleExportsPath = Join-Path $ModuleProjectDir "\exports" - $Profiles = Get-ChildItem -Path $ModuleExportsPath -Directory | %{ $_.Name} + $Profiles = Get-ChildItem -Path $ModuleExportsPath -Directory | % { $_.Name } # Update module manifest wiht profiles. $ModuleManifestPath = Join-Path $ModuleProjectDir "$FullyQualifiedModuleName.psd1" @@ -158,12 +158,13 @@ $ModulesToGenerate | ForEach-Object -ThrottleLimit $ModulesToGenerate.Count -Par # Update module psm1 with Graph session profile name. $ModulePsm1 = Join-Path $ModuleProjectDir "/$FullyQualifiedModuleName.psm1" - (Get-Content -Path $ModulePsm1) | ForEach-Object{ + (Get-Content -Path $ModulePsm1) | ForEach-Object { if ($_ -match '\$instance = \[Microsoft.Graph.PowerShell.Module\]::Instance') { # Update main psm1 with Graph session profile name and module name. $_ ' $instance.ProfileName = [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.SelectedProfile' - } else { + } + else { # Rename all Azure instances in psm1 to `Microsoft Graph`. $updatedLine = $_ -replace 'Azure', 'Microsoft Graph' # Replace all 'instance.Name' declarations with fully qualified module name. @@ -176,7 +177,7 @@ $ModulesToGenerate | ForEach-Object -ThrottleLimit $ModulesToGenerate.Count -Par # Address AutoREST bug where it looks for exports in the wrong directory. $InternalModulePsm1 = Join-Path $ModuleProjectDir "/internal/$FullyQualifiedModuleName.internal.psm1" - (Get-Content -Path $InternalModulePsm1) | ForEach-Object{ + (Get-Content -Path $InternalModulePsm1) | ForEach-Object { $updatedLine = $_ # Address AutoREST bug where it looks for exports in the wrong directory. if ($_ -match '\$exportsPath = \$PSScriptRoot') { diff --git a/tools/UpdateOpenApi.ps1 b/tools/UpdateOpenApi.ps1 index 9ffe88bf8a0..f02fa046956 100644 --- a/tools/UpdateOpenApi.ps1 +++ b/tools/UpdateOpenApi.ps1 @@ -33,7 +33,7 @@ $ModuleMapping.Keys | ForEach-Object -Begin { $RequestCount = 0 } -End { Write-H $ModuleName = $_ $ForceRefresh = $false # Check whether ForceRefresh is required, Only required for the First Request. - if ($RequestCount -eq 0){ + if ($RequestCount -eq 0) { $ForceRefresh = $true } From c8d25028db16f7774420bd21a6710db7218ab6c9 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Fri, 15 Jan 2021 16:57:24 -0800 Subject: [PATCH 04/13] Pipeline cleanup --- .../download-openapi-docs.yml | 40 ++++--- .azure-pipelines/download-openapidocs.yml | 102 ------------------ .../generate-service-modules.yml | 46 +++++--- .azure-pipelines/weekly-generation.yml | 29 +++-- 4 files changed, 68 insertions(+), 149 deletions(-) delete mode 100644 .azure-pipelines/download-openapidocs.yml diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 3526ca36dba..73be0766690 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -4,7 +4,7 @@ parameters: - name: Branch type: string - default: "weeklyOpenApiDocsDownload" + default: "WeeklyOpenApiDocsDownload" - name: BaseBranch type: string default: "dev" @@ -18,6 +18,27 @@ jobs: - template: ./install-tools.yml + - task: PowerShell@2 + name: "ComputeBranch" + displayName: "Compute weekly branch name" + inputs: + targetType: inline + script: | + $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) + Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch" + + - task: Bash@3 + displayName: "Create weekly branch" + inputs: + targetType: inline + script: | + git status + git fetch --all + git checkout $(BaseBranch) + git branch $(ComputeBranch.WeeklyBranch) + git checkout $(ComputeBranch.WeeklyBranch) + git status + - task: PowerShell@2 displayName: Download v1.0 OpenApi docs continueOnError: false @@ -49,7 +70,7 @@ jobs: } } - # TEST DATA (PLS REMOVE ME!!) + #TODO: Remove test data. $ModulesWithChanges.Add("CloudCommunications", "CloudCommunications") $ModulesWithChanges.Add("Search", "Search") $ModuleNames = $ModulesWithChanges.Keys @@ -99,19 +120,6 @@ jobs: $NewModuleReadMePath = Join-Path $(System.DefaultWorkingDirectory) "/tools/Templates/readme.md" . "$(System.DefaultWorkingDirectory)\tools\WriteToModuleReadMe.ps1" -ReadMePath $NewModuleReadMePath -FieldName "module-version" -NewFieldValue $NewMetaModuleVersion - - task: Bash@3 - displayName: "Create weekly branch: $(Branch)" - condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) - inputs: - targetType: inline - script: | - git status - git fetch --all - git checkout $(BaseBranch) - git branch $(Branch) - git checkout $(Branch) - git status - - task: Bash@3 displayName: Commit downloaded files condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) @@ -124,7 +132,7 @@ jobs: git add . git commit -m 'Weekly OpenApiDocs Download' git status - git push --set-upstream origin $(Branch) + git push --set-upstream origin $(ComputeBranch.WeeklyBranch) git status # References diff --git a/.azure-pipelines/download-openapidocs.yml b/.azure-pipelines/download-openapidocs.yml deleted file mode 100644 index 791c457dca9..00000000000 --- a/.azure-pipelines/download-openapidocs.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Generates a release build artifact (nuget) from HEAD of master for auth module. -name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) - -pool: - vmImage: "windows-latest" - -variables: - BRANCH: 'weeklyOpenApiDocsDownload' - GitUserEmail: 'GraphTooling@service.microsoft.com' - GitUserName: 'Microsoft Graph DevX Tooling' - BaseBranch: 'dev' - -schedules: - - cron: "0 0 * * WED" # Run Every Wednesday - displayName: "Weekly OpenApiDocs Download and PR" - branches: - include: - - dev - always: true - -steps: - - checkout: self - persistCredentials: true - clean: true - fetchDepth: 1 - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 - displayName: 'Run CredScan' - inputs: - debugMode: false - - - task: PowerShell@2 - displayName: "Compute Branch" - inputs: - targetType: inline - script: | - $branch = "{0}.{1}" -f "weeklyOpenApiDocsDownload", (Get-Date -Format yyyyMMdd) - Write-Host "##vso[task.setvariable variable=BRANCH;]$branch" - - - task: PowerShell@2 - displayName: "Configure User" - inputs: - targetType: 'inline' - script: | - git config --global user.email '$(GitUserEmail)' - git config --global user.name '$(GitUserName)' - - - task: PowerShell@2 - displayName: "Show Directory" - inputs: - targetType: 'inline' - script: | - ls $(System.DefaultWorkingDirectory) - ls $(System.DefaultWorkingDirectory)/tools - - - task: PowerShell@2 - displayName: Download v1.0 OpenApiDocs - continueOnError: false - inputs: - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - pwsh: true - - - task: PowerShell@2 - displayName: Download beta OpenApiDocs - continueOnError: false - inputs: - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - arguments: '-BetaGraphVersion' - pwsh: true - - - task: Bash@3 - displayName : "Create PR $(BRANCH)" - inputs: - targetType: 'inline' - script: | - git status - git checkout $(BaseBranch) - git branch $(BRANCH) - git checkout $(BRANCH) - git status - - - task: Bash@3 - displayName : "Commit Downloaded Files" - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - inputs: - targetType: 'inline' - script: | - git status - git add . - git commit -m 'Weekly OpenApiDocs Download' - git status - git push --set-upstream origin $(BRANCH) - git status - -# References -# [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables -# [1] https://hub.github.com/hub-pull-request.1.html -# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token \ No newline at end of file diff --git a/.azure-pipelines/generation-templates/generate-service-modules.yml b/.azure-pipelines/generation-templates/generate-service-modules.yml index 51b297430a8..f88ae7e811f 100644 --- a/.azure-pipelines/generation-templates/generate-service-modules.yml +++ b/.azure-pipelines/generation-templates/generate-service-modules.yml @@ -2,6 +2,8 @@ # Licensed under the MIT License. parameters: + - name: ModulesToGenerate + type: string - name: AuthModulePath type: string - name: AuthModuleDllPattern @@ -15,18 +17,29 @@ parameters: - name: EnableSigning type: boolean default: false + - name: PublishToFeed + type: boolean + default: false jobs: - job: GenerateServiceModules displayName: Service module generation pool: MsGraphDevXAzureAgents timeoutInMinutes: 360 condition: and(succeeded(), ne(stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + variables: + Branch: ${{ parameters.Branch }} + ModulesToGenerate: ${{ parameters.ModulesToGenerate }} + AuthModulePath: ${{ parameters.AuthModulePath }} + AuthModuleDllPattern: ${{ parameters.AuthModuleDllPattern }} + ServiceModulePath: ${{ parameters.ServiceModulePath }} + ModulePrefix: ${{ parameters.ModulePrefix }} + EnableSigning: ${{ parameters.EnableSigning }} + PublishToFeed: ${{ parameters.PublishToFeed }} steps: - template: ../common-templates/install-tools.yml - task: Bash@3 displayName: "Switch branch to $(Branch)" - condition: and(succeeded(), ne('${{ parameters.Branch }}', '')) inputs: targetType: inline script: | @@ -36,7 +49,6 @@ jobs: git pull git status - # Consider making generate-auth-module a template with a collection of tasks instead to avoid rebuilding and signing auth module here. - task: PowerShell@2 displayName: Build auth module enabled: true @@ -44,11 +56,12 @@ jobs: targetType: inline pwsh: true script: | - . $(System.DefaultWorkingDirectory)/tools/GenerateAuthenticationModule.ps1 -ArtifactsLocation $(Build.ArtifactStagingDirectory) -Build -EnableSigning -BuildWhenEqual -RepositoryName "LocalNugetFeed" + [bool]$EnableSigning = if ("$(EnableSigning)" -eq "true") { $true } else { $false } + . $(System.DefaultWorkingDirectory)/tools/GenerateAuthenticationModule.ps1 -ArtifactsLocation $(Build.ArtifactStagingDirectory) -Build -EnableSigning:$EnableSigning -BuildWhenEqual -RepositoryName "LocalNugetFeed" - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: ESRP DLL strong name (Auth Module) - enabled: true + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(AuthModulePath) @@ -75,7 +88,7 @@ jobs: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: "ESRP DLL code sign (Auth Module)" - enabled: true + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(AuthModulePath) @@ -120,7 +133,6 @@ jobs: } ] SessionTimeout: 20 - # End of suggestion. - task: PowerShell@2 displayName: Generate and build service modules @@ -128,11 +140,13 @@ jobs: targetType: inline pwsh: true script: | + [bool]$EnableSigning = if ("$(EnableSigning)" -eq "true") { $true } else { $false } $Modules = "$(ModulesToGenerate)" -split " " - . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -Test -UpdateAutoRest -EnableSigning -ModulesToGenerate $Modules + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -Test -UpdateAutoRest -EnableSigning:$EnableSigning -ModulesToGenerate $Modules - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: ESRP DLL strong name (Service Modules) + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(ServiceModulePath) @@ -159,6 +173,7 @@ jobs: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: "ESRP DLL code sign (Service Modules)" + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: $(ServiceModulePath) @@ -220,6 +235,7 @@ jobs: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: ESRP NuGet code sign (Service Modules) + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) inputs: ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" FolderPath: '$(Build.ArtifactStagingDirectory)\' @@ -244,18 +260,18 @@ jobs: ] SessionTimeout: 20 + - task: PublishBuildArtifacts@1 + displayName: Publish service modules build artifacts + inputs: + PathtoPublish: "$(Build.ArtifactStagingDirectory)/" + ArtifactName: "drop" + publishLocation: "Container" + - task: NuGetCommand@2 displayName: Publish NuGet to local build feed - enabled: false + condition: and(succeeded(), eq('${{ parameters.PublishToFeed }}', true)) inputs: command: push packagesToPush: "$(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg" publishVstsFeed: "0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57" allowPackageConflicts: true - - - task: PublishBuildArtifacts@1 - displayName: Publish service modules build artifacts - inputs: - PathtoPublish: "$(Build.ArtifactStagingDirectory)/" - ArtifactName: "drop" - publishLocation: "Container" diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index c5417a07249..75c3c7d67a6 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -5,18 +5,19 @@ name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) pool: MsGraphDevXAzureAgents variables: - BranchDate: "$[format('{0:yyyyMMddHHmm}', pipeline.startTime)]" - Branch: "test/weeklyOpenApiDocsDownload/$(BranchDate)" #TODO: Remove test data. - BaseBranch: "dev" + Branch: "test/WeeklyOpenApiDocsDownload" #TODO: Remove test data. + BaseBranch: "features/WeeklyReleaseUpdate" #TODO: Remove test data. trigger: none pr: none schedules: - - cron: "0 12 * * WED" # Run every wednesday at noon UTC + - cron: "59 0 * * *" #TODO: Remove test data. + # - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: include: - dev - po/* #TODO: Remove test data. + - features/* #TODO: Remove test data. always: true stages: @@ -30,21 +31,17 @@ stages: - stage: GenerateServiceModules displayName: Generate service modules - variables: - ModulesToGenerate: $[ stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] - AuthModulePath: "src/Authentication/Authentication/bin/" - AuthModuleDllPattern: "Microsoft.Graph.Authentication.dll" - ServiceModulePath: "src/" - ModulePrefix: "Microsoft.Graph" jobs: - template: generation-templates/generate-service-modules.yml parameters: - Branch: $(Branch) - EnableSigning: true - AuthModulePath: $(AuthModulePath) - AuthModuleDllPattern: $(AuthModuleDllPattern) - ServiceModulePath: $(ServiceModulePath) - ModulePrefix: $(ModulePrefix) + ModulesToGenerate: $[ stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + Branch: $[ stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['ComputeBranch.WeeklyBranch'] ] + AuthModulePath: "src/Authentication/Authentication/bin/" + AuthModuleDllPattern: "Microsoft.Graph.Authentication.dll" + ServiceModulePath: "src/" + ModulePrefix: "Microsoft.Graph" + EnableSigning: false + PublishToFeed: false # Git diff on download OpenAPI docs. Store changed modules in a variable. # Commit downloaded docs. # If modules change, bump their minor and meta-module version. From 1919d4c850dab10b740b80f882b1bf74605b8e05 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Fri, 15 Jan 2021 16:59:32 -0800 Subject: [PATCH 05/13] Update schedule run --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 75c3c7d67a6..8321958269a 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -10,7 +10,7 @@ variables: trigger: none pr: none schedules: - - cron: "59 0 * * *" #TODO: Remove test data. + - cron: "00 01 * * *" #TODO: Remove test data. # - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: From b332e235918507600237c5bbd3a0acbe5ae0670f Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Fri, 15 Jan 2021 17:02:20 -0800 Subject: [PATCH 06/13] Update weekly-generation.yml for Azure Pipelines --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 8321958269a..8cf6f928e60 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -10,7 +10,7 @@ variables: trigger: none pr: none schedules: - - cron: "00 01 * * *" #TODO: Remove test data. + - cron: "03 01 * * *" # TODO: Remove test data. # - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: From c554393d970c9e1fcfaf18b07be80a336ae03101 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Tue, 19 Jan 2021 08:53:56 -0800 Subject: [PATCH 07/13] Enable signing. --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 8cf6f928e60..a90efd1e38c 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -40,7 +40,7 @@ stages: AuthModuleDllPattern: "Microsoft.Graph.Authentication.dll" ServiceModulePath: "src/" ModulePrefix: "Microsoft.Graph" - EnableSigning: false + EnableSigning: true PublishToFeed: false # Git diff on download OpenAPI docs. Store changed modules in a variable. # Commit downloaded docs. From 6c946f67f469fbc016d92318024baed618a30152 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Tue, 19 Jan 2021 08:55:32 -0800 Subject: [PATCH 08/13] Update cron trigger. --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index a90efd1e38c..a7665a5e1d9 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -10,7 +10,7 @@ variables: trigger: none pr: none schedules: - - cron: "03 01 * * *" # TODO: Remove test data. + - cron: "16 57 * * *" # TODO: Remove test data. # - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: From 53c500205b89bdfd72c0ee24a06797caf0b11b94 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Tue, 19 Jan 2021 09:07:57 -0800 Subject: [PATCH 09/13] Update cron trigger to 17. --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index a7665a5e1d9..6c5abb236d1 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -10,7 +10,7 @@ variables: trigger: none pr: none schedules: - - cron: "16 57 * * *" # TODO: Remove test data. + - cron: "17 10 * * *" # TODO: Remove test data. # - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: From ba73acf8b396944ccc97c48d62f69409a035ea26 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Tue, 19 Jan 2021 09:13:09 -0800 Subject: [PATCH 10/13] Update weekly-generation.yml for Azure Pipelines --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 6c5abb236d1..0485f30afa1 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -10,7 +10,7 @@ variables: trigger: none pr: none schedules: - - cron: "17 10 * * *" # TODO: Remove test data. + - cron: "17 15 * * *" # TODO: Remove test data. # - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: From 262ad28ec9a26171ab27dcd47e2447172f9b166f Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Tue, 19 Jan 2021 09:19:37 -0800 Subject: [PATCH 11/13] Update weekly-generation.yml for Azure Pipelines --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 0485f30afa1..2eef6221597 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -10,7 +10,7 @@ variables: trigger: none pr: none schedules: - - cron: "17 15 * * *" # TODO: Remove test data. + - cron: "21 17 * * *" # TODO: Remove test data. # - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: From 1782e320c38ae554ae2c78b5ae6c7ef73c90d39e Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Tue, 19 Jan 2021 10:28:03 -0800 Subject: [PATCH 12/13] Code clean-up --- .../download-openapi-docs.yml | 4 - .../common-templates/install-tools.yml | 7 - .../generate-auth-module.yml | 191 ------------------ .../generate-meta-module.yml | 148 -------------- .../generate-service-modules.yml | 1 - .azure-pipelines/weekly-generation.yml | 17 +- 6 files changed, 4 insertions(+), 364 deletions(-) delete mode 100644 .azure-pipelines/generation-templates/generate-auth-module.yml delete mode 100644 .azure-pipelines/generation-templates/generate-meta-module.yml diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 73be0766690..d6d9447deb9 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -69,10 +69,6 @@ jobs: $ModulesWithChanges.Add($matches.2, $matches.1) } } - - #TODO: Remove test data. - $ModulesWithChanges.Add("CloudCommunications", "CloudCommunications") - $ModulesWithChanges.Add("Search", "Search") $ModuleNames = $ModulesWithChanges.Keys Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" diff --git a/.azure-pipelines/common-templates/install-tools.yml b/.azure-pipelines/common-templates/install-tools.yml index be0c18f3112..a3a73d85288 100644 --- a/.azure-pipelines/common-templates/install-tools.yml +++ b/.azure-pipelines/common-templates/install-tools.yml @@ -8,12 +8,6 @@ steps: debugMode: false version: 2.x - # - task: UseDotNet@2 - # displayName: "Use .NET Core SDK 3.x" - # inputs: - # debugMode: false - # version: 3.x - - task: NuGetToolInstaller@1 displayName: Install Nuget 5.7 inputs: @@ -25,7 +19,6 @@ steps: - task: PowerShell@2 displayName: Install Powershell core - enabled: false inputs: targetType: inline script: | diff --git a/.azure-pipelines/generation-templates/generate-auth-module.yml b/.azure-pipelines/generation-templates/generate-auth-module.yml deleted file mode 100644 index d7e7b778676..00000000000 --- a/.azure-pipelines/generation-templates/generate-auth-module.yml +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -parameters: - - name: AUTH_MODULE_PATH - displayName: 'Authentication Module Path' - type: string - default: 'src\Authentication\Authentication\' - - name: AUTH_MODULE_NAME - displayName: 'Authentication Module Name' - type: string - default: 'Authentication' - - name: AUTH_MODULE_DLL_PATTERN - displayName: 'Authentication Module DLL Pattern' - type: string - default: 'Microsoft.Graph.Authentication.dll' - - name: Api_Key - displayName: 'Api Key' - type: string - - name: EnableSigning - displayName: 'Enable Signing' - type: boolean - default: false - - name: BUILDNUMBER - displayName: 'Build Number' - type: string - default: $[format('{0:yyMMddHH}', pipeline.startTime)] - -jobs: -- job: GenerateAuthModule - displayName: Auth Module Generation - pool: MsGraphDevXAzureAgents - - steps: - - template: common-templates/install-tools.yml - - - task: PowerShell@2 - displayName: 'Generate and Build Auth Module' - inputs: - targetType: 'inline' - pwsh: true - script: | - pwsh $(System.DefaultWorkingDirectory)/tools/GenerateAuthenticationModule.ps1 -ArtifactsLocation $(Build.ArtifactStagingDirectory) -Build -EnableSigning -ModulePreviewNumber $(BUILDNUMBER) -RepositoryName "LocalNugetFeed" - - - task: DotNetCoreCLI@2 - displayName: 'Run Enabled Tests' - inputs: - command: 'test' - publishTestResults: true - projects: '$(System.DefaultWorkingDirectory)/src/Authentication/Authentication.Test/*.csproj' - testRunTitle: 'Run Enabled Tests' - - - task: RoslynAnalyzers@2 - displayName: 'Run Roslyn Analyzer' - inputs: - userProvideBuildInfo: 'msBuildInfo' - msBuildVersion: '16.0' - msBuildArchitecture: 'x64' - msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" $(System.DefaultWorkingDirectory)//src//Authentication//Authentication.sln /nologo /nr:false /p:VisualStudioVersion="16.0"' - - - task: CodeMetrics@1 - displayName: 'Run Code Metrics' - inputs: - Files: '$(System.DefaultWorkingDirectory)//**//Microsoft.Graph.Authentication.dll;$(System.DefaultWorkingDirectory)//**//Microsoft.Graph.Authentication.exe' - continueOnError: true - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP DLL Strong Name (Auth Module)' - enabled: true - inputs: - ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' - FolderPath: $(AUTH_MODULE_PATH) - Pattern: $(AUTH_MODULE_DLL_PATTERN) - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-233863-SN", - "operationSetCode": "StrongNameSign", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-233863-SN", - "operationSetCode": "StrongNameVerify", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 20 - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP DLL Code Signing (Auth Module)' - enabled: true - inputs: - ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' - FolderPath: $(AUTH_MODULE_PATH) - Pattern: $(AUTH_MODULE_DLL_PATTERN) - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd \"SHA256\"" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolVerify", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 20 - - - task: PowerShell@2 - displayName: 'Pack Auth Module' - inputs: - targetType: 'inline' - pwsh: true - script: | - pwsh $(System.DefaultWorkingDirectory)/tools/PackModule.ps1 -Module $(AUTH_MODULE_NAME) -ArtifactsLocation $(Build.ArtifactStagingDirectory) - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP NuGet Code Signing (Auth Module)' - enabled: true - inputs: - ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' - FolderPath: '$(Build.ArtifactStagingDirectory)\$(AUTH_MODULE_NAME)' - Pattern: 'Microsoft.Graph.$(AUTH_MODULE_NAME)*.nupkg' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetSign", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetVerify", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 20 - - - task: NuGetCommand@2 - displayName: 'Publish NuGet To Local Build Feed' - enabled: false - inputs: - command: push - packagesToPush: '$(Build.ArtifactStagingDirectory)\$(AUTH_MODULE_NAME)\Microsoft.Graph.$(AUTH_MODULE_NAME)*.nupkg' - publishVstsFeed: '0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57' - allowPackageConflicts: true - - - task: PublishBuildArtifacts@1 - displayName: Publish Microsoft.Graph.Authentication.nupkg Artifact' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(AUTH_MODULE_NAME)' - ArtifactName: 'drop' - publishLocation: 'Container' \ No newline at end of file diff --git a/.azure-pipelines/generation-templates/generate-meta-module.yml b/.azure-pipelines/generation-templates/generate-meta-module.yml deleted file mode 100644 index bf05161baaf..00000000000 --- a/.azure-pipelines/generation-templates/generate-meta-module.yml +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -parameters: - - name: ROLLUP_MODULE_PATH - displayName: 'Module Path' - type: string - default: 'src/Graph/Graph' - - name: ROLLUP_MODULE_NAME - displayName: 'Module Name' - type: string - default: 'Graph' - - name: Api_Key - displayName: 'Api Key' - type: string - - name: GRAPH_VERSION - displayName: 'Graph Version' - type: string - default: 'beta' - - name: MODULE_PREFIX - displayName: 'Module Prefix' - type: string - default: 'Microsoft.Graph' - - name: EnableSigning - displayName: 'Enable Signing' - type: boolean - default: false - - name: BUILDNUMBER - displayName: 'Build Number' - type: string - default: $[format('{0:yyMMddHH}', pipeline.startTime)] - -jobs: -- job: GenerateMetaModule - displayName: Meta Module Generation - pool: MsGraphDevXAzureAgents - - steps: - - template: common-templates/install-tools.yml - - - task: PowerShell@2 - displayName: 'Generate and Build Meta Module' - inputs: - targetType: 'inline' - pwsh: true - script: | - Get-PSRepository - $(System.DefaultWorkingDirectory)/tools/GenerateRollUpModule.ps1 -RepositoryName 'LocalNugetFeed' -ModulePreviewNumber $(BUILDNUMBER) - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP DLL Code Signing' - enabled: true - inputs: - ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' - FolderPath: $(ROLLUP_MODULE_PATH) - Pattern: '$(MODULE_PREFIX).psm1, $(MODULE_PREFIX).*.format.ps1xml, *.ps1' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd \"SHA256\"" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolVerify", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 20 - - - task: NuGetCommand@2 - displayName: 'Pack Meta Module' - inputs: - command: 'pack' - Configuration: Release - packagesToPack: '$(System.DefaultWorkingDirectory)/$(ROLLUP_MODULE_PATH)/$(MODULE_PREFIX).nuspec' - packDestination: '$(Build.ArtifactStagingDirectory)/' - versioningScheme: 'off' - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP NuGet Code Signing' - enabled: true - inputs: - ConnectedServiceName: 'microsoftgraph ESRP CodeSign DLL and NuGet (AKV)' - FolderPath: '$(Build.ArtifactStagingDirectory)/' - Pattern: 'Microsoft.Graph*.nupkg' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetSign", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetVerify", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 20 - - - task: NuGetCommand@2 - displayName: 'Publish NuGet To Local Build Feed' - enabled: false - inputs: - command: push - packagesToPush: '$(Build.ArtifactStagingDirectory)/Microsoft.Graph*.nupkg' - publishVstsFeed: '0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57' - allowPackageConflicts: true - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Microsoft.Graph.nupkg Artifact' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/' - ArtifactName: 'drop' - publishLocation: 'Container' \ No newline at end of file diff --git a/.azure-pipelines/generation-templates/generate-service-modules.yml b/.azure-pipelines/generation-templates/generate-service-modules.yml index f88ae7e811f..09292d1f572 100644 --- a/.azure-pipelines/generation-templates/generate-service-modules.yml +++ b/.azure-pipelines/generation-templates/generate-service-modules.yml @@ -51,7 +51,6 @@ jobs: - task: PowerShell@2 displayName: Build auth module - enabled: true inputs: targetType: inline pwsh: true diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 2eef6221597..eb9951d2780 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -5,19 +5,16 @@ name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) pool: MsGraphDevXAzureAgents variables: - Branch: "test/WeeklyOpenApiDocsDownload" #TODO: Remove test data. - BaseBranch: "features/WeeklyReleaseUpdate" #TODO: Remove test data. + Branch: "WeeklyOpenApiDocsDownload" + BaseBranch: "dev" trigger: none pr: none schedules: - - cron: "21 17 * * *" # TODO: Remove test data. - # - cron: "0 12 * * WED" # Run every wednesday at noon UTC + - cron: "0 12 * * WED" # Run every wednesday at noon UTC displayName: "Weekly PS SDK generation" branches: include: - dev - - po/* #TODO: Remove test data. - - features/* #TODO: Remove test data. always: true stages: @@ -40,11 +37,5 @@ stages: AuthModuleDllPattern: "Microsoft.Graph.Authentication.dll" ServiceModulePath: "src/" ModulePrefix: "Microsoft.Graph" - EnableSigning: true + EnableSigning: false PublishToFeed: false - # Git diff on download OpenAPI docs. Store changed modules in a variable. - # Commit downloaded docs. - # If modules change, bump their minor and meta-module version. - # Generate changed modules + run tests. - # Commit generated files to weekly branch. - # Create PR to dev From dfd5b7413dce5530c4f537e60cdf4d0075946894 Mon Sep 17 00:00:00 2001 From: Peter Ombwa Date: Thu, 21 Jan 2021 11:10:32 -0800 Subject: [PATCH 13/13] Update github action on:push:branch --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c9ea2c5ef4..1d3735ee470 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ name: WeeklyOpenApiDocsDownload on: push: branches: - - 'weeklyOpenApiDocsDownload.*' + - 'WeeklyOpenApiDocsDownload/*' # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: