diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index f9ead686ff..61ab6eebc1 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -78,11 +78,6 @@ parameters: type: string default: $(ci_var_defaultPoolName) - # True to add a Stress Test stage to the pipeline. - - name: enableStressTests - type: boolean - default: false - # The timeout, in minutes, for each test job. - name: testJobTimeout type: number @@ -212,19 +207,6 @@ stages: loggingArtifactsName: $(loggingArtifactsName) mdsArtifactsName: $(mdsArtifactsName) - # Run the stress tests, if desired. - - ${{ if eq(parameters.enableStressTests, true) }}: - - template: /eng/pipelines/stages/stress-tests-ci-stage.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - additionalDependsOn: - - build_sqlclient_package_stage - - build_azure_package_stage - mdsArtifactsName: $(mdsArtifactsName) - mdsPackageVersion: $(mdsPackageVersion) - azurePackageVersion: $(azurePackageVersion) - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - # Run the MDS and AKV tests. - template: /eng/pipelines/common/templates/stages/ci-run-tests-stage.yml@self parameters: diff --git a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml index 262c2b909a..d38fb14d57 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml @@ -103,12 +103,6 @@ parameters: type: boolean default: false - # True to run the stress tests stage. - - name: enableStressTests - displayName: Enable Stress Tests - type: boolean - default: false - # The target frameworks to build and run tests for on Windows. # # These are _not_ the target frameworks to build the driver packages for. @@ -173,7 +167,6 @@ extends: referenceType: Package debug: ${{ parameters.debug }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - enableStressTests: ${{ parameters.enableStressTests }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} diff --git a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml index a5a78f0f8b..7fef476e59 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml @@ -103,12 +103,6 @@ parameters: type: boolean default: false - # True to run the stress tests stage. - - name: enableStressTests - displayName: Enable Stress Tests - type: boolean - default: false - # The target frameworks to build and run tests for on Windows. # # These are _not_ the target frameworks to build the driver packages for. @@ -173,7 +167,6 @@ extends: referenceType: Project debug: ${{ parameters.debug }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - enableStressTests: ${{ parameters.enableStressTests }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} diff --git a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml index 9db1c05696..aed7672618 100644 --- a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml @@ -78,9 +78,7 @@ jobs: # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public project. # This is defined with a non-standard Platform of 'AnyCPU', and will fail - # the builds if left defined. The stress tests solution does not require - # any specific Platform, and so its solution file doesn't support any - # non-standard platforms. + # the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. diff --git a/eng/pipelines/jobs/pack-azure-package-ci-job.yml b/eng/pipelines/jobs/pack-azure-package-ci-job.yml index 95853d43e0..3b03dc0adc 100644 --- a/eng/pipelines/jobs/pack-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-azure-package-ci-job.yml @@ -107,9 +107,7 @@ jobs: # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public # project. This is defined with a non-standard Platform of 'AnyCPU', and - # will fail the builds if left defined. The stress tests solution does - # not require any specific Platform, and so its solution file doesn't - # support any non-standard platforms. + # will fail the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. diff --git a/eng/pipelines/jobs/stress-tests-ci-job.yml b/eng/pipelines/jobs/stress-tests-ci-job.yml deleted file mode 100644 index 7477295f54..0000000000 --- a/eng/pipelines/jobs/stress-tests-ci-job.yml +++ /dev/null @@ -1,194 +0,0 @@ -################################################################################ -# Licensed to the .NET Foundation under one or more agreements. The .NET -# Foundation licenses this file to you under the MIT license. See the LICENSE -# file in the project root for more information. -################################################################################ - -# This job builds and runs stress tests against an MDS NuGet package available -# as a pipeline artifact. -# -# The stress tests are located here: -# -# src/Microsoft.Data.SqlClient/tests/StressTests -# -# This template defines a job named 'run_stress_tests_job_' that can be -# depended on by downstream jobs. - -parameters: - # The suffix to append to the job name. - - name: jobNameSuffix - type: string - default: '' - - # The prefix to prepend to the job's display name: - # - # [] Run Stress Tests - # - - name: displayNamePrefix - type: string - default: '' - - # The name of the Azure Pipelines pool to use. - - name: poolName - type: string - default: '' - - # The pool VM image to use. - - name: vmImage - type: string - default: '' - - # The pipeline step to run to configure SQL Server. - - name: sqlSetupStep - type: step - - # The name of the MDS pipeline artifacts to download. - - name: mdsArtifactsName - type: string - default: '' - - # The solution file to restore/build. - - name: solution - type: string - default: '' - - # The test project to run. - - name: testProject - type: string - default: '' - - # dotnet CLI arguments for the restore step. - - name: restoreArguments - type: string - default: '' - - # dotnet CLI arguments for the build and run steps. - - name: buildArguments - type: string - default: '' - - # The list of .NET runtimes to test against. - - name: netTestRuntimes - type: object - default: [] - - # The list of .NET Framework runtimes to test against. - - name: netFrameworkTestRuntimes - type: object - default: [] - - # The stress test config file contents to write to the config file. - - name: configContent - type: string - default: '' - -jobs: -- job: run_stress_tests_job_${{ parameters.jobNameSuffix }} - displayName: '[${{ parameters.displayNamePrefix }}] Run Stress Tests' - pool: - name: ${{ parameters.poolName }} - ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} - ${{ else }}: - demands: - - imageOverride -equals ${{ parameters.vmImage }} - - variables: - # Stress test command-line arguments. - - name: testArguments - value: -a SqlClient.Stress.Tests -console - - # Explicitly unset the $PLATFORM environment variable that is set by the - # 'ADO Build properties' Library in the ADO SqlClientDrivers public project. - # This is defined with a non-standard Platform of 'AnyCPU', and will fail - # the builds if left defined. The stress tests solution does not require - # any specific Platform, and so its solution file doesn't support any - # non-standard platforms. - # - # Note that Azure Pipelines will inject this variable as PLATFORM into the - # environment of all tasks in this job. - # - # See: - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch - # - - name: Platform - value: '' - - # Do the same for $CONFIGURATION since we explicitly set it using our - # 'buildConfiguration' parameter, and we don't want the environment to - # override us. - - name: Configuration - value: '' - - steps: - - # Install the .NET SDK and Runtimes. - - template: /eng/pipelines/steps/install-dotnet.yml@self - parameters: - runtimes: [8.x, 9.x] - - # Download the pipeline artifact that contains the MDS package to test. - - task: DownloadPipelineArtifact@2 - displayName: Download MDS Artifacts - inputs: - artifactName: ${{ parameters.mdsArtifactsName }} - targetPath: $(Build.SourcesDirectory)/packages - - # Setup the local SQL Server. - - ${{ parameters.sqlSetupStep }} - - # We use the 'custom' command because the DotNetCoreCLI@2 task doesn't support - # all of our argument combinations for the different build steps. - - # Restore the solution. - - task: DotNetCoreCLI@2 - displayName: Restore Solution - inputs: - command: custom - custom: restore - projects: ${{ parameters.solution }} - arguments: ${{ parameters.restoreArguments }} - - # Build the solution. - - task: DotNetCoreCLI@2 - displayName: Build Solution - inputs: - command: custom - custom: build - projects: ${{ parameters.solution }} - arguments: ${{ parameters.buildArguments }} --no-restore - - # Write the config file. - - task: PowerShell@2 - displayName: Write Config File - inputs: - pwsh: true - targetType: inline - script: | - # Capture the multi-line JSON content into a variable. - $content = @" - ${{ parameters.configContent }} - "@ - - # Write the JSON content to the config file. - $content | Out-File -FilePath "config.json" - - # Run the stress tests for each .NET runtime. - - ${{ each runtime in parameters.netTestRuntimes }}: - - task: DotNetCoreCLI@2 - displayName: Test [${{runtime}}] - inputs: - command: custom - custom: run - projects: ${{ parameters.testProject }} - arguments: ${{ parameters.buildArguments }} --no-build -f ${{runtime}} -e STRESS_CONFIG_FILE=config.json -- $(testArguments) - - # Run the stress tests for each .NET Framework runtime. - - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: - - task: DotNetCoreCLI@2 - displayName: Test [${{runtime}}] - inputs: - command: custom - custom: run - projects: ${{ parameters.testProject }} - arguments: ${{ parameters.buildArguments }} --no-build -f ${{runtime}} -e STRESS_CONFIG_FILE=config.json -- $(testArguments) diff --git a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml index 9d205b5108..a88d4dc347 100644 --- a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml @@ -27,7 +27,7 @@ parameters: # The prefix to prepend to the job's display name: # - # [] Run Stress Tests + # [] Test Abstractions Package # - name: displayNamePrefix type: string @@ -97,9 +97,7 @@ jobs: # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public project. # This is defined with a non-standard Platform of 'AnyCPU', and will fail - # the builds if left defined. The stress tests solution does not require - # any specific Platform, and so its solution file doesn't support any - # non-standard platforms. + # the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. diff --git a/eng/pipelines/jobs/test-azure-package-ci-job.yml b/eng/pipelines/jobs/test-azure-package-ci-job.yml index 53ca26366f..5e5dffe381 100644 --- a/eng/pipelines/jobs/test-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/test-azure-package-ci-job.yml @@ -46,7 +46,7 @@ parameters: # The prefix to prepend to the job's display name: # - # [] Run Stress Tests + # [] Test Azure Package # - name: displayNamePrefix type: string @@ -158,9 +158,7 @@ jobs: # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public # project. This is defined with a non-standard Platform of 'AnyCPU', and - # will fail the builds if left defined. The stress tests solution does - # not require any specific Platform, and so its solution file doesn't - # support any non-standard platforms. + # will fail the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. diff --git a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml index 663532b905..af471001ea 100644 --- a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml @@ -50,6 +50,7 @@ pr: - NuGet.config exclude: - eng/pipelines/onebranch/* + - eng/pipelines/stress/* # Do not trigger commit or schedule runs for this pipeline. trigger: none @@ -77,12 +78,6 @@ parameters: type: boolean default: false - # True to run the stress tests stage. - - name: enableStressTests - displayName: Enable Stress Tests - type: boolean - default: false - # The target frameworks to build and run tests for on Windows. # # These are _not_ the target frameworks to build the driver packages for. @@ -132,7 +127,6 @@ extends: buildPlatforms: ${{ parameters.buildPlatforms }} referenceType: Package debug: ${{ parameters.debug }} - enableStressTests: ${{ parameters.enableStressTests }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} diff --git a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml index 868b3010ff..98411b148a 100644 --- a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml @@ -50,6 +50,7 @@ pr: - NuGet.config exclude: - eng/pipelines/onebranch/* + - eng/pipelines/stress/* # Do not trigger commit or schedule runs for this pipeline. trigger: none @@ -77,12 +78,6 @@ parameters: type: boolean default: false - # True to run the stress tests stage. - - name: enableStressTests - displayName: Enable Stress Tests - type: boolean - default: false - # The target frameworks to build and run tests for on Windows. # # These are _not_ the target frameworks to build the driver packages for. @@ -132,7 +127,6 @@ extends: buildPlatforms: ${{ parameters.buildPlatforms }} referenceType: Project debug: ${{ parameters.debug }} - enableStressTests: ${{ parameters.enableStressTests }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} diff --git a/eng/pipelines/stages/stress-tests-ci-stage.yml b/eng/pipelines/stages/stress-tests-ci-stage.yml deleted file mode 100644 index 978105d580..0000000000 --- a/eng/pipelines/stages/stress-tests-ci-stage.yml +++ /dev/null @@ -1,205 +0,0 @@ -################################################################################ -# Licensed to the .NET Foundation under one or more agreements. The .NET -# Foundation licenses this file to you under the MIT license. See the LICENSE -# file in the project root for more information. -################################################################################ - -# This stage builds and runs stress tests against an MDS NuGet package available -# as a pipeline artifact. -# -# The stress tests are located here: -# -# src/Microsoft.Data.SqlClient/tests/StressTests -# -# All tests use a localhost SQL Server configured for SQL auth via the 'sa' user -# and the provided password. -# -# This stage depends on the secrets_stage. -# -# This template defines a stage named 'run_stress_tests_stage' that can be -# depended on by downstream stages. - -parameters: - - # The names of any additional stages this stage depends on, for example the stages that publish - # the MDS package artifacts we will test. - - name: additionalDependsOn - type: object - default: [] - - # The Azure package version to stress test. This version must be available in - # one of the configured NuGet sources. - - name: azurePackageVersion - displayName: Azure Package Version - type: string - - # The type of build to produce (Debug or Release) - - name: buildConfiguration - type: string - default: Debug - values: - - Debug - - Release - - # The verbosity level for the dotnet CLI commands. - - name: dotnetVerbosity - type: string - default: normal - values: - - quiet - - minimal - - normal - - detailed - - diagnostic - - # The name of the MDS pipeline artifacts to download. - - name: mdsArtifactsName - type: string - default: MDS.Artifacts - - # The MDS package version found in the pipeline artifacts we will download. - - name: mdsPackageVersion - type: string - - # The list of .NET Framework runtimes to test against. - - name: netFrameworkTestRuntimes - type: object - default: [net462] - - # The list of .NET runtimes to test against. - - name: netTestRuntimes - type: object - default: [net8.0, net9.0, net10.0] - -stages: - - stage: run_stress_tests_stage - displayName: Run Stress Tests - dependsOn: - - secrets_stage - - ${{ each dep in parameters.additionalDependsOn }}: - - ${{ dep }} - - variables: - # The directory where dotnet artifacts will be staged. Not to be - # confused with pipeline artifact. - - name: dotnetArtifactsDir - value: $(Build.StagingDirectory)/dotnetArtifacts - - # The solution file to use for all dotnet CLI commands. - - name: solution - value: src/Microsoft.Data.SqlClient/tests/StressTests/StressTests.slnx - - # The stress test project to run. - - name: testProject - value: src/Microsoft.Data.SqlClient/tests/StressTests/SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj - - # dotnet CLI arguments common to all commands. - - name: commonArguments - value: >- - --verbosity ${{parameters.dotnetVerbosity}} - --artifacts-path $(dotnetArtifactsDir) - -p:MdsPackageVersion=${{parameters.mdsPackageVersion}} - -p:AzurePackageVersion=${{parameters.azurePackageVersion}} - - # dotnet CLI arguments for build/run commands. - - name: buildArguments - value: >- - $(commonArguments) - --configuration ${{parameters.buildConfiguration}} - - # Bring the SA password from the secrets_stage into scope here. - - name: saPassword - value: $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] - - # The contents of the config file to use for all tests. We will write - # this to a JSON file for each test job, and then point to it via the - # STRESS_CONFIG_FILE environment variable. - - name: ConfigContent - value: | - [ - { - "name": "Azure SQL", - "type": "SqlServer", - "isDefault": true, - "dataSource": "localhost", - "user": "sa", - "password": "$(saPassword)", - "supportsWindowsAuthentication": false, - "isLocal": false, - "disableMultiSubnetFailover": true, - "disableNamedPipes": true, - "encrypt": false - } - ] - - jobs: - - # -------------------------------------------------------------------------- - # Build and test on Linux. - - - template: /eng/pipelines/jobs/stress-tests-ci-job.yml@self - parameters: - jobNameSuffix: linux - displayNamePrefix: Linux - poolName: $(ci_var_defaultPoolName) - vmImage: ADO-UB22-SQL22 - sqlSetupStep: - template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self - parameters: - saPassword: $(saPassword) - mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - solution: $(solution) - testProject: $(testProject) - restoreArguments: $(commonArguments) - buildArguments: $(buildArguments) - netTestRuntimes: ${{ parameters.netTestRuntimes }} - configContent: $(ConfigContent) - - # -------------------------------------------------------------------------- - # Build and test on Windows - - - template: /eng/pipelines/jobs/stress-tests-ci-job.yml@self - parameters: - jobNameSuffix: windows - displayNamePrefix: Win - poolName: $(ci_var_defaultPoolName) - # The Windows images include a suitable .NET Framework runtime, so we - # don't have to install one explicitly. - vmImage: ADO-MMS22-SQL22 - sqlSetupStep: - template: /eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml@self - parameters: - saPassword: $(saPassword) - mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - solution: $(solution) - testProject: $(testProject) - restoreArguments: $(commonArguments) - buildArguments: $(buildArguments) - netTestRuntimes: ${{ parameters.netTestRuntimes }} - # Note that we include the .NET Framework runtimes for test runs on - # Windows. - netFrameworkTestRuntimes: ${{ parameters.netFrameworkTestRuntimes }} - configContent: $(ConfigContent) - - # -------------------------------------------------------------------------- - # Build and test on macOS. - - - template: /eng/pipelines/jobs/stress-tests-ci-job.yml@self - parameters: - jobNameSuffix: macos - displayNamePrefix: macOS - # We don't have any 1ES Hosted Pool images for macOS, so we use a - # generic one from Azure Pipelines. - poolName: Azure Pipelines - vmImage: macos-latest - sqlSetupStep: - template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self - parameters: - saPassword: $(saPassword) - mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - solution: $(solution) - testProject: $(testProject) - restoreArguments: $(commonArguments) - buildArguments: $(buildArguments) - netTestRuntimes: ${{ parameters.netTestRuntimes }} - configContent: $(ConfigContent) diff --git a/eng/pipelines/stress-tests-pipeline.yml b/eng/pipelines/stress-tests-pipeline.yml deleted file mode 100644 index e08f255e5a..0000000000 --- a/eng/pipelines/stress-tests-pipeline.yml +++ /dev/null @@ -1,81 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# This pipeline runs the stress test suite using the most recent artifacts from -# the CI-SqlClient-Package pipeline. -# -# It runs after every successful run of the following pipelines: -# -# Public project: -# - PR-SqlClient-Package -# - CI-SqlClient-Package -# -# ADO.net project: -# - MDS Main CI-Package -# -# This pipeline definition is mapped to the Stress-SqlClient pipelines: -# -# Public project: -# -# TODO -# -# ADO.net project: -# -# TODO - -# Set the pipeline run name to the day-of-year and the daily run counter. -name: $(DayOfYear)$(Rev:rr) - -# Do not trigger this pipeline for PRs, commits, or schedules. -pr: none -trigger: none - -# Trigger this pipeline after successful runs of each of the following pipelines. -resources: - pipelines: - # The PR-SqlClient-Package pipeline in the Public project. - - pipeline: upstreamPipeline1 - project: Public - source: /ADO/PR-SqlClient-Package - trigger: true - # The CI-SqlClient-Package pipeline in the Public project. - - pipeline: upstreamPipeline2 - project: Public - source: /ADO/CI-SqlClient-Package - trigger: true - # The MDS Main CI-Package pipeline in the ADO.net project. - - pipeline: upstreamPipeline3 - project: ADO.net - source: /Internal CI/MDS Main CI-Package - trigger: true - -# Pipeline parameters, visible in the Azure DevOps UI. -parameters: - - # The build configuration to use; defaults to Release. - - name: buildConfiguration - displayName: Build Configuration - type: string - default: Release - values: - - Debug - - Release - - # True to emit debug information and steps. - - name: debug - displayName: Enable debug output - type: boolean - default: false - -stages: -- stage: Placeholder - displayName: Placeholder Stage - jobs: - - job: PlaceholderJob - displayName: Placeholder Job - steps: - - script: echo "This is a placeholder stage." - displayName: Placeholder Step diff --git a/eng/pipelines/stress/stress-tests-job.yml b/eng/pipelines/stress/stress-tests-job.yml new file mode 100644 index 0000000000..40cecf6e93 --- /dev/null +++ b/eng/pipelines/stress/stress-tests-job.yml @@ -0,0 +1,180 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This job builds and runs stress tests by compiling the SqlClient projects transitively. +# +# The stress tests are located here: +# +# src/Microsoft.Data.SqlClient/tests/StressTests +# +# This template defines a job named 'stress_tests_job_' that can be depended on by +# downstream jobs. + +parameters: + + # The type of build to produce (Debug or Release) + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # True to enable debugging steps. + - name: debug + type: boolean + default: false + + # The prefix to prepend to the job's display name: + # + # [] Run Stress Tests + # + - name: displayNamePrefix + type: string + + # The verbosity level for the dotnet CLI commands. + - name: dotnetVerbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + # The suffix to append to the job name. + - name: jobNameSuffix + type: string + + # The list of .NET Framework runtimes to test against. + - name: netFrameworkTestRuntimes + type: object + default: [] + + # The list of .NET runtimes to test against. + - name: netTestRuntimes + type: object + default: [] + + # The name of the Azure Pipelines pool to use. + - name: poolName + type: string + + # The local SQL Server instance's 'sa' password, for use in the config file. + - name: saPassword + type: string + + # The step to run to configure SQL Server. This should configure a local SQL Server instance with + # an 'sa' login using the same password provided by the saPassword parameter. + - name: sqlSetupStep + type: step + + # The pool VM image to use. + - name: vmImage + type: string + +jobs: + - job: stress_tests_job_${{ parameters.jobNameSuffix }} + displayName: '[${{ parameters.displayNamePrefix }}] Run Stress Tests' + pool: + name: ${{ parameters.poolName }} + ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: + vmImage: ${{ parameters.vmImage }} + ${{ else }}: + demands: + - imageOverride -equals ${{ parameters.vmImage }} + + variables: + + # The directory where dotnet artifacts will be staged. Not to be + # confused with pipeline artifacts. + - name: dotnetArtifactsDir + value: $(Build.StagingDirectory)/dotnetArtifacts + + # The top-level project file to build and run. + - name: project + value: $(Build.SourcesDirectory)/src/Microsoft.Data.SqlClient/tests/StressTests/SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj + + # dotnet CLI arguments. + - name: dotnetArguments + value: >- + --verbosity ${{ parameters.dotnetVerbosity }} + --artifacts-path $(dotnetArtifactsDir) + --configuration ${{ parameters.buildConfiguration }} + + # The contents of the config file to use for all tests. We will write this to a JSON file and + # then point to it via the STRESS_CONFIG_FILE environment variable. + - name: configContent + value: | + [ + { + "name": "Azure SQL", + "type": "SqlServer", + "isDefault": true, + "dataSource": "localhost", + "user": "sa", + "password": "${{ parameters.saPassword }}", + "supportsWindowsAuthentication": false, + "isLocal": false, + "disableMultiSubnetFailover": true, + "disableNamedPipes": true, + "encrypt": false + } + ] + + # Stress test command-line arguments. + - name: testArguments + value: -a SqlClient.Stress.Tests -console + + steps: + + # Install the .NET SDK and Runtimes. + - template: /eng/pipelines/steps/install-dotnet.yml@self + parameters: + runtimes: [8.x, 9.x] + + # Setup the local SQL Server. + - ${{ parameters.sqlSetupStep }} + + # Write the config file. + - task: PowerShell@2 + displayName: Write Config File + inputs: + pwsh: true + targetType: inline + script: | + # Capture the multi-line JSON content into a variable. + $content = @" + $(configContent) + "@ + + # Write the JSON content to the config file. + $content | Out-File -FilePath "config.json" + + # Build the project. + - task: DotNetCoreCLI@2 + displayName: Build Project + inputs: + command: build + projects: $(project) + arguments: $(dotnetArguments) + + # Run the stress tests for each .NET runtime. + - ${{ each runtime in parameters.netTestRuntimes }}: + - task: DotNetCoreCLI@2 + displayName: Test [${{ runtime }}] + inputs: + command: run + projects: $(project) + arguments: $(dotnetArguments) --no-build -f ${{ runtime }} -e STRESS_CONFIG_FILE=config.json -- $(testArguments) + + # Run the stress tests for each .NET Framework runtime. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - task: DotNetCoreCLI@2 + displayName: Test [${{ runtime }}] + inputs: + command: run + projects: $(project) + arguments: $(dotnetArguments) --no-build -f ${{ runtime }} -e STRESS_CONFIG_FILE=config.json -- $(testArguments) diff --git a/eng/pipelines/stress/stress-tests-pipeline.yml b/eng/pipelines/stress/stress-tests-pipeline.yml new file mode 100644 index 0000000000..cec68912fb --- /dev/null +++ b/eng/pipelines/stress/stress-tests-pipeline.yml @@ -0,0 +1,101 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This pipeline runs the stress test suite against the SqlClient projects, building them +# transitively as necessary, triggered by successful runs of the following pipelines: +# +# Public project: +# Triggering pipelines: PR-SqlClient-Project, CI-SqlClient (branch main only) +# Pipeline name: Stress-SqlClient +# Pipeline URL: TODO: Create the Azure DevOps pipeline. +# +# ADO.Net project: +# Triggering pipeline: MDS Main CI (branch internal/main only) +# Pipeline name: Stress-SqlClient +# Pipeline URL: TODO: Create the Azure DevOps pipeline. + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# Do not trigger this pipeline for PRs, commits, or schedules. +pr: none +trigger: none + +# Trigger this pipeline after successful runs of the desired pipelines. +# +# The pipeline identifiers are displayed in the Azure DevOps UI, so it is helpful if they indicate +# the project, folder, and pipeline name, hence the verbose values below. +# +resources: + pipelines: + + # The PR-SqlClient-Project pipeline in the Public project. + - pipeline: Public-ADO-PR-SqlClient-Project + project: Public + source: /ADO/PR-SqlClient-Project + trigger: true + + # The CI-SqlClient pipeline in the Public project. + - pipeline: Public-ADO-CI-SqlClient + project: Public + source: /ADO/CI-SqlClient + trigger: + branches: + include: + - main + + # The MDS Main CI pipeline in the ADO.Net project. + - pipeline: ADO-Net-Internal-CI-MDS-Main-CI + project: ADO.Net + source: /Internal CI/MDS Main CI + trigger: + branches: + include: + - internal/main + +# Pipeline parameters, visible in the Azure DevOps UI. +parameters: + + # The build configuration to use; defaults to Release. + - name: buildConfiguration + displayName: Build Configuration + type: string + default: Release + values: + - Debug + - Release + + # True to emit debug information and steps. + - name: debug + displayName: Enable debug output + type: boolean + default: false + + # Dotnet CLI verbosity level. + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + +# The stages to run. +stages: + + # Generate secrets. We use these for the local SQL Server 'sa' logins. + - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self + parameters: + debug: ${{ parameters.debug }} + + # Run the stress tests. + - template: /eng/pipelines/stress/stress-tests-stage.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/stress/stress-tests-stage.yml b/eng/pipelines/stress/stress-tests-stage.yml new file mode 100644 index 0000000000..0f0000f99a --- /dev/null +++ b/eng/pipelines/stress/stress-tests-stage.yml @@ -0,0 +1,130 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This stage builds and runs stress tests against the SqlClient projects, building them transitively +# as necessary. +# +# The stress tests are located here: +# +# src/Microsoft.Data.SqlClient/tests/StressTests +# +# All tests use a localhost SQL Server configured for SQL auth via the 'sa' user and the generated +# password. +# +# This stage depends on the secrets_stage. +# +# This template defines a stage named 'stress_tests_stage' that can be depended on by downstream +# stages. + +parameters: + + # The type of build to produce (Debug or Release) + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # True to enable debugging steps. + - name: debug + type: boolean + default: false + + # The verbosity level for the dotnet CLI commands. + - name: dotnetVerbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + # The list of .NET Framework runtimes to test against. + - name: netFrameworkTestRuntimes + type: object + default: [net462] + + # The list of .NET runtimes to test against. + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + +stages: + - stage: stress_tests_stage + displayName: Run Stress Tests + dependsOn: + - secrets_stage + + variables: + # Bring the SA password from the secrets_stage into scope here. + - name: saPassword + value: $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] + + jobs: + + # ---------------------------------------------------------------------------------------------- + # Build and test on Linux. + + - template: /eng/pipelines/stress/stress-tests-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayNamePrefix: Linux + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: linux + netTestRuntimes: ${{ parameters.netTestRuntimes }} + poolName: ADO-CI-1ES-Pool + saPassword: $(saPassword) + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self + parameters: + saPassword: $(saPassword) + vmImage: ADO-UB22-SQL22 + + # ---------------------------------------------------------------------------------------------- + # Build and test on Windows + + - template: /eng/pipelines/stress/stress-tests-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayNamePrefix: Win + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows + # Note that we include the .NET Framework runtimes for test runs on Windows. + netFrameworkTestRuntimes: ${{ parameters.netFrameworkTestRuntimes }} + netTestRuntimes: ${{ parameters.netTestRuntimes }} + poolName: ADO-CI-1ES-Pool + saPassword: $(saPassword) + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml@self + parameters: + saPassword: $(saPassword) + # The Windows images include a suitable .NET Framework runtime, so we don't have to install + # one explicitly. + vmImage: ADO-MMS22-SQL22 + + # ---------------------------------------------------------------------------------------------- + # Build and test on macOS. + + - template: /eng/pipelines/stress/stress-tests-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayNamePrefix: macOS + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: macos + netTestRuntimes: ${{ parameters.netTestRuntimes }} + # We don't have any 1ES Hosted Pool images for macOS, so we use a generic one from Azure + # Pipelines. + poolName: Azure Pipelines + saPassword: $(saPassword) + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self + parameters: + saPassword: $(saPassword) + vmImage: macos-latest diff --git a/src/Microsoft.Data.SqlClient/tests/StressTests/Readme.md b/src/Microsoft.Data.SqlClient/tests/StressTests/Readme.md index f0723c3189..546428dbe9 100644 --- a/src/Microsoft.Data.SqlClient/tests/StressTests/Readme.md +++ b/src/Microsoft.Data.SqlClient/tests/StressTests/Readme.md @@ -1,26 +1,21 @@ # Microsoft.Data.SqlClient Stress Test -This Stress testing application for `Microsoft.Data.SqlClient` is under progress. +This Stress testing application for the `Microsoft.Data.SqlClient` suite is a work in progress. -This project intends to help finding a certain level of effectiveness under -unfavorable conditions, and verifying the mode of failures. +This project intends to help finding a certain level of effectiveness under unfavorable conditions, +and verifying the mode of failures. -This is a console application targeting all frameworks supported by MDS, -currently: +This is a console application targeting all frameworks supported by MDS, currently: -- .NET 8.0 +- .NET 10.0 - .NET 9.0 -- .NET Framework 4.6.2 -- .NET Framework 4.7 -- .NET Framework 4.7.1 -- .NET Framework 4.7.2 -- .NET Framework 4.8 -- .NET Framework 4.8.1 +- .NET 8.0 +- .NET Framework 4.6.2 (Windows only; higher .NET Framework versions are supported at runtime via + compatibility) ## Purpose of application for developers -Define fuzz tests for all new features/APIs in the driver and to be run before -every GA release. +Define fuzz tests for all new features/APIs in the driver and to be run before every GA release. ## Pre-Requisites @@ -40,14 +35,13 @@ Required in the config file: |`disableNamedPipes`|`true`, `false`|`true` means the connections will create just using tcp protocol.| |`encrypt`|`true`, `false`|Assigns the encrypt property of the connection strings.| -Note: The database user must have permission to create and drop databases. -Each execution of the stress tests will create a database with a name like: +Note: The database user must have permission to create and drop databases. Each execution of the +stress tests will create a database with a name like: - `StressTests-` -The database will be dropped as a best effort once testing is complete. This -allows for multiple test runs to execute in parallel against the same database -server without colliding. +The database will be dropped as a best effort once testing is complete. This allows for multiple +test runs to execute in parallel against the same database server without colliding. ## Adding new Tests @@ -55,83 +49,55 @@ server without colliding. ## Building the application -To build the application using the `StressTests.slnx` solution: - -```bash -dotnet build [-c|--configuration ] -``` - -```bash -# Builds the application for the Client Os in `Debug` Configuration for `AnyCpu` -# platform. -# -# All supported target frameworks are built by default. - -$ dotnet build -``` - -```bash -# Build the application for .Net framework 4.8.1 with `Debug` configuration. - -$ dotnet build -f net481 -``` +Build the application using the top-level project `SqlClient.Stress.Runner`: ```bash -# Build the application for .Net 9.0 with `Release` configuration. - -$ dotnet build -f net9.0 -c Release -``` - -```bash -# Cleans all build directories - -$ dotnet clean +$ cd .../src/Microsoft.Data.SqlClient/tests/StressTests +dotnet build SqlClient.Stress.Runner [-c ] ``` ## Running tests -After building the application, find the built folder with target framework and -run the `stresstest.exe` file with required arguments. +After building the application, find the built folder with target framework and run the +`stresstest.exe` file with required arguments. -Find the result in a log file inside the `logs` folder besides the command -prompt. +Find the result in a log file inside the `logs` folder besides the command prompt. -You may specify the config file by supplying an environment variable that -points to the file: +You may specify the config file by supplying an environment variable that points to the file: - `STRESS_CONFIG_FILE=/path/to/my/config.jsonc` ## Command prompt -You must run the stress tests from the root of the Stress Tests project -directory (i.e. the same directory this readme file is in). +You must run the stress tests from the root of the Stress Tests project directory (i.e. the same +directory this readme file is in). ```bash # Linux -$ cd /home/paul/dev/SqlClient/src/Microsoft.Data.SqlClient/tests/StressTests +$ cd .../src/Microsoft.Data.SqlClient/tests/StressTests # Via dotnet run CLI: -$ dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj -- -a SqlClient.Stress.Tests +$ dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner -- -a SqlClient.Stress.Tests # Via dotnet CLI: $ dotnet SqlClient.Stress.Runner/bin/Debug/net9.0/stresstest.dll -a SqlClient.Stress.Tests # With a specific config file and all output to console: -$ dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj -e STRESS_CONFIG_FILE=/path/to/config.jsonc -- -a SqlClient.Stress.Tests -console +$ dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner -e STRESS_CONFIG_FILE=/path/to/config.jsonc -- -a SqlClient.Stress.Tests -console ``` ```powershell # Windows -> cd \dev\SqlClient\src\Microsoft.Data.SqlClient\tests\StressTests +> cd ...\src\Microsoft.Data.SqlClient\tests\StressTests # Via dotnet run CLI: -> dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner\SqlClient.Stress.Runner.csproj -- -a SqlClient.Stress.Tests +> dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner -- -a SqlClient.Stress.Tests # Via executable: > .\SqlClient.Stress.Runner\bin\Debug\net481\stresstest.exe -a SqlClient.Stress.Tests # With a specific config file and all output to console: -> dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner\SqlClient.Stress.Runner.csproj -e STRESS_CONFIG_FILE=c:\path\to\config.jsonc -- -a SqlClient.Stress.Tests -console +> dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner -e STRESS_CONFIG_FILE=c:\path\to\config.jsonc -- -a SqlClient.Stress.Tests -console ``` ## Supported arguments @@ -164,14 +130,14 @@ $ dotnet run --no-build -f net9.0 --project SqlClient.Stress.Runner/SqlClient.St # Run the application for a built target framework and all discovered tests # without debugger attached and shows the test methods' names. -> .\stresstest.exe -a SqlClient.Stress.Tests -all -printMethodName +> .\stresstest.exe -a SqlClient.Stress.Tests -all -printMethodName ``` ```powershell # Run the application for a built target framework and all discovered tests and # will wait for debugger to be attached. -> .\stresstest.exe -a SqlClient.Stress.Tests -all -debug +> .\stresstest.exe -a SqlClient.Stress.Tests -all -debug ``` ```powershell diff --git a/src/Microsoft.Data.SqlClient/tests/StressTests/StressTests.slnx b/src/Microsoft.Data.SqlClient/tests/StressTests/StressTests.slnx deleted file mode 100644 index a0dfd5e503..0000000000 --- a/src/Microsoft.Data.SqlClient/tests/StressTests/StressTests.slnx +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - -