From 6b3ca6ab79dd0072c775ad462ab4605577abfd36 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 30 Mar 2026 17:21:56 +0000 Subject: [PATCH 01/29] simplify the location of auth-dev-feed so that it works for both playback and live tests. --- eng/pipelines/templates/jobs/ci.tests.yml | 4 ---- eng/pipelines/templates/steps/auth-dev-feed.yml | 1 - eng/pipelines/templates/steps/build-test.yml | 4 ++++ 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 7793d636ba66..48d012fe01e9 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -96,10 +96,6 @@ jobs: Paths: - '**' - # Authenticate to Azure Artifacts feed immediately after checkout to prevent 401 errors - # Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index 00556968468f..61ead7d7a9a7 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -38,4 +38,3 @@ steps: Write-Host "UV authentication is currently disabled. To enable, uncomment the variable assignment above." } displayName: 'Configure UV Authentication (Disabled)' - condition: false # Explicitly disabled - change to 'true' or remove to enable diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 444830268f2c..ec2ad289fb6c 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -28,6 +28,10 @@ steps: - template: /eng/pipelines/templates/steps/use-venv.yml + # Authenticate to Azure Artifacts feed immediately after checkout to prevent 401 errors + # Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml parameters: PackageName: "azure-template" From f58b1daf6673fa1e8042f21e8fec5badff0dedfe Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 30 Mar 2026 17:50:29 +0000 Subject: [PATCH 02/29] make build test echo out the feed being utilized --- eng/pipelines/templates/steps/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index ec2ad289fb6c..d2ec2068d86d 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -44,7 +44,7 @@ steps: Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - $(PIP_EXE) install -r eng/ci_tools.txt + $(PIP_EXE) install -r eng/ci_tools.txt -v $(PIP_EXE) freeze displayName: 'Prep Environment' From cf28c2f0f5f69fe6b35dff0ce87d5afc97573f3d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 31 Mar 2026 16:27:17 +0000 Subject: [PATCH 03/29] didn't actually uncomment the UV index setting --- eng/pipelines/templates/steps/auth-dev-feed.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index 61ead7d7a9a7..f539ea2c7451 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -33,8 +33,6 @@ steps: # UV will read credentials from ~/.pypirc created by PipAuthenticate task if ($env:PIP_INDEX_URL) { Write-Host "PIP Index URL detected: $env:PIP_INDEX_URL" - # Uncomment the next line to enable UV authentication - # Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)" - Write-Host "UV authentication is currently disabled. To enable, uncomment the variable assignment above." + Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)" } - displayName: 'Configure UV Authentication (Disabled)' + displayName: 'Configure UV Authentication' From adbbb3cab18ede4fd784652d8f385813cc4635ea Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 31 Mar 2026 09:29:08 -0700 Subject: [PATCH 04/29] Update eng/pipelines/templates/steps/build-test.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- eng/pipelines/templates/steps/build-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index d2ec2068d86d..4054c6d2478a 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -31,6 +31,8 @@ steps: # Authenticate to Azure Artifacts feed immediately after checkout to prevent 401 errors # Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml parameters: From 77e9ee9b9a1a132f21a1ff3feea5201f491b79f0 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 31 Mar 2026 16:31:47 +0000 Subject: [PATCH 05/29] update step name, add a requirement that is NOT YET in our dev feed to ensure that no 404s occur using uv --- sdk/core/azure-core/dev_requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 3ca5b687d635..64b69cfe8252 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -12,4 +12,5 @@ azure-data-tables opentelemetry-sdk~=1.26 opentelemetry-instrumentation-requests>=0.50b0 ../../identity/azure-identity -packaging # for version parsing in test_basic_transport_async.py \ No newline at end of file +packaging # for version parsing in test_basic_transport_async.py +ruff \ No newline at end of file From 0e824b6202ec767b18671e2d10bc5f365ec4ac9d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 31 Mar 2026 17:27:10 +0000 Subject: [PATCH 06/29] more manipulation --- .../templates/steps/auth-dev-feed.yml | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index f539ea2c7451..a5970b34e962 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -25,14 +25,34 @@ steps: artifactFeeds: $(DevFeedName) onlyAddExtraIndex: false - # Disabled UV authentication step - for future enablement when UV fully supports Azure Artifacts - # To enable: uncomment this step and ensure UV_INDEX_URL is properly configured - # The PipAuthenticate task above creates a .pypirc file in the home directory that UV can use + # UV doesn't read .pypirc for credentials like pip does. + # Convert the .pypirc credentials into a .netrc file that UV reads natively. - pwsh: | - # This step configures UV to use the same authentication as pip - # UV will read credentials from ~/.pypirc created by PipAuthenticate task + $pypircPath = Join-Path $HOME ".pypirc" + if (-not (Test-Path $pypircPath)) { + Write-Host "No .pypirc found, skipping UV auth configuration" + return + } + + $content = Get-Content $pypircPath -Raw + # Extract username and password from the pypirc + $username = $null + $password = $null + if ($content -match '(?m)^\s*username\s*=\s*(.+?)\s*$') { $username = $Matches[1] } + if ($content -match '(?m)^\s*password\s*=\s*(.+?)\s*$') { $password = $Matches[1] } + + if ($username -and $password) { + $netrcPath = Join-Path $HOME ".netrc" + $entry = "machine pkgs.dev.azure.com`nlogin $username`npassword $password" + Add-Content -Path $netrcPath -Value $entry + Write-Host "Wrote .netrc entry for pkgs.dev.azure.com (UV auth)" + } else { + Write-Host "##vso[task.logissue type=warning]Could not extract credentials from .pypirc for UV" + } + + # Also forward PIP_INDEX_URL so UV knows which index to use if ($env:PIP_INDEX_URL) { - Write-Host "PIP Index URL detected: $env:PIP_INDEX_URL" Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)" + Write-Host "UV_INDEX_URL set to: $($env:PIP_INDEX_URL)" } displayName: 'Configure UV Authentication' From e39fe81092b9ad23b206e1e64c1cc16102b3dae0 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 31 Mar 2026 19:17:13 +0000 Subject: [PATCH 07/29] UV won't pull stuff automatically into the feed? need to add pip extra index url for it. --- .../templates/steps/auth-dev-feed.yml | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index a5970b34e962..c5272b9e1026 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -25,34 +25,17 @@ steps: artifactFeeds: $(DevFeedName) onlyAddExtraIndex: false - # UV doesn't read .pypirc for credentials like pip does. - # Convert the .pypirc credentials into a .netrc file that UV reads natively. + # PipAuthenticate embeds credentials directly into PIP_INDEX_URL as https://build:@host/... + # UV doesn't read PIP_INDEX_URL or PIP_EXTRA_INDEX_URL, so we mirror the pip two-index setup: + # UV_INDEX_URL = dev feed (primary, for azure-* dev builds) + # UV_EXTRA_INDEX_URL = PyPI (fallback, for everything else) + # Without the PyPI fallback, packages absent from the dev feed get a 401/404 with no fallback. - pwsh: | - $pypircPath = Join-Path $HOME ".pypirc" - if (-not (Test-Path $pypircPath)) { - Write-Host "No .pypirc found, skipping UV auth configuration" - return - } - - $content = Get-Content $pypircPath -Raw - # Extract username and password from the pypirc - $username = $null - $password = $null - if ($content -match '(?m)^\s*username\s*=\s*(.+?)\s*$') { $username = $Matches[1] } - if ($content -match '(?m)^\s*password\s*=\s*(.+?)\s*$') { $password = $Matches[1] } - - if ($username -and $password) { - $netrcPath = Join-Path $HOME ".netrc" - $entry = "machine pkgs.dev.azure.com`nlogin $username`npassword $password" - Add-Content -Path $netrcPath -Value $entry - Write-Host "Wrote .netrc entry for pkgs.dev.azure.com (UV auth)" - } else { - Write-Host "##vso[task.logissue type=warning]Could not extract credentials from .pypirc for UV" - } - - # Also forward PIP_INDEX_URL so UV knows which index to use if ($env:PIP_INDEX_URL) { Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)" - Write-Host "UV_INDEX_URL set to: $($env:PIP_INDEX_URL)" + Write-Host "##vso[task.setvariable variable=UV_EXTRA_INDEX_URL]https://pypi.org/simple" + Write-Host "UV_INDEX_URL configured from PIP_INDEX_URL with PyPI fallback" + } else { + Write-Host "##vso[task.logissue type=warning]PIP_INDEX_URL not set - UV will use default index" } displayName: 'Configure UV Authentication' From d0c62723af253b69c774f351358c4f4881d949b9 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 31 Mar 2026 23:58:22 +0000 Subject: [PATCH 08/29] revert attempt so far. add explicit check for installing ci_tools.txt --- eng/pipelines/templates/steps/auth-dev-feed.yml | 14 ++++---------- .../templates/steps/build-package-artifacts.yml | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index c5272b9e1026..3fcde893e33b 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -25,17 +25,11 @@ steps: artifactFeeds: $(DevFeedName) onlyAddExtraIndex: false - # PipAuthenticate embeds credentials directly into PIP_INDEX_URL as https://build:@host/... - # UV doesn't read PIP_INDEX_URL or PIP_EXTRA_INDEX_URL, so we mirror the pip two-index setup: - # UV_INDEX_URL = dev feed (primary, for azure-* dev builds) - # UV_EXTRA_INDEX_URL = PyPI (fallback, for everything else) - # Without the PyPI fallback, packages absent from the dev feed get a 401/404 with no fallback. - pwsh: | + Write-Host "$(PIP_INDEX_URL)" if ($env:PIP_INDEX_URL) { + Write-Host "PIP Index URL detected: $env:PIP_INDEX_URL" Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)" - Write-Host "##vso[task.setvariable variable=UV_EXTRA_INDEX_URL]https://pypi.org/simple" - Write-Host "UV_INDEX_URL configured from PIP_INDEX_URL with PyPI fallback" - } else { - Write-Host "##vso[task.logissue type=warning]PIP_INDEX_URL not set - UV will use default index" } - displayName: 'Configure UV Authentication' + displayName: 'Configure UV Authentication (Disabled)' + condition: false # Explicitly disabled - change to 'true' or remove to enable diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 3f4c6e196ccf..0c6948c8bcd9 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -99,7 +99,7 @@ steps: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true which python - $(PIP_EXE) install -r eng/ci_tools.txt + $(PIP_EXE) install -r eng/ci_tools.txt -vv if ($env:AGENT_OS -eq "Linux") { Write-Host "Installing release reqs" From 017ef3cb847b2464c77f139b01d0236d830b04e9 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 1 Apr 2026 00:02:39 +0000 Subject: [PATCH 09/29] accidentally disabled UV --- eng/pipelines/templates/steps/auth-dev-feed.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index 3fcde893e33b..af7fc8d13524 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -31,5 +31,4 @@ steps: Write-Host "PIP Index URL detected: $env:PIP_INDEX_URL" Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)" } - displayName: 'Configure UV Authentication (Disabled)' - condition: false # Explicitly disabled - change to 'true' or remove to enable + displayName: 'Configure UV Authentication' From ce01280a7d5994692723d04dc542e9ff75a61686 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 3 Apr 2026 23:14:04 +0000 Subject: [PATCH 10/29] test a pip install --- eng/pipelines/templates/steps/build-package-artifacts.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 0c6948c8bcd9..d75aa5ceb79e 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -79,6 +79,11 @@ steps: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} + - pwsh: | + python -m pip --version + python -m pip install ruff==0.12.7 -v + displayName: "Try installing using pip directly" + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} From 2f48306c143d0bde81b25d638c1e68dcd045355a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 3 Apr 2026 23:42:02 +0000 Subject: [PATCH 11/29] ensure that new packages can be pulled in --- eng/pipelines/templates/steps/auth-dev-feed.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index af7fc8d13524..c45b15568c61 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -26,9 +26,13 @@ steps: onlyAddExtraIndex: false - pwsh: | - Write-Host "$(PIP_INDEX_URL)" if ($env:PIP_INDEX_URL) { - Write-Host "PIP Index URL detected: $env:PIP_INDEX_URL" - Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)" + # uv resolves credentials by index URL (astral-sh/uv#12651). The authenticated + # URL from PipAuthenticate is passed directly as UV_INDEX_URL. + Write-Host "##vso[task.setvariable variable=UV_INDEX_URL;issecret=true]$($env:PIP_INDEX_URL)" + # Disable keyring so uv uses the URL-embedded credentials directly. + Write-Host "##vso[task.setvariable variable=UV_KEYRING_PROVIDER]disabled" + } else { + Write-Host "##[warning]PIP_INDEX_URL not set - uv will fall back to public PyPI." } displayName: 'Configure UV Authentication' From 333736d5f43a6b52c91e150548821b04e5347ed8 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 3 Apr 2026 23:58:34 +0000 Subject: [PATCH 12/29] install pip and call --- eng/pipelines/templates/steps/build-package-artifacts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index d75aa5ceb79e..b2adb995e519 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -80,9 +80,11 @@ steps: ServiceDirectory: ${{ parameters.ServiceDirectory }} - pwsh: | + uv pip install pip python -m pip --version python -m pip install ruff==0.12.7 -v displayName: "Try installing using pip directly" + continueOnError: true - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: From 11f0d584f99e73984ee37e58d1500065e2ebe74e Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sat, 4 Apr 2026 00:37:04 +0000 Subject: [PATCH 13/29] log properly --- eng/pipelines/templates/steps/build-package-artifacts.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index b2adb995e519..6c7c2379a17a 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -80,9 +80,7 @@ steps: ServiceDirectory: ${{ parameters.ServiceDirectory }} - pwsh: | - uv pip install pip - python -m pip --version - python -m pip install ruff==0.12.7 -v + uv pip install ruff==0.12.6 -vv displayName: "Try installing using pip directly" continueOnError: true From c892017f1e2ae71463efff6cfab93c854bcd15a8 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sat, 4 Apr 2026 01:12:22 +0000 Subject: [PATCH 14/29] UV_INDEX_URL -> UV_DEFAULT_INDEX --- eng/pipelines/templates/steps/auth-dev-feed.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index c45b15568c61..9369c7c6b543 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -27,9 +27,10 @@ steps: - pwsh: | if ($env:PIP_INDEX_URL) { - # uv resolves credentials by index URL (astral-sh/uv#12651). The authenticated - # URL from PipAuthenticate is passed directly as UV_INDEX_URL. - Write-Host "##vso[task.setvariable variable=UV_INDEX_URL;issecret=true]$($env:PIP_INDEX_URL)" + # UV_DEFAULT_INDEX is the canonical replacement for the deprecated UV_INDEX_URL (uv 0.4.23+). + # PIP_INDEX_URL is set by PipAuthenticate@1 and contains embedded credentials, which uv + # will use for Basic auth against the ADO feed (and its PyPI upstream) per astral-sh/uv#12651. + Write-Host "##vso[task.setvariable variable=UV_DEFAULT_INDEX]$($env:PIP_INDEX_URL)" # Disable keyring so uv uses the URL-embedded credentials directly. Write-Host "##vso[task.setvariable variable=UV_KEYRING_PROVIDER]disabled" } else { From 90d6c479c3930b0198eb85063752aea3c1a4e908 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 6 Apr 2026 20:40:52 +0000 Subject: [PATCH 15/29] use environment variables only. name the index --- eng/pipelines/templates/steps/auth-dev-feed.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index 9369c7c6b543..2648b9ec05be 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -30,9 +30,11 @@ steps: # UV_DEFAULT_INDEX is the canonical replacement for the deprecated UV_INDEX_URL (uv 0.4.23+). # PIP_INDEX_URL is set by PipAuthenticate@1 and contains embedded credentials, which uv # will use for Basic auth against the ADO feed (and its PyPI upstream) per astral-sh/uv#12651. - Write-Host "##vso[task.setvariable variable=UV_DEFAULT_INDEX]$($env:PIP_INDEX_URL)" + Write-Host "##vso[task.setvariable variable=UV_DEFAULT_INDEX]azure-sdk=$($env:PIP_INDEX_URL)" # Disable keyring so uv uses the URL-embedded credentials directly. Write-Host "##vso[task.setvariable variable=UV_KEYRING_PROVIDER]disabled" + Write-Host "##vso[task.setvariable variable=UV_INDEX_AZURE_SDK_USERNAME]x" + Write-Host "##vso[task.setvariable variable=UV_INDEX_AZURE_SDK_PASSWORD]$(System.AccessToken)" } else { Write-Host "##[warning]PIP_INDEX_URL not set - uv will fall back to public PyPI." } From 8d60b83c2825702a84b76100767e58b074039ca1 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 6 Apr 2026 21:37:23 +0000 Subject: [PATCH 16/29] try again with the unauthed string + UV_INDX value --- eng/pipelines/templates/steps/auth-dev-feed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/auth-dev-feed.yml b/eng/pipelines/templates/steps/auth-dev-feed.yml index 2648b9ec05be..c3d74eef0a4f 100644 --- a/eng/pipelines/templates/steps/auth-dev-feed.yml +++ b/eng/pipelines/templates/steps/auth-dev-feed.yml @@ -30,7 +30,7 @@ steps: # UV_DEFAULT_INDEX is the canonical replacement for the deprecated UV_INDEX_URL (uv 0.4.23+). # PIP_INDEX_URL is set by PipAuthenticate@1 and contains embedded credentials, which uv # will use for Basic auth against the ADO feed (and its PyPI upstream) per astral-sh/uv#12651. - Write-Host "##vso[task.setvariable variable=UV_DEFAULT_INDEX]azure-sdk=$($env:PIP_INDEX_URL)" + Write-Host "##vso[task.setvariable variable=UV_DEFAULT_INDEX]azure-sdk=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" # Disable keyring so uv uses the URL-embedded credentials directly. Write-Host "##vso[task.setvariable variable=UV_KEYRING_PROVIDER]disabled" Write-Host "##vso[task.setvariable variable=UV_INDEX_AZURE_SDK_USERNAME]x" From 3bd9af391d140ab2c9e8aed4fbcdeee51c35f394 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 6 Apr 2026 21:42:20 +0000 Subject: [PATCH 17/29] target a package that ISNT in the feed, is UV able to retrieve it properly? are we accidentally clearing out variables in the test invocations? --- eng/pipelines/templates/steps/build-package-artifacts.yml | 2 +- .../azure-sdk-tools/ci_tools/scenario/dependency_resolution.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 6c7c2379a17a..c8cbb36d8bf6 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -80,7 +80,7 @@ steps: ServiceDirectory: ${{ parameters.ServiceDirectory }} - pwsh: | - uv pip install ruff==0.12.6 -vv + uv pip install ruff==0.12.5 -vv displayName: "Try installing using pip directly" continueOnError: true diff --git a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py index 7e357aee1b41..741e24ca6709 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py +++ b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py @@ -337,5 +337,7 @@ def install_packages(packages: List[str], req_file: str, python_executable: str, if req_file: commands.extend(["-r", req_file]) + import pprint + logger.info("Environment variables:\n%s", pprint.pformat(dict(os.environ))) logger.info("Installing packages. Command: %s", commands) subprocess.check_call(commands, cwd=cwd) From f1b50078b39b23be08c28af0620d265dee0ee4ca Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 6 Apr 2026 22:37:13 +0000 Subject: [PATCH 18/29] -vv install! WHAT IS HAPPENING --- eng/pipelines/templates/steps/build-package-artifacts.yml | 2 +- .../azure-sdk-tools/ci_tools/scenario/dependency_resolution.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index c8cbb36d8bf6..103ab1088a64 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -80,7 +80,7 @@ steps: ServiceDirectory: ${{ parameters.ServiceDirectory }} - pwsh: | - uv pip install ruff==0.12.5 -vv + uv pip install ruff==0.12.4 -vv displayName: "Try installing using pip directly" continueOnError: true diff --git a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py index 741e24ca6709..3cfee41d46e9 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py +++ b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py @@ -329,7 +329,7 @@ def install_packages(packages: List[str], req_file: str, python_executable: str, commands.append("install") if commands[0] == "uv": - commands.extend(["--python", python_exe]) + commands.extend(["--python", python_exe, "-vv"]) if packages: commands.extend(packages) From 3ffc9e4de2a390f4209eec4b2dd53c29850837dc Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 7 Apr 2026 01:05:56 +0000 Subject: [PATCH 19/29] remove some extra changes that were added for debugging --- eng/pipelines/templates/steps/build-package-artifacts.yml | 7 +------ .../ci_tools/scenario/dependency_resolution.py | 4 +--- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 103ab1088a64..3f4c6e196ccf 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -79,11 +79,6 @@ steps: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - - pwsh: | - uv pip install ruff==0.12.4 -vv - displayName: "Try installing using pip directly" - continueOnError: true - - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -104,7 +99,7 @@ steps: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true which python - $(PIP_EXE) install -r eng/ci_tools.txt -vv + $(PIP_EXE) install -r eng/ci_tools.txt if ($env:AGENT_OS -eq "Linux") { Write-Host "Installing release reqs" diff --git a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py index 3cfee41d46e9..7e357aee1b41 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py +++ b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py @@ -329,7 +329,7 @@ def install_packages(packages: List[str], req_file: str, python_executable: str, commands.append("install") if commands[0] == "uv": - commands.extend(["--python", python_exe, "-vv"]) + commands.extend(["--python", python_exe]) if packages: commands.extend(packages) @@ -337,7 +337,5 @@ def install_packages(packages: List[str], req_file: str, python_executable: str, if req_file: commands.extend(["-r", req_file]) - import pprint - logger.info("Environment variables:\n%s", pprint.pformat(dict(os.environ))) logger.info("Installing packages. Command: %s", commands) subprocess.check_call(commands, cwd=cwd) From 5504fdeeb4de3e8f3960af24dd24fa9fa4368735 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 7 Apr 2026 01:29:35 +0000 Subject: [PATCH 20/29] verbose output so we can see why this thing is blowing up --- eng/scripts/Language-Settings.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 3ee8871cf958..3b57c0b75b99 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -166,7 +166,8 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) # Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip if (Get-Command uv -ErrorAction SilentlyContinue) { Write-Host "Using uv pip install" - $null = uv pip install "$pathToBuild" + $installerOutput = uv pip install "$pathToBuild" -vv + Write-Host $installerOutput $freezeOutput = uv pip freeze Write-Host "Pip freeze output: $freezeOutput" } else { From 84505b1f241e4287303fb889f32ec93781a3f31f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 7 Apr 2026 01:40:35 +0000 Subject: [PATCH 21/29] pin older version of cibuildwheel during save package properties --- eng/scripts/Language-Settings.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 3b57c0b75b99..f51a8c997ea1 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -166,13 +166,13 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) # Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip if (Get-Command uv -ErrorAction SilentlyContinue) { Write-Host "Using uv pip install" - $installerOutput = uv pip install "$pathToBuild" -vv + $installerOutput = uv pip install "$pathToBuild" cibuildwheel==2.23.3 -vv Write-Host $installerOutput $freezeOutput = uv pip freeze Write-Host "Pip freeze output: $freezeOutput" } else { Write-Host "Using python -m pip install" - $null = python -m pip install "$pathToBuild" -q -I + $null = python -m pip install "$pathToBuild" cibuildwheel==2.23.3 -q -I } $scriptLoc = Join-path $RepoRoot eng scripts get_package_properties.py From e6e4ed9b010243c8e4b24ae0ed1d8841aba5671f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 7 Apr 2026 02:02:54 +0000 Subject: [PATCH 22/29] cibuildwheel needs to be pinned as it's installing a new version of patchelf and just blowing up on all plats --- eng/scripts/Language-Settings.ps1 | 4 ++-- eng/tools/azure-sdk-tools/pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index f51a8c997ea1..3b57c0b75b99 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -166,13 +166,13 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) # Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip if (Get-Command uv -ErrorAction SilentlyContinue) { Write-Host "Using uv pip install" - $installerOutput = uv pip install "$pathToBuild" cibuildwheel==2.23.3 -vv + $installerOutput = uv pip install "$pathToBuild" -vv Write-Host $installerOutput $freezeOutput = uv pip freeze Write-Host "Pip freeze output: $freezeOutput" } else { Write-Host "Using python -m pip install" - $null = python -m pip install "$pathToBuild" cibuildwheel==2.23.3 -q -I + $null = python -m pip install "$pathToBuild" -q -I } $scriptLoc = Join-path $RepoRoot eng scripts get_package_properties.py diff --git a/eng/tools/azure-sdk-tools/pyproject.toml b/eng/tools/azure-sdk-tools/pyproject.toml index 0d97d4835e3e..acb9eabea8d0 100644 --- a/eng/tools/azure-sdk-tools/pyproject.toml +++ b/eng/tools/azure-sdk-tools/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "setuptools", "pyparsing", "certifi", - "cibuildwheel", + "cibuildwheel==2.23.3", "pkginfo", "build", "packaging", From eecf6ab38e34644f73e67790ac9565b0bc00be12 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 7 Apr 2026 20:54:01 +0000 Subject: [PATCH 23/29] on newer macs, we're seeing the port resolve to an IPV6 address by default, and the proxy is not binding to an IPV6 address. As a result, on some mac runs, we are seeing failures to connect. the proxy is STARTED, but we can't see it. at least, so says the suspicions of copilot. let's see --- sdk/core/azure-core-experimental/tests/conftest.py | 2 +- sdk/core/azure-core/tests/async_tests/conftest.py | 2 +- sdk/core/azure-core/tests/conftest.py | 2 +- sdk/core/corehttp/tests/conftest.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core-experimental/tests/conftest.py b/sdk/core/azure-core-experimental/tests/conftest.py index 55134f14be5c..3d9f1674b350 100644 --- a/sdk/core/azure-core-experimental/tests/conftest.py +++ b/sdk/core/azure-core-experimental/tests/conftest.py @@ -36,7 +36,7 @@ def is_port_available(port_num): - req = urllib.request.Request("http://localhost:{}/health".format(port_num)) + req = urllib.request.Request("http://127.0.0.1:{}/health".format(port_num)) try: return urllib.request.urlopen(req).code != 200 except Exception as e: diff --git a/sdk/core/azure-core/tests/async_tests/conftest.py b/sdk/core/azure-core/tests/async_tests/conftest.py index 59cdfe4a3674..5d6a720e0bde 100644 --- a/sdk/core/azure-core/tests/async_tests/conftest.py +++ b/sdk/core/azure-core/tests/async_tests/conftest.py @@ -35,7 +35,7 @@ def is_port_available(port_num): - req = urllib.request.Request("http://localhost:{}/health".format(port_num)) + req = urllib.request.Request("http://127.0.0.1:{}/health".format(port_num)) try: return urllib.request.urlopen(req).code != 200 except Exception as e: diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index 0d9ebb6c8c24..0570f7ea9c8f 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -45,7 +45,7 @@ def is_port_available(port_num): - req = urllib.request.Request("http://localhost:{}/health".format(port_num)) + req = urllib.request.Request("http://127.0.0.1:{}/health".format(port_num)) try: return urllib.request.urlopen(req).code != 200 except Exception as e: diff --git a/sdk/core/corehttp/tests/conftest.py b/sdk/core/corehttp/tests/conftest.py index 47a5756f9e76..a8f6f2fbb295 100644 --- a/sdk/core/corehttp/tests/conftest.py +++ b/sdk/core/corehttp/tests/conftest.py @@ -23,7 +23,7 @@ def is_port_available(port_num): - req = urllib.request.Request("http://localhost:{}/health".format(port_num)) + req = urllib.request.Request("http://127.0.0.1:{}/health".format(port_num)) try: return urllib.request.urlopen(req).code != 200 except Exception as e: From d5824815e1f1cce90232c1bef380e7f140dcd9f4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 7 Apr 2026 23:57:37 +0000 Subject: [PATCH 24/29] revert the changes --- eng/pipelines/templates/steps/build-test.yml | 2 +- eng/scripts/Language-Settings.ps1 | 2 +- sdk/core/azure-core/dev_requirements.txt | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 4054c6d2478a..70ca1a534861 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -46,7 +46,7 @@ steps: Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - $(PIP_EXE) install -r eng/ci_tools.txt -v + $(PIP_EXE) install -r eng/ci_tools.txt $(PIP_EXE) freeze displayName: 'Prep Environment' diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 3b57c0b75b99..f1bc26a31b4e 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -166,7 +166,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) # Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip if (Get-Command uv -ErrorAction SilentlyContinue) { Write-Host "Using uv pip install" - $installerOutput = uv pip install "$pathToBuild" -vv + $installerOutput = uv pip install "$pathToBuild" Write-Host $installerOutput $freezeOutput = uv pip freeze Write-Host "Pip freeze output: $freezeOutput" diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 64b69cfe8252..c6c740fd4b75 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -13,4 +13,3 @@ opentelemetry-sdk~=1.26 opentelemetry-instrumentation-requests>=0.50b0 ../../identity/azure-identity packaging # for version parsing in test_basic_transport_async.py -ruff \ No newline at end of file From acc6758813a7de9bd04210088cb420efae0548a3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 8 Apr 2026 01:29:24 +0000 Subject: [PATCH 25/29] fix the remaining references to hitting pypi directly --- eng/tools/azure-sdk-tools/pypi_tools/azdo.py | 157 ++++++++++++++++++ eng/tools/azure-sdk-tools/pypi_tools/pypi.py | 69 ++++++-- .../azure-sdk-tools/tests/test_pypi_client.py | 152 ++++++++++++++--- 3 files changed, 335 insertions(+), 43 deletions(-) create mode 100644 eng/tools/azure-sdk-tools/pypi_tools/azdo.py diff --git a/eng/tools/azure-sdk-tools/pypi_tools/azdo.py b/eng/tools/azure-sdk-tools/pypi_tools/azdo.py new file mode 100644 index 000000000000..608a2ffd05fa --- /dev/null +++ b/eng/tools/azure-sdk-tools/pypi_tools/azdo.py @@ -0,0 +1,157 @@ +import base64 +import json +import logging +import os +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional +from urllib.parse import urlparse + +from packaging.version import Version, InvalidVersion, parse +from urllib3 import PoolManager, Retry + + +def pep503_normalize(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +@dataclass(frozen=True) +class AzureArtifactsFeedConfig: + organization: str + project: Optional[str] # None if the feed is organization-scoped + feed: str # feed name or GUID + api_version: str = "7.1" + + bearer_token: Optional[str] = None + pat: Optional[str] = None + + +# Pattern: https://pkgs.dev.azure.com/{org}/{project}/_packaging/{feed}/pypi/simple/ +# or org-scoped: https://pkgs.dev.azure.com/{org}/_packaging/{feed}/pypi/simple/ +_AZDO_FEED_RE = re.compile( + r"/(?P[^/]+)/(?:(?P[^/_][^/]*)/)?" r"_packaging/(?P[^/]+)/pypi/simple/?$" +) + + +def parse_pip_index_url(url: str) -> Optional[AzureArtifactsFeedConfig]: + """If *url* points to an Azure Artifacts PyPI feed, return a config; else None.""" + parsed = urlparse(url) + if "pkgs.dev.azure.com" not in parsed.hostname: + return None + + m = _AZDO_FEED_RE.search(parsed.path) + if not m: + return None + + # Embedded credentials from PipAuthenticate@1 + pat = None + if parsed.password: + pat = parsed.password + + return AzureArtifactsFeedConfig( + organization=m.group("org"), + project=m.group("project"), + feed=m.group("feed"), + pat=pat or os.environ.get("AZDO_PAT"), + ) + + +class AzureArtifactsClient: + """ + Minimal client to list package versions from an Azure Artifacts feed + via Azure DevOps Artifacts REST API. + """ + + def __init__(self, cfg: AzureArtifactsFeedConfig, base_url: str = "https://feeds.dev.azure.com"): + self._cfg = cfg + self._base_url = base_url.rstrip("/") + self._http = PoolManager( + retries=Retry(total=3, raise_on_status=True), + ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None), + ) + + def _auth_header(self) -> Dict[str, str]: + if self._cfg.bearer_token: + return {"Authorization": f"Bearer {self._cfg.bearer_token}"} + + if self._cfg.pat: + # Azure DevOps PATs can be used via HTTP Basic by base64-encoding ":". + token = base64.b64encode(f":{self._cfg.pat}".encode("utf-8")).decode("ascii") + return {"Authorization": f"Basic {token}"} + + return {} + + def _path_prefix(self) -> str: + # If project-scoped feed: /{org}/{project}/... + # If org-scoped feed: /{org}/... + if self._cfg.project: + return f"{self._cfg.organization}/{self._cfg.project}" + return self._cfg.organization + + def _get_json(self, url: str, params: Dict[str, Any]) -> Any: + headers = {"Accept": "application/json", **self._auth_header()} + r = self._http.request("GET", url, fields=params, headers=headers) + return json.loads(r.data.decode("utf-8")) + + def list_feeds(self) -> List[Dict[str, Any]]: + url = f"{self._base_url}/{self._path_prefix()}/_apis/packaging/feeds" + data = self._get_json(url, {"api-version": self._cfg.api_version}) + # Many Azure DevOps APIs return {"count": n, "value": [...]}; be tolerant. + return data["value"] if isinstance(data, dict) and "value" in data else data + + def resolve_feed_id(self) -> str: + feed = self._cfg.feed + if re.fullmatch(r"[0-9a-fA-F-]{36}", feed): + return feed + + for f in self.list_feeds(): + if f.get("name") == feed: + return f["id"] + + raise KeyError(f"Feed not found: {feed!r}") + + def get_package_record(self, package_name: str, include_deleted: bool = False) -> Dict[str, Any]: + feed_id = self.resolve_feed_id() + url = f"{self._base_url}/{self._path_prefix()}/_apis/packaging/Feeds/{feed_id}/packages" + + params = { + "api-version": self._cfg.api_version, + "protocolType": "pypi", + "packageNameQuery": package_name, + "includeAllVersions": "true", + "includeDeleted": "true" if include_deleted else "false", + } + + data = self._get_json(url, params) + packages = data["value"] if isinstance(data, dict) and "value" in data else data + + # packageNameQuery is "contains string", so choose best match. + target = pep503_normalize(package_name) + for pkg in packages: + if pep503_normalize(pkg.get("normalizedName", pkg.get("name", ""))) == target: + return pkg + for pkg in packages: + if pep503_normalize(pkg.get("name", "")) == target: + return pkg + + raise KeyError(f"Package not found in feed: {package_name!r}") + + def get_ordered_versions(self, package_name: str, include_deleted: bool = False) -> List[Version]: + pkg = self.get_package_record(package_name, include_deleted=include_deleted) + + out: List[Version] = [] + for v in pkg.get("versions", []): + if (not include_deleted) and v.get("isDeleted", False): + continue + + raw = v.get("version") + if not raw: + continue + + try: + out.append(parse(raw)) + except InvalidVersion: + logging.warning("Invalid version %r for package %s (feed=%s)", raw, package_name, self._cfg.feed) + + out.sort() + return out diff --git a/eng/tools/azure-sdk-tools/pypi_tools/pypi.py b/eng/tools/azure-sdk-tools/pypi_tools/pypi.py index b373029ff06e..6a7e02be611f 100644 --- a/eng/tools/azure-sdk-tools/pypi_tools/pypi.py +++ b/eng/tools/azure-sdk-tools/pypi_tools/pypi.py @@ -1,7 +1,6 @@ import logging from packaging.version import InvalidVersion, Version, parse import sys -import pdb from urllib3 import Retry, PoolManager import json import os @@ -16,19 +15,47 @@ def get_pypi_xmlrpc_client(): class PyPIClient: + """Unified package-index client. + + By default, reads ``PIP_INDEX_URL`` to decide the backend: + * If the URL contains ``pkgs.dev.azure.com`` → Azure Artifacts REST API. + * Otherwise → PyPI JSON API (``https://pypi.org``). + """ + def __init__(self, host="https://pypi.org"): - self._host = host - self._http = PoolManager( - retries=Retry(total=3, raise_on_status=True), ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None) - ) + index_url = os.environ.get("PIP_INDEX_URL", "") + + # Lazy import to avoid circular deps at module level. + from pypi_tools.azdo import parse_pip_index_url, AzureArtifactsClient + + azdo_cfg = parse_pip_index_url(index_url) if index_url else None + + if azdo_cfg is not None: + self._backend = "azdo" + self._azdo = AzureArtifactsClient(azdo_cfg) + else: + self._backend = "pypi" + self._host = host + self._http = PoolManager( + retries=Retry(total=3, raise_on_status=True), + ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None), + ) + + # ------------------------------------------------------------------ + # PyPI-only: raw JSON endpoints + # ------------------------------------------------------------------ def project(self, package_name): + if self._backend != "pypi": + raise NotImplementedError("project() is only available against pypi.org") response = self._http.request( "get", "{host}/pypi/{project_name}/json".format(host=self._host, project_name=package_name) ) return json.loads(response.data.decode("utf-8")) def project_release(self, package_name, version): + if self._backend != "pypi": + raise NotImplementedError("project_release() is only available against pypi.org") response = self._http.request( "get", "{host}/pypi/{project_name}/{version}/json".format( @@ -37,12 +64,18 @@ def project_release(self, package_name, version): ) return json.loads(response.data.decode("utf-8")) + # ------------------------------------------------------------------ + # Shared interface + # ------------------------------------------------------------------ + def filter_packages_for_compatibility(self, package_name, version_set): - # only need the packaging.specifiers import if we're actually executing this filter. + if self._backend != "pypi": + raise NotImplementedError( + "filter_packages_for_compatibility() requires pypi.org (needs requires_python metadata)" + ) from packaging.specifiers import InvalidSpecifier, SpecifierSet results: List[Version] = [] - for version in version_set: requires_python = self.project_release(package_name, version)["info"]["requires_python"] if requires_python: @@ -54,20 +87,25 @@ def filter_packages_for_compatibility(self, package_name, version_set): continue else: results.append(version) - return results def get_ordered_versions(self, package_name, filter_by_compatibility=False) -> List[Version]: - project = self.project(package_name) + if self._backend == "azdo": + versions = self._azdo.get_ordered_versions(package_name) + if filter_by_compatibility: + logging.warning( + "filter_by_compatibility is not supported against Azure Artifacts; returning unfiltered versions" + ) + return versions + project = self.project(package_name) versions: List[Version] = [] for package_version, files in project["releases"].items(): try: - # Skip yanked versions (no files or all files yanked) if not files or all(f.get("yanked", False) for f in files): continue versions.append(parse(package_version)) - except InvalidVersion as e: + except InvalidVersion: logging.warn(f"Invalid version {package_version} for package {package_name}") continue versions.sort() @@ -78,9 +116,7 @@ def get_ordered_versions(self, package_name, filter_by_compatibility=False) -> L return versions def get_relevant_versions(self, package_name): - """Return a tuple: (latest release, latest stable) - If there are different, it means the latest is not a stable - """ + """Return a tuple: (latest release, latest stable)""" versions = self.get_ordered_versions(package_name) stable_releases = [version for version in versions if not version.is_prerelease] return (versions[-1], stable_releases[-1]) @@ -88,7 +124,7 @@ def get_relevant_versions(self, package_name): def retrieve_versions_from_pypi(package_name: str) -> List[str]: """ - Retrieve all published versions on PyPI for the package. + Retrieve all published versions for the package from the active index. :param str package_name: The name of the package. :rtype: List[str] @@ -97,8 +133,7 @@ def retrieve_versions_from_pypi(package_name: str) -> List[str]: try: client = PyPIClient() all_versions = client.get_ordered_versions(package_name) - # Return all versions as strings return [str(v) for v in all_versions] except Exception as ex: - logging.warning("Failed to retrieve PyPI data for %s: %s", package_name, ex) + logging.warning("Failed to retrieve package data for %s: %s", package_name, ex) return [] diff --git a/eng/tools/azure-sdk-tools/tests/test_pypi_client.py b/eng/tools/azure-sdk-tools/tests/test_pypi_client.py index cb2de3dfa154..c7fa00cdacc3 100644 --- a/eng/tools/azure-sdk-tools/tests/test_pypi_client.py +++ b/eng/tools/azure-sdk-tools/tests/test_pypi_client.py @@ -1,41 +1,141 @@ -from pypi_tools.pypi import PyPIClient +from pypi_tools.pypi import PyPIClient, retrieve_versions_from_pypi from unittest.mock import patch import os import pytest -import pdb +from packaging.version import Version -class TestPyPiClient: - @pytest.mark.skipif( - os.environ.get("TF_BUILD", "None") == True, - reason=f"This test isn't worth recording and could be flaky. Skipping in CI.", - ) - def test_package_retrieve(self): +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +SKIP_IN_CI = pytest.mark.skipif( + os.environ.get("TF_BUILD", "None") == True, + reason="Live network test — skipped in CI.", +) + +PYPI_HOST = "https://pypi.org" +AZDO_FEED_URL = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + +WELL_KNOWN_PACKAGE = "azure-core" +WELL_KNOWN_VERSION = "1.8.0" # old enough to always exist everywhere +MINIMUM_EXPECTED_VERSIONS = 47 # azure-core has *far* more than this + + +def _make_client(index_url: str) -> PyPIClient: + """Create a PyPIClient whose backend is driven by *index_url*. + + Temporarily sets PIP_INDEX_URL so the constructor picks the right backend. + """ + old = os.environ.get("PIP_INDEX_URL") + try: + if index_url: + os.environ["PIP_INDEX_URL"] = index_url + elif "PIP_INDEX_URL" in os.environ: + del os.environ["PIP_INDEX_URL"] + return PyPIClient() + finally: + if old is not None: + os.environ["PIP_INDEX_URL"] = old + elif "PIP_INDEX_URL" in os.environ: + del os.environ["PIP_INDEX_URL"] + + +# --------------------------------------------------------------------------- +# Tests parametrized across backends +# --------------------------------------------------------------------------- + + +@pytest.fixture(params=[PYPI_HOST, AZDO_FEED_URL], ids=["pypi", "azdo"]) +def client(request): + return _make_client(request.param) + + +class TestGetOrderedVersions: + """Covers the dominant call pattern: ~10 call-sites do get_ordered_versions().""" + + @SKIP_IN_CI + def test_returns_sorted_version_objects(self, client): + versions = client.get_ordered_versions(WELL_KNOWN_PACKAGE) + assert len(versions) >= MINIMUM_EXPECTED_VERSIONS + assert all(isinstance(v, Version) for v in versions) + assert versions == sorted(versions) + + @SKIP_IN_CI + def test_known_version_present(self, client): + versions = client.get_ordered_versions(WELL_KNOWN_PACKAGE) + version_strs = [str(v) for v in versions] + assert WELL_KNOWN_VERSION in version_strs + + +class TestGetRelevantVersions: + """Covers detect_breaking_changes.py usage of get_relevant_versions().""" + + @SKIP_IN_CI + def test_returns_latest_and_latest_stable(self, client): + latest, latest_stable = client.get_relevant_versions(WELL_KNOWN_PACKAGE) + assert isinstance(latest, Version) + assert isinstance(latest_stable, Version) + assert not latest_stable.is_prerelease + assert latest_stable <= latest + + +class TestRetrieveVersions: + """Covers the convenience wrapper used by verify_sdist.py / verify_whl.py.""" + + @SKIP_IN_CI + @pytest.mark.parametrize("index_url", [PYPI_HOST, AZDO_FEED_URL], ids=["pypi", "azdo"]) + def test_retrieve_versions_returns_strings(self, index_url): + old = os.environ.get("PIP_INDEX_URL") + try: + if index_url: + os.environ["PIP_INDEX_URL"] = index_url + versions = retrieve_versions_from_pypi(WELL_KNOWN_PACKAGE) + finally: + if old is not None: + os.environ["PIP_INDEX_URL"] = old + elif "PIP_INDEX_URL" in os.environ: + del os.environ["PIP_INDEX_URL"] + + assert len(versions) >= MINIMUM_EXPECTED_VERSIONS + assert all(isinstance(v, str) for v in versions) + assert WELL_KNOWN_VERSION in versions + + +# --------------------------------------------------------------------------- +# PyPI-only tests (project / project_release / filter_packages_for_compatibility) +# --------------------------------------------------------------------------- + + +class TestPyPIOnlyMethods: + """Methods that only work against pypi.org JSON API. + + Callers: discover_unpublished_packages.py, output_old_packages.py. + """ + + @SKIP_IN_CI + def test_project_returns_info_and_releases(self): client = PyPIClient() - result = client.project("azure-core") + result = client.project(WELL_KNOWN_PACKAGE) - # we won't _exhaustively_ check this, but we can sanity check a few proxy values to ensure we haven't broken anything - assert result["info"]["name"] == "azure-core" - assert len(result["releases"].keys()) > 47 + assert result["info"]["name"] == WELL_KNOWN_PACKAGE + assert len(result["releases"]) > MINIMUM_EXPECTED_VERSIONS assert "1.25.1" in result["releases"] - assert "1.10.0" in result["releases"] + assert WELL_KNOWN_VERSION in result["releases"] - @pytest.mark.skipif( - os.environ.get("TF_BUILD", "None") == True, - reason=f"This test isn't worth recording and could be flaky. Skipping in CI.", - ) - def test_package_version_retrieve(self): + @SKIP_IN_CI + def test_project_release_returns_version_info(self): client = PyPIClient() - result = client.project_release("azure-core", "1.8.0") + result = client.project_release(WELL_KNOWN_PACKAGE, WELL_KNOWN_VERSION) - assert result["info"]["name"] == "azure-core" - assert result["info"]["release_url"] == "https://pypi.org/project/azure-core/1.8.0/" + assert result["info"]["name"] == WELL_KNOWN_PACKAGE + assert result["info"]["release_url"] == f"https://pypi.org/project/{WELL_KNOWN_PACKAGE}/{WELL_KNOWN_VERSION}/" + @SKIP_IN_CI @patch("pypi_tools.pypi.sys") - def test_package_filter_for_compatibility(self, mock_sys): + def test_filter_packages_for_compatibility(self, mock_sys): mock_sys.version_info = (2, 7, 0) client = PyPIClient() - result = client.get_ordered_versions("azure-core", True) - unfiltered_results = client.get_ordered_versions("azure-core", False) - - assert len(result) < len(unfiltered_results) + filtered = client.get_ordered_versions(WELL_KNOWN_PACKAGE, True) + unfiltered = client.get_ordered_versions(WELL_KNOWN_PACKAGE, False) + assert len(filtered) < len(unfiltered) From 48388288d105b96f3f54946b59fac83f61e9adb6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 8 Apr 2026 02:08:09 +0000 Subject: [PATCH 26/29] update project_release and test_pypi_client.py --- eng/tools/azure-sdk-tools/pypi_tools/pypi.py | 36 +++++++++++-------- .../azure-sdk-tools/tests/test_pypi_client.py | 28 ++++++++++----- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/eng/tools/azure-sdk-tools/pypi_tools/pypi.py b/eng/tools/azure-sdk-tools/pypi_tools/pypi.py index 6a7e02be611f..d7fc7fd03ffc 100644 --- a/eng/tools/azure-sdk-tools/pypi_tools/pypi.py +++ b/eng/tools/azure-sdk-tools/pypi_tools/pypi.py @@ -41,28 +41,36 @@ def __init__(self, host="https://pypi.org"): ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None), ) + def _pypi_http(self): + """Lazy PoolManager for pypi.org fallback when on AzDO backend.""" + if not hasattr(self, "_pypi_http_pool"): + self._pypi_http_pool = PoolManager( + retries=Retry(total=3, raise_on_status=True), + ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None), + ) + return self._pypi_http_pool + + def _pypi_json_request(self, path): + """GET from pypi.org JSON API, using the active backend's http pool if on pypi, else fallback.""" + if self._backend == "pypi": + url = "{host}{path}".format(host=self._host, path=path) + response = self._http.request("get", url) + else: + url = "https://pypi.org{path}".format(path=path) + response = self._pypi_http().request("get", url) + return json.loads(response.data.decode("utf-8")) + # ------------------------------------------------------------------ - # PyPI-only: raw JSON endpoints + # PyPI JSON endpoints (fall back to pypi.org when on AzDO backend) # ------------------------------------------------------------------ def project(self, package_name): if self._backend != "pypi": raise NotImplementedError("project() is only available against pypi.org") - response = self._http.request( - "get", "{host}/pypi/{project_name}/json".format(host=self._host, project_name=package_name) - ) - return json.loads(response.data.decode("utf-8")) + return self._pypi_json_request("/pypi/{}/json".format(package_name)) def project_release(self, package_name, version): - if self._backend != "pypi": - raise NotImplementedError("project_release() is only available against pypi.org") - response = self._http.request( - "get", - "{host}/pypi/{project_name}/{version}/json".format( - host=self._host, project_name=package_name, version=version - ), - ) - return json.loads(response.data.decode("utf-8")) + return self._pypi_json_request("/pypi/{}/{}/json".format(package_name, version)) # ------------------------------------------------------------------ # Shared interface diff --git a/eng/tools/azure-sdk-tools/tests/test_pypi_client.py b/eng/tools/azure-sdk-tools/tests/test_pypi_client.py index c7fa00cdacc3..186837dd0349 100644 --- a/eng/tools/azure-sdk-tools/tests/test_pypi_client.py +++ b/eng/tools/azure-sdk-tools/tests/test_pypi_client.py @@ -103,7 +103,25 @@ def test_retrieve_versions_returns_strings(self, index_url): # --------------------------------------------------------------------------- -# PyPI-only tests (project / project_release / filter_packages_for_compatibility) +# project_release — works on both backends (AzDO falls back to pypi.org) +# --------------------------------------------------------------------------- + + +class TestProjectRelease: + """Covers functions.py:888 usage of project_release() for requires_dist.""" + + @SKIP_IN_CI + def test_project_release_returns_version_info(self, client): + result = client.project_release(WELL_KNOWN_PACKAGE, WELL_KNOWN_VERSION) + + assert result["info"]["name"] == WELL_KNOWN_PACKAGE + assert result["info"]["release_url"] == f"https://pypi.org/project/{WELL_KNOWN_PACKAGE}/{WELL_KNOWN_VERSION}/" + # requires_dist is what the mindep resolver reads + assert "requires_dist" in result["info"] + + +# --------------------------------------------------------------------------- +# PyPI-only tests (project / filter_packages_for_compatibility) # --------------------------------------------------------------------------- @@ -123,14 +141,6 @@ def test_project_returns_info_and_releases(self): assert "1.25.1" in result["releases"] assert WELL_KNOWN_VERSION in result["releases"] - @SKIP_IN_CI - def test_project_release_returns_version_info(self): - client = PyPIClient() - result = client.project_release(WELL_KNOWN_PACKAGE, WELL_KNOWN_VERSION) - - assert result["info"]["name"] == WELL_KNOWN_PACKAGE - assert result["info"]["release_url"] == f"https://pypi.org/project/{WELL_KNOWN_PACKAGE}/{WELL_KNOWN_VERSION}/" - @SKIP_IN_CI @patch("pypi_tools.pypi.sys") def test_filter_packages_for_compatibility(self, mock_sys): From 6741df5dccb08623ee87a1689f5829108b117968 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 8 Apr 2026 02:10:51 +0000 Subject: [PATCH 27/29] ensure that we auth the dev feed on generating job matrix and regression matrix --- eng/pipelines/templates/jobs/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 61d063daa808..ccb16f34440e 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -285,6 +285,9 @@ jobs: inputs: versionSpec: '3.12' - template: /eng/pipelines/templates/steps/use-venv.yml + - template: ../steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} @@ -323,6 +326,11 @@ jobs: inputs: versionSpec: '3.12' - template: /eng/pipelines/templates/steps/use-venv.yml + + - template: ../steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + - pwsh: | $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true From 2b15c696ca1d0b7e789757bd333d34385f40757b Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 8 Apr 2026 17:20:45 +0000 Subject: [PATCH 28/29] account for requests maximum on 2.32.5 --- .../ci_tools/scenario/dependency_resolution.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py index 7e357aee1b41..86924647ceaa 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py +++ b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py @@ -86,7 +86,11 @@ }, } -PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES = {} +PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES = { + "<3.10.0": { + "requests": "2.32.5" + } +} # This is used to actively _add_ requirements to the install set. These are used to actively inject # a new requirement specifier to the set of packages being installed. From 2f7ac381fcadc87331c4a0ee7afa91877bef1a67 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 8 Apr 2026 17:36:56 +0000 Subject: [PATCH 29/29] apply formatting --- .../ci_tools/scenario/dependency_resolution.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py index 86924647ceaa..1e6a96720547 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py +++ b/eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py @@ -86,11 +86,7 @@ }, } -PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES = { - "<3.10.0": { - "requests": "2.32.5" - } -} +PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES = {"<3.10.0": {"requests": "2.32.5"}} # This is used to actively _add_ requirements to the install set. These are used to actively inject # a new requirement specifier to the set of packages being installed.