Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
6b3ca6a
simplify the location of auth-dev-feed so that it works for both play…
scbedd Mar 30, 2026
f58b1da
make build test echo out the feed being utilized
scbedd Mar 30, 2026
cf28c2f
didn't actually uncomment the UV index setting
scbedd Mar 31, 2026
adbbb3c
Update eng/pipelines/templates/steps/build-test.yml
scbedd Mar 31, 2026
77e9ee9
update step name, add a requirement that is NOT YET in our dev feed t…
scbedd Mar 31, 2026
0e824b6
more manipulation
scbedd Mar 31, 2026
e39fe81
UV won't pull stuff automatically into the feed? need to add pip extr…
scbedd Mar 31, 2026
d0c6272
revert attempt so far. add explicit check for installing ci_tools.txt
scbedd Mar 31, 2026
017ef3c
accidentally disabled UV
scbedd Apr 1, 2026
ce01280
test a pip install
scbedd Apr 3, 2026
2f48306
ensure that new packages can be pulled in
scbedd Apr 3, 2026
333736d
install pip and call
scbedd Apr 3, 2026
11f0d58
log properly
scbedd Apr 4, 2026
c892017
UV_INDEX_URL -> UV_DEFAULT_INDEX
scbedd Apr 4, 2026
90d6c47
use environment variables only. name the index
scbedd Apr 6, 2026
8d60b83
try again with the unauthed string + UV_INDX value
scbedd Apr 6, 2026
3bd9af3
target a package that ISNT in the feed, is UV able to retrieve it pro…
scbedd Apr 6, 2026
f1b5007
-vv install! WHAT IS HAPPENING
scbedd Apr 6, 2026
3ffc9e4
remove some extra changes that were added for debugging
scbedd Apr 7, 2026
5504fde
verbose output so we can see why this thing is blowing up
scbedd Apr 7, 2026
84505b1
pin older version of cibuildwheel during save package properties
scbedd Apr 7, 2026
e6e4ed9
cibuildwheel needs to be pinned as it's installing a new version of p…
scbedd Apr 7, 2026
eecf6ab
on newer macs, we're seeing the port resolve to an IPV6 address by de…
scbedd Apr 7, 2026
d582481
revert the changes
scbedd Apr 7, 2026
acc6758
fix the remaining references to hitting pypi directly
scbedd Apr 8, 2026
4838828
update project_release and test_pypi_client.py
scbedd Apr 8, 2026
6741df5
ensure that we auth the dev feed on generating job matrix and regress…
scbedd Apr 8, 2026
2b15c69
account for requests maximum on 2.32.5
scbedd Apr 8, 2026
2f7ac38
apply formatting
scbedd Apr 8, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions eng/pipelines/templates/jobs/ci.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,6 @@ jobs:
Paths:
- '**'

# Authenticate to Azure Artifacts feed immediately after checkout to prevent 401 errors
# Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org
- template: /eng/pipelines/templates/steps/auth-dev-feed.yml

- template: /eng/pipelines/templates/steps/download-package-artifacts.yml

- template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
Expand Down
8 changes: 8 additions & 0 deletions eng/pipelines/templates/jobs/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,9 @@ jobs:
inputs:
versionSpec: '3.12'
- template: /eng/pipelines/templates/steps/use-venv.yml
- template: ../steps/auth-dev-feed.yml
parameters:
DevFeedName: ${{ parameters.DevFeedName }}
- template: /eng/common/pipelines/templates/steps/save-package-properties.yml
parameters:
ServiceDirectory: ${{parameters.ServiceDirectory}}
Expand Down Expand Up @@ -323,6 +326,11 @@ jobs:
inputs:
versionSpec: '3.12'
- template: /eng/pipelines/templates/steps/use-venv.yml

- template: ../steps/auth-dev-feed.yml
parameters:
DevFeedName: ${{ parameters.DevFeedName }}

- pwsh: |
$ErrorActionPreference = 'Stop'
$PSNativeCommandUseErrorActionPreference = $true
Expand Down
22 changes: 11 additions & 11 deletions eng/pipelines/templates/steps/auth-dev-feed.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,17 @@ steps:
artifactFeeds: $(DevFeedName)
onlyAddExtraIndex: false

# Disabled UV authentication step - for future enablement when UV fully supports Azure Artifacts
# To enable: uncomment this step and ensure UV_INDEX_URL is properly configured
# The PipAuthenticate task above creates a .pypirc file in the home directory that UV can use
- pwsh: |
# This step configures UV to use the same authentication as pip
# UV will read credentials from ~/.pypirc created by PipAuthenticate task
if ($env:PIP_INDEX_URL) {
Write-Host "PIP Index URL detected: $env:PIP_INDEX_URL"
# Uncomment the next line to enable UV authentication
# Write-Host "##vso[task.setvariable variable=UV_INDEX_URL]$($env:PIP_INDEX_URL)"
Write-Host "UV authentication is currently disabled. To enable, uncomment the variable assignment above."
# UV_DEFAULT_INDEX is the canonical replacement for the deprecated UV_INDEX_URL (uv 0.4.23+).
# PIP_INDEX_URL is set by PipAuthenticate@1 and contains embedded credentials, which uv
# will use for Basic auth against the ADO feed (and its PyPI upstream) per astral-sh/uv#12651.
Write-Host "##vso[task.setvariable variable=UV_DEFAULT_INDEX]azure-sdk=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
# Disable keyring so uv uses the URL-embedded credentials directly.
Write-Host "##vso[task.setvariable variable=UV_KEYRING_PROVIDER]disabled"
Write-Host "##vso[task.setvariable variable=UV_INDEX_AZURE_SDK_USERNAME]x"
Write-Host "##vso[task.setvariable variable=UV_INDEX_AZURE_SDK_PASSWORD]$(System.AccessToken)"
} else {
Write-Host "##[warning]PIP_INDEX_URL not set - uv will fall back to public PyPI."
}
displayName: 'Configure UV Authentication (Disabled)'
condition: false # Explicitly disabled - change to 'true' or remove to enable
displayName: 'Configure UV Authentication'
6 changes: 6 additions & 0 deletions eng/pipelines/templates/steps/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ steps:

- template: /eng/pipelines/templates/steps/use-venv.yml

# Authenticate to Azure Artifacts feed immediately after checkout to prevent 401 errors
# Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org
- template: /eng/pipelines/templates/steps/auth-dev-feed.yml
parameters:
DevFeedName: ${{ parameters.DevFeedName }}

- template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
parameters:
PackageName: "azure-template"
Expand Down
3 changes: 2 additions & 1 deletion eng/scripts/Language-Settings.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,8 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory)
# Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip
if (Get-Command uv -ErrorAction SilentlyContinue) {
Write-Host "Using uv pip install"
$null = uv pip install "$pathToBuild"
$installerOutput = uv pip install "$pathToBuild"
Write-Host $installerOutput
$freezeOutput = uv pip freeze
Write-Host "Pip freeze output: $freezeOutput"
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
},
}

PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES = {}
PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES = {"<3.10.0": {"requests": "2.32.5"}}

# This is used to actively _add_ requirements to the install set. These are used to actively inject
# a new requirement specifier to the set of packages being installed.
Expand Down
157 changes: 157 additions & 0 deletions eng/tools/azure-sdk-tools/pypi_tools/azdo.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
import base64
import json
import logging
import os
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse

from packaging.version import Version, InvalidVersion, parse
from urllib3 import PoolManager, Retry


def pep503_normalize(name: str) -> str:
return re.sub(r"[-_.]+", "-", name).lower()


@dataclass(frozen=True)
class AzureArtifactsFeedConfig:
organization: str
project: Optional[str] # None if the feed is organization-scoped
feed: str # feed name or GUID
api_version: str = "7.1"

bearer_token: Optional[str] = None
pat: Optional[str] = None


# Pattern: https://pkgs.dev.azure.com/{org}/{project}/_packaging/{feed}/pypi/simple/
# or org-scoped: https://pkgs.dev.azure.com/{org}/_packaging/{feed}/pypi/simple/
_AZDO_FEED_RE = re.compile(
r"/(?P<org>[^/]+)/(?:(?P<project>[^/_][^/]*)/)?" r"_packaging/(?P<feed>[^/]+)/pypi/simple/?$"
)


def parse_pip_index_url(url: str) -> Optional[AzureArtifactsFeedConfig]:
"""If *url* points to an Azure Artifacts PyPI feed, return a config; else None."""
parsed = urlparse(url)
if "pkgs.dev.azure.com" not in parsed.hostname:
return None

m = _AZDO_FEED_RE.search(parsed.path)
if not m:
return None

# Embedded credentials from PipAuthenticate@1
pat = None
if parsed.password:
pat = parsed.password

return AzureArtifactsFeedConfig(
organization=m.group("org"),
project=m.group("project"),
feed=m.group("feed"),
pat=pat or os.environ.get("AZDO_PAT"),
)


class AzureArtifactsClient:
"""
Minimal client to list package versions from an Azure Artifacts feed
via Azure DevOps Artifacts REST API.
"""

def __init__(self, cfg: AzureArtifactsFeedConfig, base_url: str = "https://feeds.dev.azure.com"):
self._cfg = cfg
self._base_url = base_url.rstrip("/")
self._http = PoolManager(
retries=Retry(total=3, raise_on_status=True),
ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None),
)

def _auth_header(self) -> Dict[str, str]:
if self._cfg.bearer_token:
return {"Authorization": f"Bearer {self._cfg.bearer_token}"}

if self._cfg.pat:
# Azure DevOps PATs can be used via HTTP Basic by base64-encoding ":<PAT>".
token = base64.b64encode(f":{self._cfg.pat}".encode("utf-8")).decode("ascii")
return {"Authorization": f"Basic {token}"}

return {}

def _path_prefix(self) -> str:
# If project-scoped feed: /{org}/{project}/...
# If org-scoped feed: /{org}/...
if self._cfg.project:
return f"{self._cfg.organization}/{self._cfg.project}"
return self._cfg.organization

def _get_json(self, url: str, params: Dict[str, Any]) -> Any:
headers = {"Accept": "application/json", **self._auth_header()}
r = self._http.request("GET", url, fields=params, headers=headers)
return json.loads(r.data.decode("utf-8"))

def list_feeds(self) -> List[Dict[str, Any]]:
url = f"{self._base_url}/{self._path_prefix()}/_apis/packaging/feeds"
data = self._get_json(url, {"api-version": self._cfg.api_version})
# Many Azure DevOps APIs return {"count": n, "value": [...]}; be tolerant.
return data["value"] if isinstance(data, dict) and "value" in data else data

def resolve_feed_id(self) -> str:
feed = self._cfg.feed
if re.fullmatch(r"[0-9a-fA-F-]{36}", feed):
return feed

for f in self.list_feeds():
if f.get("name") == feed:
return f["id"]

raise KeyError(f"Feed not found: {feed!r}")

def get_package_record(self, package_name: str, include_deleted: bool = False) -> Dict[str, Any]:
feed_id = self.resolve_feed_id()
url = f"{self._base_url}/{self._path_prefix()}/_apis/packaging/Feeds/{feed_id}/packages"

params = {
"api-version": self._cfg.api_version,
"protocolType": "pypi",
"packageNameQuery": package_name,
"includeAllVersions": "true",
"includeDeleted": "true" if include_deleted else "false",
}

data = self._get_json(url, params)
packages = data["value"] if isinstance(data, dict) and "value" in data else data

# packageNameQuery is "contains string", so choose best match.
target = pep503_normalize(package_name)
for pkg in packages:
if pep503_normalize(pkg.get("normalizedName", pkg.get("name", ""))) == target:
return pkg
for pkg in packages:
if pep503_normalize(pkg.get("name", "")) == target:
return pkg

raise KeyError(f"Package not found in feed: {package_name!r}")

def get_ordered_versions(self, package_name: str, include_deleted: bool = False) -> List[Version]:
pkg = self.get_package_record(package_name, include_deleted=include_deleted)

out: List[Version] = []
for v in pkg.get("versions", []):
if (not include_deleted) and v.get("isDeleted", False):
continue

raw = v.get("version")
if not raw:
continue

try:
out.append(parse(raw))
except InvalidVersion:
logging.warning("Invalid version %r for package %s (feed=%s)", raw, package_name, self._cfg.feed)

out.sort()
return out
Loading
Loading