From d15379c32e2a19456bcb322fe72ee7a52aa2df8d Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 15:02:52 +0000 Subject: [PATCH 01/24] copier --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 283cac3d..93885c56 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-44-gea357db +_commit: v0.0.8-1-g7ff0593 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1e863fed..c97f0fc8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,8 +112,8 @@ jobs: - name: install new dependencies env: + # Provide a fake token so it doesn't attempt to access AWS to generate a new one when the script is run if CodeArtifact is set as the registry CODEARTIFACT_AUTH_TOKEN: 'faketoken' - UV_NO_CACHE: 'true' run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI python $RUNNER_TEMP/replace_private_package_registries.py From 29b0e546ca45e4b9a44ace0bfc1609946630ac8a Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 16:46:04 +0000 Subject: [PATCH 02/24] staging initial workflow --- .../workflows/publish_to_staging.jinja | 97 +++++++++++++++++++ template/pyproject.toml.jinja | 4 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 template/.github/workflows/publish_to_staging.jinja diff --git a/template/.github/workflows/publish_to_staging.jinja b/template/.github/workflows/publish_to_staging.jinja new file mode 100644 index 00000000..2ccca5ef --- /dev/null +++ b/template/.github/workflows/publish_to_staging.jinja @@ -0,0 +1,97 @@ +{% raw %}name: Publish to PyPI + +on: + workflow_dispatch: + +env: + PYTHONUNBUFFERED: True + PRE_COMMIT_HOME: ${{ github.workspace }}/.precommit_cache + +permissions: + id-token: write + contents: write # needed for mutex + +jobs: + lint: + name: Pre-commit + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + + - name: Install latest versions of python packages + uses: ./.github/actions/install_deps_uv + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + + - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 + if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 + uses: ben-z/gh-action-mutex@{% endraw %}{{ gha_mutex }}{% raw %} + with: + branch: mutex-venv-{% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %} + timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it + + - name: Cache Pre-commit hooks + uses: actions/cache@{% endraw %}{{ gha_cache }}{% raw %} + env: + cache-name: cache-pre-commit-hooks + with: + path: ${{ env.PRE_COMMIT_HOME }} + key: {% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %}-build-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + {% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %}-build-${{ env.cache-name }}- + + - name: Run pre-commit + run: pre-commit run -a + + test: + needs: [ lint ] + strategy: + matrix: + os: + - "{% endraw %}{{ gha_linux_runner }}{% raw %}" +{% endraw %}{% if use_windows_in_ci %} - {{ gha_windows_runner }}{% endif %}{% raw %} + python-version: +{% endraw %}{% for item in python_ci_versions %} + - {{ item }} +{% endfor %}{% raw %} + include: + - os: "{% endraw %}{{ gha_linux_runner }}{% raw %}" + python-version: "{% endraw %}{{ python_ci_versions[0] }}{% raw %}" + JOB_MATCHING_DEV_ENV: true + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + + - name: Install python tooling + uses: ./.github/actions/install_deps_uv + with: + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: "{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}" + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + + - name: Unit test + run: uv run pytest --durations=5 + + build: + needs: [ test ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + + - name: Install python tooling + uses: ./.github/actions/install_deps_uv + with: + python-version: {% endraw %}{{ python_version }}{% raw %}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: "{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}" + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + + - name: Build package + run: uv build --no-sources{% endraw %} diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 6b0e1f30..0936aae6 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -54,8 +54,10 @@ default = true name = "code-artifact-primary" username = "aws" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" +publish-url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/" [[tool.uv.index]] name = "code-artifact-staging" username = "aws" -url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/"{% endraw %}{% endif %} +url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/" +publish-url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/"{% endraw %}{% endif %} From 11f1a3a9dcfbe9d8d5f02c446dd242ccc928d6f7 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 16:47:23 +0000 Subject: [PATCH 03/24] file name --- .../{publish_to_staging.jinja => publish_to_staging.yaml.jinja} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename template/.github/workflows/{publish_to_staging.jinja => publish_to_staging.yaml.jinja} (98%) diff --git a/template/.github/workflows/publish_to_staging.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja similarity index 98% rename from template/.github/workflows/publish_to_staging.jinja rename to template/.github/workflows/publish_to_staging.yaml.jinja index 2ccca5ef..a750be27 100644 --- a/template/.github/workflows/publish_to_staging.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -1,4 +1,4 @@ -{% raw %}name: Publish to PyPI +{% raw %}name: Publish to Staging Package Registry on: workflow_dispatch: From 84fe265eaec67e1707adab93b55ada4fa1d7a9fd Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 16:54:13 +0000 Subject: [PATCH 04/24] add publish --- template/.github/workflows/publish_to_staging.yaml.jinja | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index a750be27..49ee4635 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -94,4 +94,7 @@ jobs: code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Build package - run: uv build --no-sources{% endraw %} + run: uv build --no-sources + + - name: Publish package + run: uv publish --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} From 60f2f3c0ec709b3ba27ff91e9f14563d53059931 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 16:55:23 +0000 Subject: [PATCH 05/24] deps for precommit --- template/.github/workflows/publish_to_staging.yaml.jinja | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index 49ee4635..78b19e29 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -22,7 +22,10 @@ jobs: - name: Install latest versions of python packages uses: ./.github/actions/install_deps_uv with: - python-version: {% endraw %}{{ python_version }}{% raw %} + python-version: {% endraw %}{{ python_version }}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 From 440c0038c7efebea5215613daa5629d5e62d1dbe Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 17:00:38 +0000 Subject: [PATCH 06/24] auth for publish --- template/.github/workflows/publish_to_staging.yaml.jinja | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index 78b19e29..08f6d666 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -98,6 +98,14 @@ jobs: - name: Build package run: uv build --no-sources +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Staging-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + +{% endraw %}{% endif %}{% raw %} - name: Publish package run: uv publish --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} From fd5f17a515d83d038e686a3ee4ff5126ccbf4730 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 17:16:24 +0000 Subject: [PATCH 07/24] add setuptools --- template/pyproject.toml.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 0936aae6..b238a3c0 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -21,7 +21,7 @@ dependencies = [ [dependency-groups] dev = [ - # Specific to this template + # Specific to this repository # Managed by upstream template @@ -29,6 +29,7 @@ dev = [ "pytest>={% endraw %}{{ pytest_version }}{% raw %}", "pytest-cov>={% endraw %}{{ pytest_cov_version }}{% raw %}", "pytest-randomly>={% endraw %}{{ pytest_randomly_version }}{% raw %}", + "setuptools">={% endraw %}{{ setuptools_version }}{% raw %}", # likely related to the kludge for license-files below {% endraw %}{% if create_docs %}{% raw %} "sphinx=={% endraw %}{{ sphinx_version }}{% raw %}",{% endraw %}{% endif %}{% raw %} {% endraw %}{% if is_frozen_executable %}{% raw %} "pyinstaller>={% endraw %}{{ pyinstaller_version }}{% raw %}",{% endraw %}{% endif %}{% raw %} ] From 1f9ffadf4ebcebeaa41a4857316cfe6c06a61afe Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 17:16:47 +0000 Subject: [PATCH 08/24] copier --- .copier-answers.yml | 2 +- extensions/context.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 93885c56..c4814aba 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.8-1-g7ff0593 +_commit: v0.0.8-2-gfd4c577 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/extensions/context.py b/extensions/context.py index 92d01e0a..219c572f 100644 --- a/extensions/context.py +++ b/extensions/context.py @@ -28,6 +28,7 @@ def hook(self, context: dict[Any, Any]) -> dict[Any, Any]: context["ephemeral_pulumi_deploy_version"] = "0.0.2" context["pydantic_version"] = "2.10.6" context["pyinstaller_version"] = "6.12.0" + context["setuptools_version"] = "76.0.0" context["gha_checkout"] = "v4.2.2" context["gha_setup_python"] = "v5.4.0" From 6e3d9d329b74f91ee1c265acf3345606ade551ee Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 17:17:36 +0000 Subject: [PATCH 09/24] syntax --- template/pyproject.toml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index b238a3c0..6ce59ebe 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -29,7 +29,7 @@ dev = [ "pytest>={% endraw %}{{ pytest_version }}{% raw %}", "pytest-cov>={% endraw %}{{ pytest_cov_version }}{% raw %}", "pytest-randomly>={% endraw %}{{ pytest_randomly_version }}{% raw %}", - "setuptools">={% endraw %}{{ setuptools_version }}{% raw %}", # likely related to the kludge for license-files below + "setuptools>={% endraw %}{{ setuptools_version }}{% raw %}", # likely related to the kludge for license-files below {% endraw %}{% if create_docs %}{% raw %} "sphinx=={% endraw %}{{ sphinx_version }}{% raw %}",{% endraw %}{% endif %}{% raw %} {% endraw %}{% if is_frozen_executable %}{% raw %} "pyinstaller>={% endraw %}{{ pyinstaller_version }}{% raw %}",{% endraw %}{% endif %}{% raw %} ] From 8d52a53d183d93b56fb007dcfcd26feeb5a6984e Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:05:39 +0000 Subject: [PATCH 10/24] auth --- template/.github/workflows/publish_to_staging.yaml.jinja | 7 +++++++ template/pyproject.toml.jinja | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index 08f6d666..a2015b2b 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -95,7 +95,14 @@ jobs: code-artifact-auth-role-name: CoreInfraBaseAccess code-artifact-auth-role-account-id: "{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}" code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Installing any dependencies that uv may need for build (sometimes it likes to install setuptools...even if it's already in the package dependencies) + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/CoreInfraBaseAccess + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} +{% endraw %}{% endif %}{% raw %} - name: Build package run: uv build --no-sources {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 6ce59ebe..d6e49997 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -29,7 +29,6 @@ dev = [ "pytest>={% endraw %}{{ pytest_version }}{% raw %}", "pytest-cov>={% endraw %}{{ pytest_cov_version }}{% raw %}", "pytest-randomly>={% endraw %}{{ pytest_randomly_version }}{% raw %}", - "setuptools>={% endraw %}{{ setuptools_version }}{% raw %}", # likely related to the kludge for license-files below {% endraw %}{% if create_docs %}{% raw %} "sphinx=={% endraw %}{{ sphinx_version }}{% raw %}",{% endraw %}{% endif %}{% raw %} {% endraw %}{% if is_frozen_executable %}{% raw %} "pyinstaller>={% endraw %}{{ pyinstaller_version }}{% raw %}",{% endraw %}{% endif %}{% raw %} ] From c263fb2f8fc8783e70a814c937ff73b89a96d3d2 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:10:20 +0000 Subject: [PATCH 11/24] actually auth --- .../.github/workflows/publish_to_staging.yaml.jinja | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index a2015b2b..947f0199 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -104,9 +104,11 @@ jobs: {% endraw %}{% endif %}{% raw %} - name: Build package - run: uv build --no-sources + run: | + . .devcontainer/code-artifact-auth.sh + uv build --no-sources {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} - - name: OIDC Auth for CodeArtifact + - name: OIDC Auth for Publishing to CodeArtifact uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} with: role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Staging-{% endraw %}{{ repo_name }}{% raw %} @@ -115,4 +117,6 @@ jobs: {% endraw %}{% endif %}{% raw %} - name: Publish package - run: uv publish --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv publish --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} From fbb762fe7e411a50451444b899de66998286fcad Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:17:05 +0000 Subject: [PATCH 12/24] add publish username --- template/pyproject.toml.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index d6e49997..2cdf7477 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -53,11 +53,13 @@ publish-url = "https://test.pypi.org/legacy/"{% endraw %}{% else %}{% raw %} default = true name = "code-artifact-primary" username = "aws" +publish-username = "aws" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" publish-url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/" [[tool.uv.index]] name = "code-artifact-staging" username = "aws" +publish-username = "aws" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/" publish-url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/"{% endraw %}{% endif %} From 8d7e2eb87809ae899b0e5eb0bc674e8e1047e9ee Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:20:25 +0000 Subject: [PATCH 13/24] auth --- .copier-answers.yml | 2 +- template/.devcontainer/code-artifact-auth.sh.jinja | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index c4814aba..a110634d 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.8-2-gfd4c577 +_commit: v0.0.8-3-g2c970ed _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index c46d3356..58a2d798 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -41,7 +41,9 @@ else set +x export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + export UV_INDEX_CODE_ARTIFACT_PRIMARY_PUBLISH_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + export UV_INDEX_CODE_ARTIFACT_STAGING_PUBLISH_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" set -x fi{% endraw %}{% else %}{% raw %}# Placeholder file not being used by these copier template answers{% endraw %}{% endif %} From 67f16aca6cdfec48353dc9fa2c6674152b8892d2 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:29:49 +0000 Subject: [PATCH 14/24] copier --- .copier-answers.yml | 2 +- template/.devcontainer/code-artifact-auth.sh.jinja | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index a110634d..9b074fba 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.8-3-g2c970ed +_commit: v0.0.8-4-g5211c85 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index 58a2d798..3395ef56 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -6,7 +6,7 @@ if [ -z "$AWS_PROFILE" ] && [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$CODEARTIFACT_A echo "No AWS profile, access key, or auth token found, cannot proceed." exit 1 else - # Only regenerate the token if it doesn't exist or wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) + # Only regenerate the token if it wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) if [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then echo "Fetching CodeArtifact token" if [ -z "$CI" ]; then @@ -30,7 +30,7 @@ else fi set +x - export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ --domain {% endraw %}{{ repo_org_name }}{% raw %} \ --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} \ --region {% endraw %}{{ aws_org_home_region }}{% raw %} \ From 045394b3a8812f1dda609bfc4adc0ef9aaf6785d Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:39:17 +0000 Subject: [PATCH 15/24] env debug --- template/.github/workflows/publish_to_staging.yaml.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index 947f0199..ccf46c0e 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -118,5 +118,6 @@ jobs: - name: Publish package run: | + env {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} uv publish --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} From f5294d7b55088609111c16369085a5b0b0b9a8b9 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:41:18 +0000 Subject: [PATCH 16/24] verbose --- template/.github/workflows/publish_to_staging.yaml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index ccf46c0e..ad781f87 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -81,7 +81,7 @@ jobs: run: uv run pytest --durations=5 build: - needs: [ test ] + # needs: [ test ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} steps: From 4d272375d314bdb9b33f5c79cb158eb19bd228ff Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:42:32 +0000 Subject: [PATCH 17/24] actually verobes --- template/.github/workflows/publish_to_staging.yaml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index ad781f87..afa00b13 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -120,4 +120,4 @@ jobs: run: | env {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} - uv publish --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} + uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} From 4ea4000ff00b3d6631b539dab8835345eccd147a Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:43:04 +0000 Subject: [PATCH 18/24] env duebug --- template/.github/workflows/publish_to_staging.yaml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index afa00b13..3ae0a7c1 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -118,6 +118,6 @@ jobs: - name: Publish package run: | - env {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + env uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} From 732e812df1b5168ca1e5705472b1164bd6791c41 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:48:15 +0000 Subject: [PATCH 19/24] twine envvars --- .copier-answers.yml | 2 +- template/.devcontainer/code-artifact-auth.sh.jinja | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 9b074fba..2057ee3e 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.8-4-g5211c85 +_commit: v0.0.8-5-g77c618a _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index 3395ef56..6766017b 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -38,12 +38,12 @@ else --output text $PROFILE_ARGS) set -x fi - + # uv sometimes uses has better luck when setting the twine env vars + export TWINE_USERNAME=aws set +x export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" - export UV_INDEX_CODE_ARTIFACT_PRIMARY_PUBLISH_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + export TWINE_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" - export UV_INDEX_CODE_ARTIFACT_STAGING_PUBLISH_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" set -x fi{% endraw %}{% else %}{% raw %}# Placeholder file not being used by these copier template answers{% endraw %}{% endif %} From 7361e86dbca2785a5743e3670d5be858ff0ed187 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 18:52:31 +0000 Subject: [PATCH 20/24] more twine --- template/.github/workflows/publish_to_staging.yaml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index 3ae0a7c1..0500e19a 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -120,4 +120,4 @@ jobs: run: | {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} env - uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging{% else %}testpypi{% endif %} + uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging --username aws --password "$TWINE_PASSWORD"{% else %}testpypi{% endif %} From 419336a1daf52d425aa4d83a2a51ac52a2c21d8a Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 19:03:26 +0000 Subject: [PATCH 21/24] real staging --- template/.github/workflows/publish_to_staging.yaml.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index 0500e19a..3d92f9bc 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -81,7 +81,7 @@ jobs: run: uv run pytest --durations=5 build: - # needs: [ test ] + needs: [ test ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} steps: @@ -119,5 +119,4 @@ jobs: - name: Publish package run: | {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} - env uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging --username aws --password "$TWINE_PASSWORD"{% else %}testpypi{% endif %} From 34dd95a0958ae791f07094f51725003b384377f9 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 19:10:14 +0000 Subject: [PATCH 22/24] add basic primary pbulish --- template/.github/workflows/publish.yaml.jinja | 43 ++++++++++++++++--- .../workflows/publish_to_staging.yaml.jinja | 2 +- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 59f8a554..18b38786 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -1,4 +1,4 @@ -{% raw %}name: Publish to PyPI +{% raw %}name: Publish to Production Package Registry on: workflow_dispatch: @@ -22,7 +22,10 @@ jobs: - name: Install latest versions of python packages uses: ./.github/actions/install_deps_uv with: - python-version: {% endraw %}{{ python_version }}{% raw %} + python-version: {% endraw %}{{ python_version }}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 @@ -69,7 +72,10 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Unit test run: uv run pytest --durations=5 @@ -85,7 +91,34 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: {% endraw %}{{ python_version }}{% raw %} + python-version: {% endraw %}{{ python_version }}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Installing any dependencies that uv may need for build (sometimes it likes to install setuptools...even if it's already in the package dependencies) + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/CoreInfraBaseAccess + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} +{% endraw %}{% endif %}{% raw %} - name: Build package - run: uv build --no-sources{% endraw %} + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv build --no-sources + +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + +{% endraw %}{% endif %}{% raw %} + + - name: Publish package + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}testpypi{% endif %} diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja index 3d92f9bc..0c9c30bc 100644 --- a/template/.github/workflows/publish_to_staging.yaml.jinja +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -105,7 +105,7 @@ jobs: {% endraw %}{% endif %}{% raw %} - name: Build package run: | - . .devcontainer/code-artifact-auth.sh +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} uv build --no-sources {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} - name: OIDC Auth for Publishing to CodeArtifact From 7b71ca07eb215bbd95bdcd8b76b74f1aeacb10f4 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 19:10:33 +0000 Subject: [PATCH 23/24] fi pypi --- template/.github/workflows/publish.yaml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 18b38786..39c0ddda 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -121,4 +121,4 @@ jobs: - name: Publish package run: | {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} - uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}testpypi{% endif %} + uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}pypi{% endif %} From 6383fe66d3c52ffe8bf0aaf4f214455df3693302 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 21:04:54 +0000 Subject: [PATCH 24/24] copier tag --- .copier-answers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 2057ee3e..78644df7 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.8-5-g77c618a +_commit: v0.0.9 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: