From a4ba6109961c5ee4d2e7e8a0615a111f8f718312 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 13:28:54 +0000 Subject: [PATCH 01/41] copie --- .copier-answers.yml | 2 +- copier.yml | 8 ++++++++ tests/copier_data/data1.yaml | 1 + tests/copier_data/data2.yaml | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 1d835d8b..3e9240bd 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.6-6-g2b24a38 +_commit: v0.0.7-1-g0894e82 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/copier.yml b/copier.yml index 16cc0165..0529335b 100644 --- a/copier.yml +++ b/copier.yml @@ -26,6 +26,14 @@ python_version: help: What version of Python is used for development? default: "3.12.7" +python_package_registry: + type: str + help: What registry should Python Packgaes be installed from? + choices: + - PyPI + - AWS CodeArtifact + default: PyPI + python_ci_versions: type: str help: What versions should Python run CI on the instantiated template? diff --git a/tests/copier_data/data1.yaml b/tests/copier_data/data1.yaml index a1a2903e..58959d82 100644 --- a/tests/copier_data/data1.yaml +++ b/tests/copier_data/data1.yaml @@ -5,6 +5,7 @@ description: Doing amazing things ssh_port_number: 12345 use_windows_in_ci: false +python_package_registry: PyPI aws_identity_center_id: d-9145c20053 aws_org_home_region: us-west-2 aws_production_account_id: 123456789012 diff --git a/tests/copier_data/data2.yaml b/tests/copier_data/data2.yaml index 09f111ae..7544c0f3 100644 --- a/tests/copier_data/data2.yaml +++ b/tests/copier_data/data2.yaml @@ -5,6 +5,7 @@ description: Doing crazy things! So many things! ssh_port_number: 54321 use_windows_in_ci: true +python_package_registry: AWS CodeArtifact aws_identity_center_id: d-9145c20053 aws_org_home_region: us-east-1 aws_production_account_id: 123456789012 From c1f11162c8f65e1d4590394669769d47f9f129ce Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 13:47:15 +0000 Subject: [PATCH 02/41] codeartifact question --- .copier-answers.yml | 2 +- copier.yml | 19 +++++++++++++++++++ tests/copier_data/data2.yaml | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 3e9240bd..687f459b 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-1-g0894e82 +_commit: v0.0.7-5-ge3a7e35 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/copier.yml b/copier.yml index 0529335b..b73e5092 100644 --- a/copier.yml +++ b/copier.yml @@ -43,6 +43,25 @@ python_ci_versions: - "3.13.2" +aws_identity_center_id: + type: str + help: What's the ID of your Organization's AWS Identity center, e.g. d-9145c20053? + when: "{{ python_package_registry == 'AWS CodeArtifact' }}" + +aws_org_home_region: + type: str + help: What is the home region of the AWS Organization (where all of the central infrastructure is deployed)? + default: us-east-1 + when: "{{ python_package_registry == 'AWS CodeArtifact' }}" + +aws_central_infrastructure_account_id: + type: str + help: What's the ID of your Organization's AWS Account containing Central Infrastructure (e.g. CodeArtifact)? + when: "{{ python_package_registry == 'AWS CodeArtifact' }}" + + + + # Questions specific to this template diff --git a/tests/copier_data/data2.yaml b/tests/copier_data/data2.yaml index 7544c0f3..55d3282f 100644 --- a/tests/copier_data/data2.yaml +++ b/tests/copier_data/data2.yaml @@ -6,6 +6,7 @@ ssh_port_number: 54321 use_windows_in_ci: true python_package_registry: AWS CodeArtifact +aws_central_infrastructure_account_id: 012321432543 aws_identity_center_id: d-9145c20053 aws_org_home_region: us-east-1 aws_production_account_id: 123456789012 From d18d0dbd6bf988ecc5c0f9d05d5536888ecceace Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 14:00:03 +0000 Subject: [PATCH 03/41] copier update --- .copier-answers.yml | 2 +- .devcontainer/create-aws-profile.sh | 11 ++++++----- copier.yml | 6 ++++++ template/.devcontainer/create-aws-profile.sh | 11 ++++++----- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 687f459b..ab4be6a1 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-5-ge3a7e35 +_commit: v0.0.7-6-g7d6bd40 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/create-aws-profile.sh b/.devcontainer/create-aws-profile.sh index 8f93c4ba..85f26be9 100644 --- a/.devcontainer/create-aws-profile.sh +++ b/.devcontainer/create-aws-profile.sh @@ -9,14 +9,15 @@ else LOCALSTACK_ENDPOINT_URL="http://localstack:4566" fi +cat >> ~/.aws/credentials <> ~/.aws/config <> ~/.aws/credentials <> ~/.aws/credentials <> ~/.aws/config <> ~/.aws/credentials < Date: Thu, 13 Mar 2025 14:10:13 +0000 Subject: [PATCH 04/41] dockercompose --- .copier-answers.yml | 2 +- template/.devcontainer/docker-compose.yml.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index ab4be6a1..63f0a4af 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-6-g7d6bd40 +_commit: v0.0.7-7-gbddb576 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/docker-compose.yml.jinja b/template/.devcontainer/docker-compose.yml.jinja index c2f4e1b7..879aafca 100644 --- a/template/.devcontainer/docker-compose.yml.jinja +++ b/template/.devcontainer/docker-compose.yml.jinja @@ -15,7 +15,7 @@ services: - "{% endraw %}{{ ssh_port_number }}{% raw %}:2222" environment: - AWS_PROFILE=localstack - - AWS_DEFAULT_REGION={% endraw %}{{ aws_region_for_stack }}{% raw %} + - AWS_DEFAULT_REGION={% endraw %}{{ aws_region_for_stack if aws_region_for_stack else "us-east-1" }}{% raw %} volumes: From 3e3031ab670f2b06d43894c537aa55aba0ab0f24 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 14:48:12 +0000 Subject: [PATCH 05/41] coiper for profile --- .copier-answers.yml | 2 +- .devcontainer/create-aws-profile.sh | 19 +++++---- template/.devcontainer/create-aws-profile.sh | 23 ---------- .../.devcontainer/create-aws-profile.sh.jinja | 42 +++++++++++++++++++ .../.devcontainer/docker-compose.yml.jinja | 2 +- 5 files changed, 56 insertions(+), 32 deletions(-) delete mode 100644 template/.devcontainer/create-aws-profile.sh create mode 100644 template/.devcontainer/create-aws-profile.sh.jinja diff --git a/.copier-answers.yml b/.copier-answers.yml index 63f0a4af..d410d72a 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-7-gbddb576 +_commit: v0.0.7-14-g733a745 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/create-aws-profile.sh b/.devcontainer/create-aws-profile.sh index 85f26be9..65982f89 100644 --- a/.devcontainer/create-aws-profile.sh +++ b/.devcontainer/create-aws-profile.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh set -ex mkdir -p ~/.aws @@ -9,15 +9,20 @@ else LOCALSTACK_ENDPOINT_URL="http://localstack:4566" fi -cat >> ~/.aws/credentials <> ~/.aws/config <> ~/.aws/credentials <> ~/.aws/credentials <> ~/.aws/config <> ~/.aws/config <> ~/.aws/credentials < Date: Thu, 13 Mar 2025 14:54:36 +0000 Subject: [PATCH 06/41] copier --- .copier-answers.yml | 2 +- template/.devcontainer/create-aws-profile.sh.jinja | 2 +- template/.devcontainer/docker-compose.yml.jinja | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index d410d72a..02f63254 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-14-g733a745 +_commit: v0.0.7-16-g31112c5 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/create-aws-profile.sh.jinja b/template/.devcontainer/create-aws-profile.sh.jinja index ab351e58..2801384c 100644 --- a/template/.devcontainer/create-aws-profile.sh.jinja +++ b/template/.devcontainer/create-aws-profile.sh.jinja @@ -19,7 +19,7 @@ region = {% endraw %}{{ aws_org_home_region }}{% raw %}{% endraw %}{% endif %}{% {% endraw %}{% if aws_identity_center_id is defined and aws_identity_center_id != "" %}{% raw %}[sso-session org] sso_start_url = https://{% endraw %}{{ aws_identity_center_id }}{% raw %}.awsapps.com/start sso_region = {% endraw %}{{ aws_org_home_region }}{% raw %} -sso_registration_scopes = sso:account:access{% raw %}{% endraw %}{% endif %}{% raw %} +sso_registration_scopes = sso:account:access{% endraw %}{% endif %}{% raw %} [profile localstack] region={% endraw %}{{ aws_org_home_region if (aws_org_home_region is defined and aws_org_home_region != "") else "us-east-1" }}{% raw %} diff --git a/template/.devcontainer/docker-compose.yml.jinja b/template/.devcontainer/docker-compose.yml.jinja index a496898f..1edb2121 100644 --- a/template/.devcontainer/docker-compose.yml.jinja +++ b/template/.devcontainer/docker-compose.yml.jinja @@ -15,7 +15,7 @@ services: - "{% endraw %}{{ ssh_port_number }}{% raw %}:2222" environment: - AWS_PROFILE=localstack - - AWS_DEFAULT_REGION={% endraw %}{{ aws_region_for_stack if (aws_org_home_region is defined and aws_org_home_region != "") else "us-east-1" }}{% raw %} + - AWS_DEFAULT_REGION={% endraw %}{{ aws_region_for_stack if (aws_region_for_stack is defined and aws_region_for_stack != "") else "us-east-1" }}{% raw %} volumes: From abb6da7faca23e2db42fef0d549e526dc21bc6ef Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 15:06:59 +0000 Subject: [PATCH 07/41] on create --- .copier-answers.yml | 2 +- template/.devcontainer/on-create-command.sh.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 02f63254..9df926d4 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-16-g31112c5 +_commit: v0.0.7-17-g9d45bba _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/on-create-command.sh.jinja b/template/.devcontainer/on-create-command.sh.jinja index 4e74d3fd..5e501d3a 100644 --- a/template/.devcontainer/on-create-command.sh.jinja +++ b/template/.devcontainer/on-create-command.sh.jinja @@ -7,6 +7,6 @@ git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }} sh .devcontainer/on-create-command-boilerplate.sh -sh .devcontainer/manual-setup-deps.sh +{% endraw %}{% if python_package_registry is not defined or python_package_registry == "PyPi" %}{% raw %}sh .devcontainer/manual-setup-deps.sh{% endraw %}{% endif %}{% raw %} pre-commit install --install-hooks{% endraw %} From 39af82f8336b409eedd6cc9f9e4809bfbb0bcbb6 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 15:09:29 +0000 Subject: [PATCH 08/41] coiper --- .copier-answers.yml | 2 +- template/.devcontainer/on-create-command.sh.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 9df926d4..68aad6c6 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-17-g9d45bba +_commit: v0.0.7-18-g5487b2a _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/on-create-command.sh.jinja b/template/.devcontainer/on-create-command.sh.jinja index 5e501d3a..1d876173 100644 --- a/template/.devcontainer/on-create-command.sh.jinja +++ b/template/.devcontainer/on-create-command.sh.jinja @@ -7,6 +7,6 @@ git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }} sh .devcontainer/on-create-command-boilerplate.sh -{% endraw %}{% if python_package_registry is not defined or python_package_registry == "PyPi" %}{% raw %}sh .devcontainer/manual-setup-deps.sh{% endraw %}{% endif %}{% raw %} +{% endraw %}{% if python_package_registry is not defined or python_package_registry == "PyPI" %}{% raw %}sh .devcontainer/manual-setup-deps.sh{% endraw %}{% endif %}{% raw %} pre-commit install --install-hooks{% endraw %} From 1736bfcf80f2b5fb82947af3216b6eb91c7400f9 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 15:17:58 +0000 Subject: [PATCH 09/41] update --- .copier-answers.yml | 2 +- .devcontainer/on-create-command.sh | 4 ++-- template/.devcontainer/on-create-command.sh.jinja | 4 ++-- template/.devcontainer/post-start-command.sh.jinja | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 68aad6c6..ff152293 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-18-g5487b2a +_commit: v0.0.7-20-gae5920f _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/on-create-command.sh b/.devcontainer/on-create-command.sh index b7d1edc5..15d2255e 100644 --- a/.devcontainer/on-create-command.sh +++ b/.devcontainer/on-create-command.sh @@ -7,6 +7,6 @@ git config --global --add safe.directory /workspaces/copier-python-package-templ sh .devcontainer/on-create-command-boilerplate.sh -sh .devcontainer/manual-setup-deps.sh - pre-commit install --install-hooks + +sh .devcontainer/manual-setup-deps.sh diff --git a/template/.devcontainer/on-create-command.sh.jinja b/template/.devcontainer/on-create-command.sh.jinja index 1d876173..a8b625a4 100644 --- a/template/.devcontainer/on-create-command.sh.jinja +++ b/template/.devcontainer/on-create-command.sh.jinja @@ -7,6 +7,6 @@ git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }} sh .devcontainer/on-create-command-boilerplate.sh -{% endraw %}{% if python_package_registry is not defined or python_package_registry == "PyPI" %}{% raw %}sh .devcontainer/manual-setup-deps.sh{% endraw %}{% endif %}{% raw %} +pre-commit install --install-hooks{% endraw %}{% if python_package_registry is not defined or python_package_registry == "PyPI" %} -pre-commit install --install-hooks{% endraw %} +{% raw %}sh .devcontainer/manual-setup-deps.sh{% endraw %}{% endif %} diff --git a/template/.devcontainer/post-start-command.sh.jinja b/template/.devcontainer/post-start-command.sh.jinja index 2e7f8386..448daec0 100644 --- a/template/.devcontainer/post-start-command.sh.jinja +++ b/template/.devcontainer/post-start-command.sh.jinja @@ -3,4 +3,5 @@ set -ex # For some reason the directory is not setup correctly and causes build of devcontainer to fail since # it doesn't have access to the workspace directory. This can normally be done in post-start-command -git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }} +git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }}{% if python_package_registry is defined and python_package_registry == "PyPI" %}{% raw %} +echo "!!! In order to install dependencies, you must authenticate into the private registry, so run this script to complete the process: sh .devcontainer/manual-setup-deps.sh"{% endraw %}{% endif %} From 92507d2d0f658ef6ee17b0e0a1b95109b24b57ab Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 15:18:52 +0000 Subject: [PATCH 10/41] not equal --- template/.devcontainer/post-start-command.sh.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.devcontainer/post-start-command.sh.jinja b/template/.devcontainer/post-start-command.sh.jinja index 448daec0..91a8953f 100644 --- a/template/.devcontainer/post-start-command.sh.jinja +++ b/template/.devcontainer/post-start-command.sh.jinja @@ -3,5 +3,5 @@ set -ex # For some reason the directory is not setup correctly and causes build of devcontainer to fail since # it doesn't have access to the workspace directory. This can normally be done in post-start-command -git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }}{% if python_package_registry is defined and python_package_registry == "PyPI" %}{% raw %} +git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }}{% if python_package_registry is defined and python_package_registry != "PyPI" %}{% raw %} echo "!!! In order to install dependencies, you must authenticate into the private registry, so run this script to complete the process: sh .devcontainer/manual-setup-deps.sh"{% endraw %}{% endif %} From ff64a8b469b06c1b74cedeb8c14e3b9d2ad4f612 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 15:20:06 +0000 Subject: [PATCH 11/41] copier --- .copier-answers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index ff152293..21bdc89d 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-20-gae5920f +_commit: v0.0.7-22-gf36c22e _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: From b1120ca4c86a442ada21f6b204eda1fc22f0cb4d Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 15:36:50 +0000 Subject: [PATCH 12/41] add to uv --- template/pyproject.toml.jinja | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 65a7c830..1cec0017 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -39,7 +39,7 @@ license-files = [] # kludge until this bug is fixed https://github.com/pypa/setu [tool.uv] package = true -[[tool.uv.index]] +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %}[[tool.uv.index]] name = "pypi" url = "https://pypi.org/simple/" publish-url = "https://upload.pypi.org/legacy/" @@ -47,4 +47,13 @@ publish-url = "https://upload.pypi.org/legacy/" [[tool.uv.index]] name = "testpypi" url = "https://test.pypi.org/simple/" -publish-url = "https://test.pypi.org/legacy/"{% endraw %} +publish-url = "https://test.pypi.org/legacy/"{% endraw %}{% else %}{% raw %} + +[[tool.uv.index]] +name = "code-artifact-primary" +url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" + +[[tool.uv.index]] +name = "code-artifact-staging" +url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/" +{% endraw %}{% endif %} From 23cbc31d7d1baa7c2f43f43efd291a2fd61b5b4c Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:03:32 +0000 Subject: [PATCH 13/41] copier scripts --- .copier-answers.yml | 2 +- .devcontainer/manual-setup-deps.sh | 21 +++++++++++++++++ .../.devcontainer/code-artifact-auth.sh.jinja | 20 ++++++++++++++++ .../.devcontainer/manual-setup-deps.sh.jinja | 23 +++++++++++++++++++ .../.devcontainer/post-start-command.sh.jinja | 2 +- 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 template/.devcontainer/code-artifact-auth.sh.jinja diff --git a/.copier-answers.yml b/.copier-answers.yml index 21bdc89d..c3c776ea 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-22-gf36c22e +_commit: v0.0.7-23-g1d1a72e _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/manual-setup-deps.sh b/.devcontainer/manual-setup-deps.sh index de23afb0..70e9c2db 100644 --- a/.devcontainer/manual-setup-deps.sh +++ b/.devcontainer/manual-setup-deps.sh @@ -1,6 +1,8 @@ #!/usr/bin/env sh # can pass in the full major.minor.patch version of python as an optional argument # can set `--skip-lock` as optional argument to just install dependencies without verifying lock file +# can set `--optionally-lock` to check for a uv.lock file in the project directory and only respect the lock if it already exists (useful for initially instantiating the repository) (mutually exclusive with --skip-lock) + set -ex # Ensure that uv won't use the default system Python @@ -8,20 +10,39 @@ python_version="3.12.7" # Parse arguments skip_lock=false +optionally_lock=false while [ "$#" -gt 0 ]; do case $1 in --skip-lock) skip_lock=true ;; + --optionally-lock) optionally_lock=true ;; *) python_version="${1:-$python_version}" ;; # Take the first non-flag argument as the input esac shift done +# Ensure that --skip-lock and --optionally-lock are mutually exclusive +if [ "$skip_lock" = "true" ] && [ "$optionally_lock" = "true" ]; then + echo "Error: --skip-lock and --optionally-lock cannot be used together." >&2 + exit 1 +fi + export UV_PYTHON="$python_version" export UV_PYTHON_PREFERENCE=only-system SCRIPT_DIR="$(dirname "$0")" PROJECT_ROOT_DIR="$(realpath "$SCRIPT_DIR/..")" +# If optionally_lock is set, decide whether to skip locking based on the presence of uv.lock +if [ "$optionally_lock" = "true" ]; then + if [ ! -f "$PROJECT_ROOT_DIR/uv.lock" ]; then + skip_lock=true + else + skip_lock=false + fi +fi + + + # Ensure that the lock file is in a good state if [ "$skip_lock" = "false" ]; then uv lock --check --directory "$PROJECT_ROOT_DIR" diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja new file mode 100644 index 00000000..e730de80 --- /dev/null +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -0,0 +1,20 @@ +{% if python_package_registry is defined and python_package_registry == "AWS CodeArtifact" %}{% raw %}#!/usr/bin/env bash +set -e + +# If none of these are set we can't possibly continue and should fail so you can fix it +if [ -z "$AWS_PROFILE" ] && [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then + echo "No AWS profile, access key, or auth token found, cannot proceed." + exit 1 +else + # Only regenerate the token if it doesn't exist or wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) + if [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then + echo "Fetching CodeArtifact token" + export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token --domain {% endraw %}{{ repo_org_name }}{% raw %} --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} --region {% endraw %}{{ aws_org_home_region }}{% raw %} --query authorizationToken --output text --profile {% endraw %}{{ core_infra_base_access_profile_name }}{% raw %}) + fi + + export UV_INDEX_CODE_ARTIFACT_PRIMARY_USERNAME=aws + export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + export UV_INDEX_CODE_ARTIFACT_STAGING_USERNAME=aws + export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + +fi{% endraw %}{% endif %} diff --git a/template/.devcontainer/manual-setup-deps.sh.jinja b/template/.devcontainer/manual-setup-deps.sh.jinja index a107af84..bcfdf8b0 100644 --- a/template/.devcontainer/manual-setup-deps.sh.jinja +++ b/template/.devcontainer/manual-setup-deps.sh.jinja @@ -1,6 +1,8 @@ {% raw %}#!/usr/bin/env sh # can pass in the full major.minor.patch version of python as an optional argument # can set `--skip-lock` as optional argument to just install dependencies without verifying lock file +# can set `--optionally-lock` to check for a uv.lock file in the project directory and only respect the lock if it already exists (useful for initially instantiating the repository) (mutually exclusive with --skip-lock) + set -ex # Ensure that uv won't use the default system Python @@ -8,20 +10,41 @@ python_version="{% endraw %}{{ python_version }}{% raw %}" # Parse arguments skip_lock=false +optionally_lock=false while [ "$#" -gt 0 ]; do case $1 in --skip-lock) skip_lock=true ;; + --optionally-lock) optionally_lock=true ;; *) python_version="${1:-$python_version}" ;; # Take the first non-flag argument as the input esac shift done +# Ensure that --skip-lock and --optionally-lock are mutually exclusive +if [ "$skip_lock" = "true" ] && [ "$optionally_lock" = "true" ]; then + echo "Error: --skip-lock and --optionally-lock cannot be used together." >&2 + exit 1 +fi + export UV_PYTHON="$python_version" export UV_PYTHON_PREFERENCE=only-system SCRIPT_DIR="$(dirname "$0")" PROJECT_ROOT_DIR="$(realpath "$SCRIPT_DIR/..")" +# If optionally_lock is set, decide whether to skip locking based on the presence of uv.lock +if [ "$optionally_lock" = "true" ]; then + if [ ! -f "$PROJECT_ROOT_DIR/uv.lock" ]; then + skip_lock=true + else + skip_lock=false + fi +fi + +{% endraw %}{% if python_package_registry is defined and python_package_registry == "AWS CodeArtifact" %}{% raw %} +aws sso login --profile={% endraw %}{{ core_infra_base_access_profile_name }}{% raw %} +. "$SCRIPT_DIR/code-artifact-auth.sh"{% endraw %}{% endif %}{% raw %} + # Ensure that the lock file is in a good state if [ "$skip_lock" = "false" ]; then uv lock --check --directory "$PROJECT_ROOT_DIR" diff --git a/template/.devcontainer/post-start-command.sh.jinja b/template/.devcontainer/post-start-command.sh.jinja index 91a8953f..a8ded17e 100644 --- a/template/.devcontainer/post-start-command.sh.jinja +++ b/template/.devcontainer/post-start-command.sh.jinja @@ -4,4 +4,4 @@ set -ex # For some reason the directory is not setup correctly and causes build of devcontainer to fail since # it doesn't have access to the workspace directory. This can normally be done in post-start-command git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }}{% if python_package_registry is defined and python_package_registry != "PyPI" %}{% raw %} -echo "!!! In order to install dependencies, you must authenticate into the private registry, so run this script to complete the process: sh .devcontainer/manual-setup-deps.sh"{% endraw %}{% endif %} +echo "!!! In order to install dependencies, you must authenticate into the private registry, so run this script to complete the process: source .devcontainer/manual-setup-deps.sh"{% endraw %}{% endif %} From 3709b2dca2ba782220abc45636a776e98a772f99 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:04:20 +0000 Subject: [PATCH 14/41] add default --- template/pyproject.toml.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 1cec0017..e1abc88d 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -52,6 +52,7 @@ publish-url = "https://test.pypi.org/legacy/"{% endraw %}{% else %}{% raw %} [[tool.uv.index]] name = "code-artifact-primary" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" +default = true [[tool.uv.index]] name = "code-artifact-staging" From 9b2f83334e59202177344f940645df464343fd10 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:08:49 +0000 Subject: [PATCH 15/41] whitespace --- template/pyproject.toml.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index e1abc88d..94ff9318 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -56,5 +56,4 @@ default = true [[tool.uv.index]] name = "code-artifact-staging" -url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/" -{% endraw %}{% endif %} +url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/"{% endraw %}{% endif %} From adf654485257bb92840db211b32849fe1b53b1e6 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:14:45 +0000 Subject: [PATCH 16/41] update test data --- tests/copier_data/data3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/copier_data/data3.yaml b/tests/copier_data/data3.yaml index fedea476..65f2df82 100644 --- a/tests/copier_data/data3.yaml +++ b/tests/copier_data/data3.yaml @@ -5,6 +5,7 @@ description: Doing amazing things ssh_port_number: 12345 use_windows_in_ci: false +python_package_registry: PyPI aws_identity_center_id: d-9145c20053 aws_org_home_region: us-west-2 aws_production_account_id: 123456789012 From 73450b659e403cd60949c9c21d536a97f864ceac Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:17:20 +0000 Subject: [PATCH 17/41] test data --- .copier-answers.yml | 2 +- tests/copier_data/data2.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index c3c776ea..ccc547ad 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-23-g1d1a72e +_commit: v0.0.7-24-gff5c8d4 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/tests/copier_data/data2.yaml b/tests/copier_data/data2.yaml index 55d3282f..ae4507ad 100644 --- a/tests/copier_data/data2.yaml +++ b/tests/copier_data/data2.yaml @@ -7,6 +7,7 @@ use_windows_in_ci: true python_package_registry: AWS CodeArtifact aws_central_infrastructure_account_id: 012321432543 +core_infra_base_access_profile_name: MyAccessRole aws_identity_center_id: d-9145c20053 aws_org_home_region: us-east-1 aws_production_account_id: 123456789012 From 551a1600c6cc1de2e823abb9d9b31caf6b1f047d Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:18:05 +0000 Subject: [PATCH 18/41] optionally lock --- .copier-answers.yml | 2 +- .devcontainer/on-create-command.sh | 2 +- template/.devcontainer/on-create-command.sh.jinja | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index ccc547ad..0bb9ac1a 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-24-gff5c8d4 +_commit: v0.0.7-25-gdb24499 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/on-create-command.sh b/.devcontainer/on-create-command.sh index 15d2255e..4127cc28 100644 --- a/.devcontainer/on-create-command.sh +++ b/.devcontainer/on-create-command.sh @@ -9,4 +9,4 @@ sh .devcontainer/on-create-command-boilerplate.sh pre-commit install --install-hooks -sh .devcontainer/manual-setup-deps.sh +sh .devcontainer/manual-setup-deps.sh --optionally-lock diff --git a/template/.devcontainer/on-create-command.sh.jinja b/template/.devcontainer/on-create-command.sh.jinja index a8b625a4..2f70a5bb 100644 --- a/template/.devcontainer/on-create-command.sh.jinja +++ b/template/.devcontainer/on-create-command.sh.jinja @@ -9,4 +9,4 @@ sh .devcontainer/on-create-command-boilerplate.sh pre-commit install --install-hooks{% endraw %}{% if python_package_registry is not defined or python_package_registry == "PyPI" %} -{% raw %}sh .devcontainer/manual-setup-deps.sh{% endraw %}{% endif %} +{% raw %}sh .devcontainer/manual-setup-deps.sh --optionally-lock{% endraw %}{% endif %} From f25b5af88819d977a8179cc2155347308870104f Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:24:57 +0000 Subject: [PATCH 19/41] fix CI --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 2 ++ template/.devcontainer/code-artifact-auth.sh.jinja | 2 ++ template/.devcontainer/manual-setup-deps.sh.jinja | 1 - 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 0bb9ac1a..56d82ef9 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-25-gdb24499 +_commit: v0.0.7-26-ge3aa2c8 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eff6c83f..53772419 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,6 +107,8 @@ jobs: - name: install new dependencies + env: + CODEARTIFACT_AUTH_TOKEN=faketoken run: | sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index e730de80..95b482d6 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -9,6 +9,8 @@ else # Only regenerate the token if it doesn't exist or wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) if [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then echo "Fetching CodeArtifact token" + # TODO: only re-login if the sso credentials have expired + aws sso login --profile={% endraw %}{{ core_infra_base_access_profile_name }}{% raw %} export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token --domain {% endraw %}{{ repo_org_name }}{% raw %} --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} --region {% endraw %}{{ aws_org_home_region }}{% raw %} --query authorizationToken --output text --profile {% endraw %}{{ core_infra_base_access_profile_name }}{% raw %}) fi diff --git a/template/.devcontainer/manual-setup-deps.sh.jinja b/template/.devcontainer/manual-setup-deps.sh.jinja index bcfdf8b0..09a5ad71 100644 --- a/template/.devcontainer/manual-setup-deps.sh.jinja +++ b/template/.devcontainer/manual-setup-deps.sh.jinja @@ -42,7 +42,6 @@ if [ "$optionally_lock" = "true" ]; then fi {% endraw %}{% if python_package_registry is defined and python_package_registry == "AWS CodeArtifact" %}{% raw %} -aws sso login --profile={% endraw %}{{ core_infra_base_access_profile_name }}{% raw %} . "$SCRIPT_DIR/code-artifact-auth.sh"{% endraw %}{% endif %}{% raw %} # Ensure that the lock file is in a good state From 17aefca8ed60600730a2ce8bdfcf8a16c3aee250 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 18:33:27 +0000 Subject: [PATCH 20/41] CI syntax --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 56d82ef9..8b4a1538 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-26-ge3aa2c8 +_commit: v0.0.7-27-g5f97e2a _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53772419..7c9dbe66 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -108,7 +108,7 @@ jobs: - name: install new dependencies env: - CODEARTIFACT_AUTH_TOKEN=faketoken + CODEARTIFACT_AUTH_TOKEN: 'faketoken' run: | sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them From 5c4ee913a2797b1262595b7fb45edf7b26e2a53d Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 20:46:07 +0000 Subject: [PATCH 21/41] update auth script --- .copier-answers.yml | 2 +- .../.devcontainer/code-artifact-auth.sh.jinja | 31 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 8b4a1538..cf870931 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-27-g5f97e2a +_commit: v0.0.7-29-g02bf43f _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index 95b482d6..a2fd6388 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -9,9 +9,32 @@ else # Only regenerate the token if it doesn't exist or wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) if [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then echo "Fetching CodeArtifact token" - # TODO: only re-login if the sso credentials have expired - aws sso login --profile={% endraw %}{{ core_infra_base_access_profile_name }}{% raw %} - export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token --domain {% endraw %}{{ repo_org_name }}{% raw %} --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} --region {% endraw %}{{ aws_org_home_region }}{% raw %} --query authorizationToken --output text --profile {% endraw %}{{ core_infra_base_access_profile_name }}{% raw %}) + if [ -z "$CI" ]; then + PROFILE_ARGS="--profile={% endraw %}{{ core_infra_base_access_profile_name }}{% raw %}" + else + PROFILE_ARGS="" + fi + + # Check if AWS credentials are valid by trying to retrieve the caller identity. + # If the ARN is not returned, assume credentials are expired or not set correctly. + caller_identity=$(aws sts get-caller-identity --region={% endraw %}{{ aws_org_home_region }}{% raw %} $PROFILE_ARGS --query Arn --output text 2>/dev/null || echo "") + if [ -z "$caller_identity" ]; then + if [ -n "$CI" ]; then + echo "Error: In CI environment, aws sso login should never be called...something is wrong with this script or your workflow...perhaps you did not OIDC Auth yet in CI?" + exit 1 + fi + echo "SSO credentials not found or expired, logging in..." + aws sso login $PROFILE_ARGS + else + echo "Using existing AWS credentials: $caller_identity" + fi + + export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + --domain {% endraw %}{{ repo_org_name }}{% raw %} \ + --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} \ + --region {% endraw %}{{ aws_org_home_region }}{% raw %} \ + --query authorizationToken \ + --output text $PROFILE_ARGS) fi export UV_INDEX_CODE_ARTIFACT_PRIMARY_USERNAME=aws @@ -19,4 +42,4 @@ else export UV_INDEX_CODE_ARTIFACT_STAGING_USERNAME=aws export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" -fi{% endraw %}{% endif %} +fi{% endraw %}{% else %}{% raw %}# Placeholder file not being used by these copier template answers{% endraw %}{% endif %} From de730034311834b0bd72d85a4fda3e904d711db2 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 20:56:20 +0000 Subject: [PATCH 22/41] update uv action --- .copier-answers.yml | 2 +- .github/actions/install_deps_uv/action.yml | 23 +++++++++++++++++++ .../actions/install_deps_uv/action.yml | 23 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index cf870931..52481aa3 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-29-g02bf43f +_commit: v0.0.7-30-gc89c13a _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/actions/install_deps_uv/action.yml b/.github/actions/install_deps_uv/action.yml index 55de308c..b540c759 100644 --- a/.github/actions/install_deps_uv/action.yml +++ b/.github/actions/install_deps_uv/action.yml @@ -14,6 +14,19 @@ inputs: description: What's the relative path to the project? required: false default: ./ + code-artifact-auth-role-name: + type: string + description: What's the role name to use for CodeArtifact authentication? + required: false + default: no-code-artifact + code-artifact-auth-role-account-id: + type: string + description: What's the AWS Account ID that the role is in? + required: false + code-artifact-auth-region: + type: string + description: What region should the role use? + required: false runs: @@ -41,12 +54,22 @@ runs: run: .github/actions/install_deps_uv/install-ci-tooling.ps1 ${{ env.PYTHON_VERSION }} shell: pwsh + - name: OIDC Auth for CodeArtifact + if: ${{ inputs.code-artifact-auth-role-name != "no-code-artifact" }} + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }} + aws-region: ${{ inputs.code-artifact-auth-region }} + - name: Install Dependencies (Linux) if: ${{ inputs.uv-sync && runner.os == 'Linux' }} run: | sh .devcontainer/manual-setup-deps.sh ${{ env.PYTHON_VERSION }} shell: bash + + + - name: Install Dependencies (Windows) if: ${{ inputs.uv-sync && runner.os == 'Windows' }} run: .github/actions/install_deps_uv/manual-setup-deps.ps1 ${{ env.PYTHON_VERSION }} diff --git a/template/.github/actions/install_deps_uv/action.yml b/template/.github/actions/install_deps_uv/action.yml index 55de308c..b540c759 100644 --- a/template/.github/actions/install_deps_uv/action.yml +++ b/template/.github/actions/install_deps_uv/action.yml @@ -14,6 +14,19 @@ inputs: description: What's the relative path to the project? required: false default: ./ + code-artifact-auth-role-name: + type: string + description: What's the role name to use for CodeArtifact authentication? + required: false + default: no-code-artifact + code-artifact-auth-role-account-id: + type: string + description: What's the AWS Account ID that the role is in? + required: false + code-artifact-auth-region: + type: string + description: What region should the role use? + required: false runs: @@ -41,12 +54,22 @@ runs: run: .github/actions/install_deps_uv/install-ci-tooling.ps1 ${{ env.PYTHON_VERSION }} shell: pwsh + - name: OIDC Auth for CodeArtifact + if: ${{ inputs.code-artifact-auth-role-name != "no-code-artifact" }} + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }} + aws-region: ${{ inputs.code-artifact-auth-region }} + - name: Install Dependencies (Linux) if: ${{ inputs.uv-sync && runner.os == 'Linux' }} run: | sh .devcontainer/manual-setup-deps.sh ${{ env.PYTHON_VERSION }} shell: bash + + + - name: Install Dependencies (Windows) if: ${{ inputs.uv-sync && runner.os == 'Windows' }} run: .github/actions/install_deps_uv/manual-setup-deps.ps1 ${{ env.PYTHON_VERSION }} From d18606e26113ec14fb88ede6896e33ec2fa009f5 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:03:00 +0000 Subject: [PATCH 23/41] auth in CI --- template/.github/workflows/ci.yaml.jinja | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/template/.github/workflows/ci.yaml.jinja b/template/.github/workflows/ci.yaml.jinja index 7483e1c0..efa74d07 100644 --- a/template/.github/workflows/ci.yaml.jinja +++ b/template/.github/workflows/ci.yaml.jinja @@ -25,7 +25,10 @@ jobs: - name: Install latest versions of python packages uses: ./.github/actions/install_deps_uv with: - python-version: {% endraw %}{{ python_version }}{% raw %} + python-version: {% endraw %}{{ python_version }}{% raw %}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 From ebb94f3ae59653f663820542f51dbd1bca544b1a Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:03:28 +0000 Subject: [PATCH 24/41] more CI --- template/.github/workflows/ci.yaml.jinja | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/template/.github/workflows/ci.yaml.jinja b/template/.github/workflows/ci.yaml.jinja index efa74d07..56f39e4c 100644 --- a/template/.github/workflows/ci.yaml.jinja +++ b/template/.github/workflows/ci.yaml.jinja @@ -75,7 +75,10 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Unit test run: uv run pytest tests/unit --cov-report=xml --durations=5 @@ -112,7 +115,11 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + - name: Build executable run: uv run pyinstaller pyinstaller.spec --log-level=DEBUG - name: Upload executable artifact @@ -142,7 +149,10 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Build docs working-directory: ./docs From 5d9f2b141215699b3938e959891e1df72b5ab4e8 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:05:16 +0000 Subject: [PATCH 25/41] copier --- .copier-answers.yml | 2 +- .github/actions/install_deps_uv/action.yml | 2 +- template/.github/actions/install_deps_uv/action.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 52481aa3..64dfd47e 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-30-gc89c13a +_commit: v0.0.7-31-gf8e05d9 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/actions/install_deps_uv/action.yml b/.github/actions/install_deps_uv/action.yml index b540c759..017b8d05 100644 --- a/.github/actions/install_deps_uv/action.yml +++ b/.github/actions/install_deps_uv/action.yml @@ -55,7 +55,7 @@ runs: shell: pwsh - name: OIDC Auth for CodeArtifact - if: ${{ inputs.code-artifact-auth-role-name != "no-code-artifact" }} + if: ${{ inputs.code-artifact-auth-role-name != 'no-code-artifact' }} uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }} diff --git a/template/.github/actions/install_deps_uv/action.yml b/template/.github/actions/install_deps_uv/action.yml index b540c759..017b8d05 100644 --- a/template/.github/actions/install_deps_uv/action.yml +++ b/template/.github/actions/install_deps_uv/action.yml @@ -55,7 +55,7 @@ runs: shell: pwsh - name: OIDC Auth for CodeArtifact - if: ${{ inputs.code-artifact-auth-role-name != "no-code-artifact" }} + if: ${{ inputs.code-artifact-auth-role-name != 'no-code-artifact' }} uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }} From 61a10139bd9ce8ffcafd4c63f16bd3787ea505c2 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:06:23 +0000 Subject: [PATCH 26/41] region --- template/.github/workflows/ci.yaml.jinja | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/template/.github/workflows/ci.yaml.jinja b/template/.github/workflows/ci.yaml.jinja index 56f39e4c..53d05bf5 100644 --- a/template/.github/workflows/ci.yaml.jinja +++ b/template/.github/workflows/ci.yaml.jinja @@ -28,7 +28,7 @@ jobs: python-version: {% endraw %}{{ python_version }}{% raw %}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} code-artifact-auth-role-name: CoreInfraBaseAccess code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} - code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 @@ -78,7 +78,7 @@ jobs: python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} code-artifact-auth-role-name: CoreInfraBaseAccess code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} - code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Unit test run: uv run pytest tests/unit --cov-report=xml --durations=5 @@ -118,7 +118,7 @@ jobs: python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} code-artifact-auth-role-name: CoreInfraBaseAccess code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} - code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Build executable run: uv run pyinstaller pyinstaller.spec --log-level=DEBUG @@ -152,7 +152,7 @@ jobs: python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} code-artifact-auth-role-name: CoreInfraBaseAccess code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} - code-artifact-auth-role-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Build docs working-directory: ./docs From 0f7006107ef058f7c7f60eaaed13522deba05393 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:07:18 +0000 Subject: [PATCH 27/41] quotes --- template/.github/workflows/ci.yaml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.github/workflows/ci.yaml.jinja b/template/.github/workflows/ci.yaml.jinja index 53d05bf5..66081e5e 100644 --- a/template/.github/workflows/ci.yaml.jinja +++ b/template/.github/workflows/ci.yaml.jinja @@ -151,7 +151,7 @@ jobs: with: python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} code-artifact-auth-role-name: CoreInfraBaseAccess - code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-role-account-id: "{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}" code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Build docs From 36c317b845e560a3c1b48c689d1652458213fcc3 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:20:08 +0000 Subject: [PATCH 28/41] mas kscert --- .copier-answers.yml | 2 +- template/.devcontainer/code-artifact-auth.sh.jinja | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 64dfd47e..b00676f1 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-31-gf8e05d9 +_commit: v0.0.7-32-gbd0b25b _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index a2fd6388..02feb791 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -38,8 +38,10 @@ else fi export UV_INDEX_CODE_ARTIFACT_PRIMARY_USERNAME=aws + echo "::add-mask::$UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD" # ensure this doesn't show up in the CI logs export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" export UV_INDEX_CODE_ARTIFACT_STAGING_USERNAME=aws + echo "::add-mask::$UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD" # ensure this doesn't show up in the CI logs export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" fi{% endraw %}{% else %}{% raw %}# Placeholder file not being used by these copier template answers{% endraw %}{% endif %} From 54327d4495c5a8d168a86f06aeb342b73b103fc2 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:24:02 +0000 Subject: [PATCH 29/41] coiper --- .copier-answers.yml | 2 +- template/.devcontainer/code-artifact-auth.sh.jinja | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index b00676f1..5d1f3274 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-32-gbd0b25b +_commit: v0.0.7-33-geffce09 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index 02feb791..2c1f810a 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -1,5 +1,5 @@ {% if python_package_registry is defined and python_package_registry == "AWS CodeArtifact" %}{% raw %}#!/usr/bin/env bash -set -e +set -ex # If none of these are set we can't possibly continue and should fail so you can fix it if [ -z "$AWS_PROFILE" ] && [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then @@ -29,19 +29,23 @@ else echo "Using existing AWS credentials: $caller_identity" fi + set +x export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ --domain {% endraw %}{{ repo_org_name }}{% raw %} \ --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} \ --region {% endraw %}{{ aws_org_home_region }}{% raw %} \ --query authorizationToken \ --output text $PROFILE_ARGS) + set -x fi export UV_INDEX_CODE_ARTIFACT_PRIMARY_USERNAME=aws - echo "::add-mask::$UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD" # ensure this doesn't show up in the CI logs + set +x export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + set -x export UV_INDEX_CODE_ARTIFACT_STAGING_USERNAME=aws - echo "::add-mask::$UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD" # ensure this doesn't show up in the CI logs + set +x export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + set -x fi{% endraw %}{% else %}{% raw %}# Placeholder file not being used by these copier template answers{% endraw %}{% endif %} From ecdad6b4583aa895159e95aa0c7284ad8be9f4ad Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:41:26 +0000 Subject: [PATCH 30/41] rearrange default --- template/pyproject.toml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 94ff9318..688f2b23 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -50,9 +50,9 @@ url = "https://test.pypi.org/simple/" publish-url = "https://test.pypi.org/legacy/"{% endraw %}{% else %}{% raw %} [[tool.uv.index]] +default = true name = "code-artifact-primary" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" -default = true [[tool.uv.index]] name = "code-artifact-staging" From 673bb162b721414c8f23b19fd6df00632fa2229c Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:45:03 +0000 Subject: [PATCH 31/41] copier --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 5d1f3274..082526ab 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-33-geffce09 +_commit: v0.0.7-34-g4b41354 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7c9dbe66..8f2a59df 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -110,6 +110,8 @@ jobs: env: CODEARTIFACT_AUTH_TOKEN: 'faketoken' run: | + # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI + find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{/^default = true$/d}' {} + sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them git add . From 6244893a729fa8cddb962343522916b016e9674b Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Thu, 13 Mar 2025 21:50:32 +0000 Subject: [PATCH 32/41] debug --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 082526ab..228baa56 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-34-g4b41354 +_commit: v0.0.7-35-g6ca46a6 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8f2a59df..1a005faf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,6 +112,7 @@ jobs: run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{/^default = true$/d}' {} + + cat pyproject.toml sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them git add . From 82d35d0fc3f2200e0d6b2b1d171a6c292c32c547 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 10:50:49 +0000 Subject: [PATCH 33/41] ci --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 228baa56..e9f61cfc 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-35-g6ca46a6 +_commit: v0.0.7-36-gb375850 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1a005faf..1bc4d8e1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -111,7 +111,7 @@ jobs: CODEARTIFACT_AUTH_TOKEN: 'faketoken' run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI - find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{/^default = true$/d}' {} + + find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{s/^\(default = true\)$/\1\nname = "pypi"\nurl = "https:\/\/pypi.org\/simple\/"\n[[tool.uv.index]]/}' {} + cat pyproject.toml sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them From c6e13519c052e76b26f58ff1d638d94bbdd2ad6d Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 10:53:49 +0000 Subject: [PATCH 34/41] More debug --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index e9f61cfc..1a946508 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-36-gb375850 +_commit: v0.0.7-37-gbc06055 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1bc4d8e1..a76c254e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,6 +112,7 @@ jobs: run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{s/^\(default = true\)$/\1\nname = "pypi"\nurl = "https:\/\/pypi.org\/simple\/"\n[[tool.uv.index]]/}' {} + + ls uv.* -a cat pyproject.toml sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them From cf80c83b22aa868d280fccf4911e61eb6c3fbde0 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 10:55:46 +0000 Subject: [PATCH 35/41] more ci --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 1a946508..8f62f7cb 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-37-gbc06055 +_commit: v0.0.7-38-gaf08c6a _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a76c254e..dc20be09 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,6 +109,7 @@ jobs: - name: install new dependencies env: CODEARTIFACT_AUTH_TOKEN: 'faketoken' + UV_NO_CACHE: 'true' run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{s/^\(default = true\)$/\1\nname = "pypi"\nurl = "https:\/\/pypi.org\/simple\/"\n[[tool.uv.index]]/}' {} + From c9af6ebb6b6e5412628d1d2d82dd318a8111e0ee Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 10:56:36 +0000 Subject: [PATCH 36/41] no ls --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 8f62f7cb..27df687d 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-38-gaf08c6a +_commit: v0.0.7-39-g1efb56b _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dc20be09..32d1ba0f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -113,7 +113,6 @@ jobs: run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{s/^\(default = true\)$/\1\nname = "pypi"\nurl = "https:\/\/pypi.org\/simple\/"\n[[tool.uv.index]]/}' {} + - ls uv.* -a cat pyproject.toml sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them From 2f57b39439400a15da7ed61f019a2c82c16fb078 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 10:58:51 +0000 Subject: [PATCH 37/41] no failfast --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 27df687d..7392e004 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-39-g1efb56b +_commit: v0.0.7-40-gba2c9c5 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 32d1ba0f..dd42174a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,6 +57,7 @@ jobs: lint-matrix: needs: [ pre-commit ] strategy: + fail-fast: false matrix: os: - "ubuntu-24.04" From e0eeebd42ea7ee6fc345ff941f36c1ce5cbefd15 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 13:08:48 +0000 Subject: [PATCH 38/41] python --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 2 +- .../replace_private_package_registries.py | 57 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/replace_private_package_registries.py diff --git a/.copier-answers.yml b/.copier-answers.yml index 7392e004..e551bc58 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-40-gba2c9c5 +_commit: v0.0.7-41-ge3d7a12 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dd42174a..61b5b72e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -113,7 +113,7 @@ jobs: UV_NO_CACHE: 'true' run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI - find . -maxdepth 3 -type f -name "pyproject.toml" -exec sed -i '/^\[\[tool\.uv\.index\]\]/, /^\[\[/{s/^\(default = true\)$/\1\nname = "pypi"\nurl = "https:\/\/pypi.org\/simple\/"\n[[tool.uv.index]]/}' {} + + python .github/workflows/replace_private_package_registries.py cat pyproject.toml sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them diff --git a/.github/workflows/replace_private_package_registries.py b/.github/workflows/replace_private_package_registries.py new file mode 100644 index 00000000..f17947b0 --- /dev/null +++ b/.github/workflows/replace_private_package_registries.py @@ -0,0 +1,57 @@ +"""Update any project files that point to a private package registry to use public ones. + +Since the CI pipelines for testing these copier templates don't have access to private registries, we can't test installing from them as part of CI. + +Seems minimal risk, since the only problem we'd be missing is if the pyproject.toml (or similar config files) had syntax errors that would have been +caught by pre-commit. +""" + +import re +from pathlib import Path + + +def process_file(file_path: Path): + # Read the entire file content + content = file_path.read_text() + + # Regex to match a block starting with [[tool.uv.index]] + # until the next block header (a line starting with [[) or the end of the file. + pattern = re.compile(r"(\[\[tool\.uv\.index\]\].*?)(?=\n\[\[|$)", re.DOTALL) + + # Find all uv.index blocks. + blocks = pattern.findall(content) + + # Check if any block contains "default = true" + if not any("default = true" in block for block in blocks): + print(f"No changes in: {file_path}") + return + + # If at least one block contains "default = true", remove all uv.index blocks. + new_content = pattern.sub("", content) + + # Ensure file ends with a newline before appending the new block. + if not new_content.endswith("\n"): + new_content += "\n" + + # Append the new block. + new_block = '[[tool.uv.index]]\nname = "pypi"\nurl = "https://pypi.org/simple/"\n' + new_content += new_block + + # Write the updated content back to the file. + _ = file_path.write_text(new_content) + print(f"Updated file: {file_path}") + + +def main(): + base_dir = Path(".") + # Use rglob to find all pyproject.toml files recursively. + for file_path in base_dir.rglob("pyproject.toml"): + # Check if the file is at most two levels deep. + # The relative path's parts count should be <= 3 (e.g. "pyproject.toml" is 1 part, + # "subdir/pyproject.toml" is 2 parts, and "subdir/subsubdir/pyproject.toml" is 3 parts). + if len(file_path.relative_to(base_dir).parts) <= 3: + process_file(file_path) + + +if __name__ == "__main__": + main() From f26a314525ae97d2336cce2ddee09d12cb2551fa Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 13:15:25 +0000 Subject: [PATCH 39/41] move file --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index e551bc58..e04a06a9 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-41-ge3d7a12 +_commit: v0.0.7-43-g83c9863 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 61b5b72e..1e863fed 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,7 +57,6 @@ jobs: lint-matrix: needs: [ pre-commit ] strategy: - fail-fast: false matrix: os: - "ubuntu-24.04" @@ -78,6 +77,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4.2.2 + - name: Move python script that replaces private package registry information to temp folder so it doesn't get deleted + run: | + mv .github/workflows/replace_private_package_registries.py $RUNNER_TEMP + - name: Install python tooling uses: ./.github/actions/install_deps_uv with: @@ -113,8 +116,7 @@ jobs: UV_NO_CACHE: 'true' run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI - python .github/workflows/replace_private_package_registries.py - cat pyproject.toml + python $RUNNER_TEMP/replace_private_package_registries.py sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them git add . From 43ae0b9ebde8f74359b913a996317564d1291cd1 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 13:17:31 +0000 Subject: [PATCH 40/41] username --- template/pyproject.toml.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 688f2b23..6b0e1f30 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -52,8 +52,10 @@ publish-url = "https://test.pypi.org/legacy/"{% endraw %}{% else %}{% raw %} [[tool.uv.index]] default = true name = "code-artifact-primary" +username = "aws" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" [[tool.uv.index]] name = "code-artifact-staging" +username = "aws" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/"{% endraw %}{% endif %} From 6e729bde71cb898b151ccdf5545ada8a6c0dc312 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 14 Mar 2025 13:18:53 +0000 Subject: [PATCH 41/41] update script --- .copier-answers.yml | 2 +- template/.devcontainer/code-artifact-auth.sh.jinja | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index e04a06a9..283cac3d 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-43-g83c9863 +_commit: v0.0.7-44-gea357db _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index 2c1f810a..c46d3356 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -39,12 +39,8 @@ else set -x fi - export UV_INDEX_CODE_ARTIFACT_PRIMARY_USERNAME=aws set +x export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" - set -x - export UV_INDEX_CODE_ARTIFACT_STAGING_USERNAME=aws - set +x export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" set -x