diff --git a/.copier-answers.yml b/.copier-answers.yml index 283cac3d..78644df7 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.7-44-gea357db +_commit: v0.0.9 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1e863fed..c97f0fc8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,8 +112,8 @@ jobs: - name: install new dependencies env: + # Provide a fake token so it doesn't attempt to access AWS to generate a new one when the script is run if CodeArtifact is set as the registry CODEARTIFACT_AUTH_TOKEN: 'faketoken' - UV_NO_CACHE: 'true' run: | # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI python $RUNNER_TEMP/replace_private_package_registries.py diff --git a/extensions/context.py b/extensions/context.py index 92d01e0a..219c572f 100644 --- a/extensions/context.py +++ b/extensions/context.py @@ -28,6 +28,7 @@ def hook(self, context: dict[Any, Any]) -> dict[Any, Any]: context["ephemeral_pulumi_deploy_version"] = "0.0.2" context["pydantic_version"] = "2.10.6" context["pyinstaller_version"] = "6.12.0" + context["setuptools_version"] = "76.0.0" context["gha_checkout"] = "v4.2.2" context["gha_setup_python"] = "v5.4.0" diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja index c46d3356..6766017b 100644 --- a/template/.devcontainer/code-artifact-auth.sh.jinja +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -6,7 +6,7 @@ if [ -z "$AWS_PROFILE" ] && [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$CODEARTIFACT_A echo "No AWS profile, access key, or auth token found, cannot proceed." exit 1 else - # Only regenerate the token if it doesn't exist or wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) + # Only regenerate the token if it wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) if [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then echo "Fetching CodeArtifact token" if [ -z "$CI" ]; then @@ -30,7 +30,7 @@ else fi set +x - export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ --domain {% endraw %}{{ repo_org_name }}{% raw %} \ --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} \ --region {% endraw %}{{ aws_org_home_region }}{% raw %} \ @@ -38,9 +38,11 @@ else --output text $PROFILE_ARGS) set -x fi - + # uv sometimes uses has better luck when setting the twine env vars + export TWINE_USERNAME=aws set +x export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + export TWINE_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" set -x diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 59f8a554..39c0ddda 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -1,4 +1,4 @@ -{% raw %}name: Publish to PyPI +{% raw %}name: Publish to Production Package Registry on: workflow_dispatch: @@ -22,7 +22,10 @@ jobs: - name: Install latest versions of python packages uses: ./.github/actions/install_deps_uv with: - python-version: {% endraw %}{{ python_version }}{% raw %} + python-version: {% endraw %}{{ python_version }}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 @@ -69,7 +72,10 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Unit test run: uv run pytest --durations=5 @@ -85,7 +91,34 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: {% endraw %}{{ python_version }}{% raw %} + python-version: {% endraw %}{{ python_version }}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Installing any dependencies that uv may need for build (sometimes it likes to install setuptools...even if it's already in the package dependencies) + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/CoreInfraBaseAccess + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} +{% endraw %}{% endif %}{% raw %} - name: Build package - run: uv build --no-sources{% endraw %} + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv build --no-sources + +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + +{% endraw %}{% endif %}{% raw %} + + - name: Publish package + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}pypi{% endif %} diff --git a/template/.github/workflows/publish_to_staging.yaml.jinja b/template/.github/workflows/publish_to_staging.yaml.jinja new file mode 100644 index 00000000..0c9c30bc --- /dev/null +++ b/template/.github/workflows/publish_to_staging.yaml.jinja @@ -0,0 +1,122 @@ +{% raw %}name: Publish to Staging Package Registry + +on: + workflow_dispatch: + +env: + PYTHONUNBUFFERED: True + PRE_COMMIT_HOME: ${{ github.workspace }}/.precommit_cache + +permissions: + id-token: write + contents: write # needed for mutex + +jobs: + lint: + name: Pre-commit + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + + - name: Install latest versions of python packages + uses: ./.github/actions/install_deps_uv + with: + python-version: {% endraw %}{{ python_version }}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + + - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 + if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 + uses: ben-z/gh-action-mutex@{% endraw %}{{ gha_mutex }}{% raw %} + with: + branch: mutex-venv-{% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %} + timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it + + - name: Cache Pre-commit hooks + uses: actions/cache@{% endraw %}{{ gha_cache }}{% raw %} + env: + cache-name: cache-pre-commit-hooks + with: + path: ${{ env.PRE_COMMIT_HOME }} + key: {% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %}-build-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + {% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %}-build-${{ env.cache-name }}- + + - name: Run pre-commit + run: pre-commit run -a + + test: + needs: [ lint ] + strategy: + matrix: + os: + - "{% endraw %}{{ gha_linux_runner }}{% raw %}" +{% endraw %}{% if use_windows_in_ci %} - {{ gha_windows_runner }}{% endif %}{% raw %} + python-version: +{% endraw %}{% for item in python_ci_versions %} + - {{ item }} +{% endfor %}{% raw %} + include: + - os: "{% endraw %}{{ gha_linux_runner }}{% raw %}" + python-version: "{% endraw %}{{ python_ci_versions[0] }}{% raw %}" + JOB_MATCHING_DEV_ENV: true + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + + - name: Install python tooling + uses: ./.github/actions/install_deps_uv + with: + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: "{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}" + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + + - name: Unit test + run: uv run pytest --durations=5 + + build: + needs: [ test ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + + - name: Install python tooling + uses: ./.github/actions/install_deps_uv + with: + python-version: {% endraw %}{{ python_version }}{% raw %}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: "{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}" + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Installing any dependencies that uv may need for build (sometimes it likes to install setuptools...even if it's already in the package dependencies) + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/CoreInfraBaseAccess + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + +{% endraw %}{% endif %}{% raw %} + - name: Build package + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv build --no-sources +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Staging-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + +{% endraw %}{% endif %}{% raw %} + + - name: Publish package + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-staging --username aws --password "$TWINE_PASSWORD"{% else %}testpypi{% endif %} diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 6b0e1f30..2cdf7477 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -21,7 +21,7 @@ dependencies = [ [dependency-groups] dev = [ - # Specific to this template + # Specific to this repository # Managed by upstream template @@ -53,9 +53,13 @@ publish-url = "https://test.pypi.org/legacy/"{% endraw %}{% else %}{% raw %} default = true name = "code-artifact-primary" username = "aws" +publish-username = "aws" url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" +publish-url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/" [[tool.uv.index]] name = "code-artifact-staging" username = "aws" -url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/"{% endraw %}{% endif %} +publish-username = "aws" +url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/" +publish-url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/"{% endraw %}{% endif %}