diff --git a/.copier-answers.yml b/.copier-answers.yml index 1d835d8b..283cac3d 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.6-6-g2b24a38 +_commit: v0.0.7-44-gea357db _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/create-aws-profile.sh b/.devcontainer/create-aws-profile.sh index 8f93c4ba..65982f89 100644 --- a/.devcontainer/create-aws-profile.sh +++ b/.devcontainer/create-aws-profile.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh set -ex mkdir -p ~/.aws @@ -10,10 +10,16 @@ else fi cat >> ~/.aws/config <> ~/.aws/credentials <&2 + exit 1 +fi + export UV_PYTHON="$python_version" export UV_PYTHON_PREFERENCE=only-system SCRIPT_DIR="$(dirname "$0")" PROJECT_ROOT_DIR="$(realpath "$SCRIPT_DIR/..")" +# If optionally_lock is set, decide whether to skip locking based on the presence of uv.lock +if [ "$optionally_lock" = "true" ]; then + if [ ! -f "$PROJECT_ROOT_DIR/uv.lock" ]; then + skip_lock=true + else + skip_lock=false + fi +fi + + + # Ensure that the lock file is in a good state if [ "$skip_lock" = "false" ]; then uv lock --check --directory "$PROJECT_ROOT_DIR" diff --git a/.devcontainer/on-create-command.sh b/.devcontainer/on-create-command.sh index b7d1edc5..4127cc28 100644 --- a/.devcontainer/on-create-command.sh +++ b/.devcontainer/on-create-command.sh @@ -7,6 +7,6 @@ git config --global --add safe.directory /workspaces/copier-python-package-templ sh .devcontainer/on-create-command-boilerplate.sh -sh .devcontainer/manual-setup-deps.sh - pre-commit install --install-hooks + +sh .devcontainer/manual-setup-deps.sh --optionally-lock diff --git a/.github/actions/install_deps_uv/action.yml b/.github/actions/install_deps_uv/action.yml index 55de308c..017b8d05 100644 --- a/.github/actions/install_deps_uv/action.yml +++ b/.github/actions/install_deps_uv/action.yml @@ -14,6 +14,19 @@ inputs: description: What's the relative path to the project? required: false default: ./ + code-artifact-auth-role-name: + type: string + description: What's the role name to use for CodeArtifact authentication? + required: false + default: no-code-artifact + code-artifact-auth-role-account-id: + type: string + description: What's the AWS Account ID that the role is in? + required: false + code-artifact-auth-region: + type: string + description: What region should the role use? + required: false runs: @@ -41,12 +54,22 @@ runs: run: .github/actions/install_deps_uv/install-ci-tooling.ps1 ${{ env.PYTHON_VERSION }} shell: pwsh + - name: OIDC Auth for CodeArtifact + if: ${{ inputs.code-artifact-auth-role-name != 'no-code-artifact' }} + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }} + aws-region: ${{ inputs.code-artifact-auth-region }} + - name: Install Dependencies (Linux) if: ${{ inputs.uv-sync && runner.os == 'Linux' }} run: | sh .devcontainer/manual-setup-deps.sh ${{ env.PYTHON_VERSION }} shell: bash + + + - name: Install Dependencies (Windows) if: ${{ inputs.uv-sync && runner.os == 'Windows' }} run: .github/actions/install_deps_uv/manual-setup-deps.ps1 ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eff6c83f..1e863fed 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,6 +77,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4.2.2 + - name: Move python script that replaces private package registry information to temp folder so it doesn't get deleted + run: | + mv .github/workflows/replace_private_package_registries.py $RUNNER_TEMP + - name: Install python tooling uses: ./.github/actions/install_deps_uv with: @@ -107,7 +111,12 @@ jobs: - name: install new dependencies + env: + CODEARTIFACT_AUTH_TOKEN: 'faketoken' + UV_NO_CACHE: 'true' run: | + # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI + python $RUNNER_TEMP/replace_private_package_registries.py sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock # Add everything to git so that pre-commit recognizes the files and runs on them git add . diff --git a/.github/workflows/replace_private_package_registries.py b/.github/workflows/replace_private_package_registries.py new file mode 100644 index 00000000..f17947b0 --- /dev/null +++ b/.github/workflows/replace_private_package_registries.py @@ -0,0 +1,57 @@ +"""Update any project files that point to a private package registry to use public ones. + +Since the CI pipelines for testing these copier templates don't have access to private registries, we can't test installing from them as part of CI. + +Seems minimal risk, since the only problem we'd be missing is if the pyproject.toml (or similar config files) had syntax errors that would have been +caught by pre-commit. +""" + +import re +from pathlib import Path + + +def process_file(file_path: Path): + # Read the entire file content + content = file_path.read_text() + + # Regex to match a block starting with [[tool.uv.index]] + # until the next block header (a line starting with [[) or the end of the file. + pattern = re.compile(r"(\[\[tool\.uv\.index\]\].*?)(?=\n\[\[|$)", re.DOTALL) + + # Find all uv.index blocks. + blocks = pattern.findall(content) + + # Check if any block contains "default = true" + if not any("default = true" in block for block in blocks): + print(f"No changes in: {file_path}") + return + + # If at least one block contains "default = true", remove all uv.index blocks. + new_content = pattern.sub("", content) + + # Ensure file ends with a newline before appending the new block. + if not new_content.endswith("\n"): + new_content += "\n" + + # Append the new block. + new_block = '[[tool.uv.index]]\nname = "pypi"\nurl = "https://pypi.org/simple/"\n' + new_content += new_block + + # Write the updated content back to the file. + _ = file_path.write_text(new_content) + print(f"Updated file: {file_path}") + + +def main(): + base_dir = Path(".") + # Use rglob to find all pyproject.toml files recursively. + for file_path in base_dir.rglob("pyproject.toml"): + # Check if the file is at most two levels deep. + # The relative path's parts count should be <= 3 (e.g. "pyproject.toml" is 1 part, + # "subdir/pyproject.toml" is 2 parts, and "subdir/subsubdir/pyproject.toml" is 3 parts). + if len(file_path.relative_to(base_dir).parts) <= 3: + process_file(file_path) + + +if __name__ == "__main__": + main() diff --git a/copier.yml b/copier.yml index 16cc0165..ec7c606a 100644 --- a/copier.yml +++ b/copier.yml @@ -26,6 +26,14 @@ python_version: help: What version of Python is used for development? default: "3.12.7" +python_package_registry: + type: str + help: What registry should Python Packgaes be installed from? + choices: + - PyPI + - AWS CodeArtifact + default: PyPI + python_ci_versions: type: str help: What versions should Python run CI on the instantiated template? @@ -35,6 +43,31 @@ python_ci_versions: - "3.13.2" +aws_identity_center_id: + type: str + help: What's the ID of your Organization's AWS Identity center, e.g. d-9145c20053? + when: "{{ python_package_registry == 'AWS CodeArtifact' }}" + +aws_org_home_region: + type: str + help: What is the home region of the AWS Organization (where all of the central infrastructure is deployed)? + default: us-east-1 + when: "{{ python_package_registry == 'AWS CodeArtifact' }}" + +aws_central_infrastructure_account_id: + type: str + help: What's the ID of your Organization's AWS Account containing Central Infrastructure (e.g. CodeArtifact)? + when: "{{ python_package_registry == 'AWS CodeArtifact' }}" + +core_infra_base_access_profile_name: + type: str + help: What's the AWS Identity Center Profile name for base access to the Central Infrastructure account (i.e. to read from CodeArtifact)? + when: "{{ python_package_registry == 'AWS CodeArtifact' }}" + default: CoreInfraBaseAccess + + + + # Questions specific to this template diff --git a/template/.devcontainer/code-artifact-auth.sh.jinja b/template/.devcontainer/code-artifact-auth.sh.jinja new file mode 100644 index 00000000..c46d3356 --- /dev/null +++ b/template/.devcontainer/code-artifact-auth.sh.jinja @@ -0,0 +1,47 @@ +{% if python_package_registry is defined and python_package_registry == "AWS CodeArtifact" %}{% raw %}#!/usr/bin/env bash +set -ex + +# If none of these are set we can't possibly continue and should fail so you can fix it +if [ -z "$AWS_PROFILE" ] && [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then + echo "No AWS profile, access key, or auth token found, cannot proceed." + exit 1 +else + # Only regenerate the token if it doesn't exist or wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build) + if [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then + echo "Fetching CodeArtifact token" + if [ -z "$CI" ]; then + PROFILE_ARGS="--profile={% endraw %}{{ core_infra_base_access_profile_name }}{% raw %}" + else + PROFILE_ARGS="" + fi + + # Check if AWS credentials are valid by trying to retrieve the caller identity. + # If the ARN is not returned, assume credentials are expired or not set correctly. + caller_identity=$(aws sts get-caller-identity --region={% endraw %}{{ aws_org_home_region }}{% raw %} $PROFILE_ARGS --query Arn --output text 2>/dev/null || echo "") + if [ -z "$caller_identity" ]; then + if [ -n "$CI" ]; then + echo "Error: In CI environment, aws sso login should never be called...something is wrong with this script or your workflow...perhaps you did not OIDC Auth yet in CI?" + exit 1 + fi + echo "SSO credentials not found or expired, logging in..." + aws sso login $PROFILE_ARGS + else + echo "Using existing AWS credentials: $caller_identity" + fi + + set +x + export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + --domain {% endraw %}{{ repo_org_name }}{% raw %} \ + --domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} \ + --region {% endraw %}{{ aws_org_home_region }}{% raw %} \ + --query authorizationToken \ + --output text $PROFILE_ARGS) + set -x + fi + + set +x + export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN" + set -x + +fi{% endraw %}{% else %}{% raw %}# Placeholder file not being used by these copier template answers{% endraw %}{% endif %} diff --git a/template/.devcontainer/create-aws-profile.sh b/template/.devcontainer/create-aws-profile.sh deleted file mode 100644 index 8f93c4ba..00000000 --- a/template/.devcontainer/create-aws-profile.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -set -ex - -mkdir -p ~/.aws - -if [ "$GITHUB_ACTIONS" = "true" ]; then - LOCALSTACK_ENDPOINT_URL="http://localhost:4566" -else - LOCALSTACK_ENDPOINT_URL="http://localstack:4566" -fi - -cat >> ~/.aws/config <> ~/.aws/credentials <> ~/.aws/config <> ~/.aws/credentials <&2 + exit 1 +fi + export UV_PYTHON="$python_version" export UV_PYTHON_PREFERENCE=only-system SCRIPT_DIR="$(dirname "$0")" PROJECT_ROOT_DIR="$(realpath "$SCRIPT_DIR/..")" +# If optionally_lock is set, decide whether to skip locking based on the presence of uv.lock +if [ "$optionally_lock" = "true" ]; then + if [ ! -f "$PROJECT_ROOT_DIR/uv.lock" ]; then + skip_lock=true + else + skip_lock=false + fi +fi + +{% endraw %}{% if python_package_registry is defined and python_package_registry == "AWS CodeArtifact" %}{% raw %} +. "$SCRIPT_DIR/code-artifact-auth.sh"{% endraw %}{% endif %}{% raw %} + # Ensure that the lock file is in a good state if [ "$skip_lock" = "false" ]; then uv lock --check --directory "$PROJECT_ROOT_DIR" diff --git a/template/.devcontainer/on-create-command.sh.jinja b/template/.devcontainer/on-create-command.sh.jinja index 4e74d3fd..2f70a5bb 100644 --- a/template/.devcontainer/on-create-command.sh.jinja +++ b/template/.devcontainer/on-create-command.sh.jinja @@ -7,6 +7,6 @@ git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }} sh .devcontainer/on-create-command-boilerplate.sh -sh .devcontainer/manual-setup-deps.sh +pre-commit install --install-hooks{% endraw %}{% if python_package_registry is not defined or python_package_registry == "PyPI" %} -pre-commit install --install-hooks{% endraw %} +{% raw %}sh .devcontainer/manual-setup-deps.sh --optionally-lock{% endraw %}{% endif %} diff --git a/template/.devcontainer/post-start-command.sh.jinja b/template/.devcontainer/post-start-command.sh.jinja index 2e7f8386..a8ded17e 100644 --- a/template/.devcontainer/post-start-command.sh.jinja +++ b/template/.devcontainer/post-start-command.sh.jinja @@ -3,4 +3,5 @@ set -ex # For some reason the directory is not setup correctly and causes build of devcontainer to fail since # it doesn't have access to the workspace directory. This can normally be done in post-start-command -git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }} +git config --global --add safe.directory /workspaces/{% endraw %}{{ repo_name }}{% if python_package_registry is defined and python_package_registry != "PyPI" %}{% raw %} +echo "!!! In order to install dependencies, you must authenticate into the private registry, so run this script to complete the process: source .devcontainer/manual-setup-deps.sh"{% endraw %}{% endif %} diff --git a/template/.github/actions/install_deps_uv/action.yml b/template/.github/actions/install_deps_uv/action.yml index 55de308c..017b8d05 100644 --- a/template/.github/actions/install_deps_uv/action.yml +++ b/template/.github/actions/install_deps_uv/action.yml @@ -14,6 +14,19 @@ inputs: description: What's the relative path to the project? required: false default: ./ + code-artifact-auth-role-name: + type: string + description: What's the role name to use for CodeArtifact authentication? + required: false + default: no-code-artifact + code-artifact-auth-role-account-id: + type: string + description: What's the AWS Account ID that the role is in? + required: false + code-artifact-auth-region: + type: string + description: What region should the role use? + required: false runs: @@ -41,12 +54,22 @@ runs: run: .github/actions/install_deps_uv/install-ci-tooling.ps1 ${{ env.PYTHON_VERSION }} shell: pwsh + - name: OIDC Auth for CodeArtifact + if: ${{ inputs.code-artifact-auth-role-name != 'no-code-artifact' }} + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }} + aws-region: ${{ inputs.code-artifact-auth-region }} + - name: Install Dependencies (Linux) if: ${{ inputs.uv-sync && runner.os == 'Linux' }} run: | sh .devcontainer/manual-setup-deps.sh ${{ env.PYTHON_VERSION }} shell: bash + + + - name: Install Dependencies (Windows) if: ${{ inputs.uv-sync && runner.os == 'Windows' }} run: .github/actions/install_deps_uv/manual-setup-deps.ps1 ${{ env.PYTHON_VERSION }} diff --git a/template/.github/workflows/ci.yaml.jinja b/template/.github/workflows/ci.yaml.jinja index 7483e1c0..66081e5e 100644 --- a/template/.github/workflows/ci.yaml.jinja +++ b/template/.github/workflows/ci.yaml.jinja @@ -25,7 +25,10 @@ jobs: - name: Install latest versions of python packages uses: ./.github/actions/install_deps_uv with: - python-version: {% endraw %}{{ python_version }}{% raw %} + python-version: {% endraw %}{{ python_version }}{% raw %}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 @@ -72,7 +75,10 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Unit test run: uv run pytest tests/unit --cov-report=xml --durations=5 @@ -109,7 +115,11 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + - name: Build executable run: uv run pyinstaller pyinstaller.spec --log-level=DEBUG - name: Upload executable artifact @@ -139,7 +149,10 @@ jobs: - name: Install python tooling uses: ./.github/actions/install_deps_uv with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: "{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}" + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - name: Build docs working-directory: ./docs diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 65a7c830..6b0e1f30 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -39,7 +39,7 @@ license-files = [] # kludge until this bug is fixed https://github.com/pypa/setu [tool.uv] package = true -[[tool.uv.index]] +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %}[[tool.uv.index]] name = "pypi" url = "https://pypi.org/simple/" publish-url = "https://upload.pypi.org/legacy/" @@ -47,4 +47,15 @@ publish-url = "https://upload.pypi.org/legacy/" [[tool.uv.index]] name = "testpypi" url = "https://test.pypi.org/simple/" -publish-url = "https://test.pypi.org/legacy/"{% endraw %} +publish-url = "https://test.pypi.org/legacy/"{% endraw %}{% else %}{% raw %} + +[[tool.uv.index]] +default = true +name = "code-artifact-primary" +username = "aws" +url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-primary/simple/" + +[[tool.uv.index]] +name = "code-artifact-staging" +username = "aws" +url = "https://{% endraw %}{{ repo_org_name }}{% raw %}-{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}.d.codeartifact.{% endraw %}{{ aws_org_home_region }}{% raw %}.amazonaws.com/pypi/{% endraw %}{{ repo_org_name }}{% raw %}-staging/simple/"{% endraw %}{% endif %} diff --git a/tests/copier_data/data1.yaml b/tests/copier_data/data1.yaml index a1a2903e..58959d82 100644 --- a/tests/copier_data/data1.yaml +++ b/tests/copier_data/data1.yaml @@ -5,6 +5,7 @@ description: Doing amazing things ssh_port_number: 12345 use_windows_in_ci: false +python_package_registry: PyPI aws_identity_center_id: d-9145c20053 aws_org_home_region: us-west-2 aws_production_account_id: 123456789012 diff --git a/tests/copier_data/data2.yaml b/tests/copier_data/data2.yaml index 09f111ae..ae4507ad 100644 --- a/tests/copier_data/data2.yaml +++ b/tests/copier_data/data2.yaml @@ -5,6 +5,9 @@ description: Doing crazy things! So many things! ssh_port_number: 54321 use_windows_in_ci: true +python_package_registry: AWS CodeArtifact +aws_central_infrastructure_account_id: 012321432543 +core_infra_base_access_profile_name: MyAccessRole aws_identity_center_id: d-9145c20053 aws_org_home_region: us-east-1 aws_production_account_id: 123456789012 diff --git a/tests/copier_data/data3.yaml b/tests/copier_data/data3.yaml index fedea476..65f2df82 100644 --- a/tests/copier_data/data3.yaml +++ b/tests/copier_data/data3.yaml @@ -5,6 +5,7 @@ description: Doing amazing things ssh_port_number: 12345 use_windows_in_ci: false +python_package_registry: PyPI aws_identity_center_id: d-9145c20053 aws_org_home_region: us-west-2 aws_production_account_id: 123456789012