diff --git a/.dockerignore b/.dockerignore index b4c8c5b..a7721e7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ .env* -!.env-sample tests .github .git diff --git a/.env-sample b/.env-sample new file mode 100644 index 0000000..3405b1e --- /dev/null +++ b/.env-sample @@ -0,0 +1,7 @@ +AWS_ACCESS_KEY_ID=... +AWS_SECRET_ACCESS_KEY=... +AWS_DEFAULT_REGION=us-east-1 +AWS_S3_BUCKET=... +AWS_BUCKET="${AWS_S3_BUCKET}" + +HTTP=https://${AWS_S3_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com diff --git a/.gitignore b/.gitignore index 0d5a0e1..50baacd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -.env +.env* +!.env-sample /tests/.cleanup-pids /tests/.snapshots-overwrite /tests/.aws-destinations diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1eb5c76 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +ARG REGISTRY_PATH=docker.io/paritytech + +FROM docker.io/library/ubuntu:latest + +ARG VCS_REF=master +ARG BUILD_DATE="" +ARG UID=1000 +ARG GID=1000 +ARG VERSION=0.0.1 + +LABEL summary="Releng scripts" \ + name="${REGISTRY_PATH}/gnupg" \ + maintainer="devops-team@parity.io" \ + version="${VERSION}" \ + description="Releng scripts" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.source="https://github.com/paritytech/scripts/blob/${VCS_REF}/dockerfiles/releng-scripts/Dockerfile" \ + io.parity.image.documentation="https://github.com/paritytech/scripts/blob/${VCS_REF}/dockerfiles/releng-scripts/README.md" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.created="${BUILD_DATE}" + +RUN apt-get update && apt-get install -yq --no-install-recommends ca-certificates bash jq unzip curl && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" && \ + unzip "/tmp/awscliv2.zip" && rm "/tmp/awscliv2.zip" && \ + /aws/install && rm -rf /aws && \ + apt -yq remove ca-certificates unzip && apt -yq autoremove && \ + aws --version + +WORKDIR /scripts + +COPY . . + +RUN set -x \ + && groupadd -g $GID nonroot \ + && useradd -u $UID -g $GID -s /bin/bash -m nonroot + +USER nonroot:nonroot + +ENTRYPOINT [ "./rs" ] diff --git a/README.md b/README.md index b2018b6..653fe43 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,106 @@ -# TOC - -- [Introduction](#introduction) -- [Reserving crates](#reserving-crates) -- [Usage](#usage) - - [GitHub Actions](#usage-github-actions) - - [GitLab Jobs](#usage-gitlab-jobs) - - [Locally](#usage-locally) -- [Development](#development) - - [Repository structure](#development-repository-structure) - - [External consumption](#development-repository-structure-external-consumption) - - [Tools](#development-repository-structure-tools) - - [Maintenance](#development-repository-structure-maintenance) - - [Tests](#development-tests) - - [Linters](#development-linters) - - [S3](#development-s3) - -# Introduction - -This repository contains scripts managed and used by -[release-engineering](https://github.com/orgs/paritytech/teams/release-engineering). +# Introduction + +This repository contains scripts managed and used by [release-engineering](https://github.com/orgs/paritytech/teams/release-engineering). See the [Tools wiki page](https://github.com/paritytech/releng-scripts/wiki/Tools#TOC) for information on the functionality provided through this repository. -# Reserving crates +# rs + +The commands offerered by `rs` can be access via script, GHS, Docker, etc.. +Those use cases are described in the documentation. + +The following chapters explain what those commands **are** and how to use them. + +You may find convenient testing using: + + alias rs='docker run --rm -it -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e AWS_BUCKET paritytech/releng-scripts' + +# `rs version` + +Get the version: + + docker run --rm -it paritytech/releng-scripts version + +output: + + 0.0.1 + +# `rs upload` + +Uplooad an artifact. + +Check the help with `rs upload --help` + + rs upload --bucket $AWS_BUCKET custom foo/bar s3 tests/fixtures/foo.txt -1. Go to https://github.com/paritytech/releng-scripts/actions/workflows/reserve-crate.yml -2. Click the "Run workflow" button to access the workflow's form -3. Fill and send the workflow's form. After that a workflow run - ([example](https://github.com/paritytech/releng-scripts/actions/runs/3642900863/attempts/1)) - will be created; you might need to refresh the page in order to see it. -4. Wait for the workflow run to finish +# `rs download` -# Usage +Download an artifact. + +# `rs delete` + +Delete an artifact. + +# Reserving crates + +1. Go to + +2. Click the "Run workflow" button to access the workflow’s form + +3. Fill and send the workflow’s form. After that a workflow run + ([example](https://github.com/paritytech/releng-scripts/actions/runs/3642900863/attempts/1)) + will be created; you might need to refresh the page in order to see it. + +4. Wait for the workflow run to finish + +# Usage ## Docker The image is available as `paritytech/releng-scripts` and usable as: -```bash -# Show the help -docker run --rm -it paritytech/releng-scripts -# Show the version -docker run --rm -it paritytech/releng-scripts version -``` + # Show the help + docker run --rm -it paritytech/releng-scripts + # Show the version + docker run --rm -it paritytech/releng-scripts version -## GitHub Actions - -```yaml -jobs: - upload-artifact: - name: My workflow - runs-on: ubuntu-latest - steps: - - name: First step - run: | - git clone --depth=1 https://github.com/paritytech/releng-scripts - ./releng-scripts/foo ... -``` +## GitHub Actions -## GitLab Jobs + jobs: + upload-artifact: + name: My workflow + runs-on: ubuntu-latest + steps: + - name: First step + run: | + git clone --depth=1 https://github.com/paritytech/releng-scripts + ./releng-scripts/foo ... -```yaml -job: - script: - - git clone --depth=1 https://github.com/paritytech/releng-scripts - - ./releng-scripts/foo ... -``` +## GitLab Jobs -## Locally + job: + script: + - git clone --depth=1 https://github.com/paritytech/releng-scripts + - ./releng-scripts/foo ... + +## Locally Clone this repository and run the scripts -# Development +# Contributing -## Repository structure +## Repository structure -### External consumption +### External consumption If a script is meant for external consumption, such as the tools' entrypoints, -then avoid adding file extensions to them since that's more subject to breaking -changes in case we want to change the script's programming language later. +then avoid adding file extensions to them since that’s more subject to breaking +changes in case we want to change the script’s programming language later. Adding the extension is encouraged for files which are not meant for external consumption, i.e. scripts which are used only internally or are run through some command runner such as `just`. -Here's an example: +Here’s an example: ``` /repository @@ -99,24 +115,24 @@ include an extension. On the other hand, `upload.sh`, which corresponds to the `upload` subcommand of `releng-scripts`, can keep its extension because it's not meant for external consumption, as it's invoked by `releng-scripts`. -### Tools +### Tools Tools are organized with the following hierarchy: -- Their entrypoints are located at the root of the repository for - ease-of-external-consumption's sake. +- Their entrypoints are located at the root of the repository for + ease-of-external-consumption’s sake. - Avoid including the extension to those files because that's more subject to - breaking changes in case we want to change the tool's programming language - later. + Avoid including the extension to those files because that's more subject to + breaking changes in case we want to change the tool's programming language + later. - Please maintain an entry to the tools in `./justfile` for ease-of-use's sake. + Please maintain an entry to the tools in `./justfile` for ease-of-use's sake. -- In case the tool has subcommands, they are located at `./cmd/$TOOL/$SUBCOMMAND` +- In case the tool has subcommands, they are located at `./cmd/$TOOL/$SUBCOMMAND` - This is to avoid noisy handling of too many commands within a single file. + This is to avoid noisy handling of too many commands within a single file. -Here's an example: +Here’s an example: ``` /repository @@ -129,14 +145,14 @@ Here's an example: `releng-scripts` is the tool's entrypoint and `upload.sh` corresponds to the `upload` subcommand of `releng-scripts`. -### Maintenance +### Maintenance The `./tasks` directory groups scripts for tasks related to project maintenance, such as running linters and tests. -Please maintain an entry to those scripts in `./justfile` for ease-of-use's sake. +Please maintain an entry to those scripts in `./justfile` for ease-of-use’s sake. -## Tests +## Tests Run the test: `just tests` @@ -144,22 +160,20 @@ Update the snapshots: `just tests --update` Delete stale snapshots: `just tests --delete-stale-snapshots` -## Linters +## Linters `just linters` -## S3 +## S3 -For testing out the S3 backend you can use https://github.com/adobe/S3Mock. +For testing out the S3 backend you can use [S3Mock](https://github.com/adobe/S3Mock). Set up the following environment variables: -``` -export AWS_ACCESS_KEY_ID=1234567890 -export AWS_SECRET_ACCESS_KEY=valid-test-key-ref -export AWS_DEFAULT_REGION=us-east-1 -export AWS_BUCKET=test -``` + export AWS_ACCESS_KEY_ID=1234567890 + export AWS_SECRET_ACCESS_KEY=valid-test-key-ref + export AWS_DEFAULT_REGION=us-east-1 + export AWS_BUCKET=test Then start S3Mock: diff --git a/README_src.adoc b/README_src.adoc new file mode 100644 index 0000000..a871d6f --- /dev/null +++ b/README_src.adoc @@ -0,0 +1,21 @@ +:toc: + +== Introduction + +This repository contains scripts managed and used by https://github.com/orgs/paritytech/teams/release-engineering[release-engineering]. + +See the https://github.com/paritytech/releng-scripts/wiki/Tools#TOC[Tools wiki page] for information on the functionality provided through this repository. + +include::doc/rs.adoc[] + +include::doc/crate-reservation.adoc[leveloffset=+1] + +include::doc/usage.adoc[leveloffset=+1] + +include::doc/contributing.adoc[leveloffset=+1] + +== License + +---- +include::LICENSE[] +---- diff --git a/cmd/releng-scripts/upload.sh b/cmd/releng-scripts/upload.sh index 527d963..ca3ba0b 100755 --- a/cmd/releng-scripts/upload.sh +++ b/cmd/releng-scripts/upload.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# requirements: +# jq, aws + set -Eeu -o pipefail shopt -s inherit_errexit @@ -45,7 +48,7 @@ upload_to_s3() { "${backend_upload_args[@]}" "${forwarded_backend_args[@]}" -- - "$file" + "${file}" "$destination" ) @@ -83,6 +86,7 @@ upload_to_s3() { ;; *) log info "Checking metadata for destination $destination" + echo "$response" if echo -n "$response" | jq -e; then # Valid JSON response for metadata, which means that the file exists log error "File already exists: $destination" @@ -222,7 +226,7 @@ main() { local remote_destination="${upload_dir:+$upload_dir/}$filename" if [ ! "${DRY_RUN:-}" ]; then - log "Uploading file: $file" + log "Uploading file: $file to bucket $bucket" log "Upload destination: $remote_destination" fi diff --git a/doc/contributing.adoc b/doc/contributing.adoc new file mode 100644 index 0000000..ede1904 --- /dev/null +++ b/doc/contributing.adoc @@ -0,0 +1,91 @@ += Contributing + +== Repository structure + +=== External consumption + +If a script is meant for external consumption, such as the tools' entrypoints, +then avoid adding file extensions to them since that's more subject to breaking +changes in case we want to change the script's programming language later. +Adding the extension is encouraged for files which are not meant for external +consumption, i.e. scripts which are used only internally or are run through some +command runner such as `just`. + +Here's an example: + + /repository + ├── cmd + │ └── rs + │ └── upload.sh + └── rs + +`rs` is a tool entrypoint meant for external consumption, therefore it doesn't +include an extension. On the other hand, `upload.sh`, which corresponds to the +`upload` subcommand of `rs`, can keep its extension because it's not meant for +external consumption, as it's invoked by `rs`. + +=== Tools + +Tools are organized with the following hierarchy: + +- Their entrypoints are located at the root of the repository for + ease-of-external-consumption's sake. + + Avoid including the extension to those files because that's more subject to + breaking changes in case we want to change the tool's programming language + later. + + Please maintain an entry to the tools in `./justfile` for ease-of-use's sake. + +- In case the tool has subcommands, they are located at `./cmd/$TOOL/$SUBCOMMAND` + + This is to avoid noisy handling of too many commands within a single file. + +Here's an example: + + /repository + ├── cmd + │ └── rs + │ └── upload.sh + └── rs + +`rs` is the tool's entrypoint and `upload.sh` corresponds to the `upload` +subcommand of `rs`. + +=== Maintenance + +The `./tasks` directory groups scripts for tasks related to project maintenance, +such as running linters and tests. + +Please maintain an entry to those scripts in `./justfile` for ease-of-use's sake. + +== Tests + +Run the test: `just tests` + +Update the snapshots: `just tests --update` + +Delete stale snapshots: `just tests --delete-stale-snapshots` + +== Linters + +`just linters` + +== S3 + +For testing out the S3 backend you can use https://github.com/adobe/S3Mock[S3Mock]. + +Set up the following environment variables: + + export AWS_ACCESS_KEY_ID=1234567890 + export AWS_SECRET_ACCESS_KEY=valid-test-key-ref + export AWS_DEFAULT_REGION=us-east-1 + export AWS_BUCKET=test + +Then start S3Mock: + +`./tasks/start-s3-mock.sh` + +Then try to upload a file: + +`just rs upload custom foo/bar s3 --s3mock tests/fixtures/foo.txt` diff --git a/doc/crate-reservation.adoc b/doc/crate-reservation.adoc new file mode 100644 index 0000000..1baf9f7 --- /dev/null +++ b/doc/crate-reservation.adoc @@ -0,0 +1,9 @@ + += Reserving crates + +. Go to https://github.com/paritytech/releng-scripts/actions/workflows/reserve-crate.yml +. Click the "Run workflow" button to access the workflow's form +. Fill and send the workflow's form. After that a workflow run + (https://github.com/paritytech/releng-scripts/actions/runs/3642900863/attempts/1[example]) + will be created; you might need to refresh the page in order to see it. +. Wait for the workflow run to finish diff --git a/doc/rs-delete.adoc b/doc/rs-delete.adoc new file mode 100644 index 0000000..d6069b5 --- /dev/null +++ b/doc/rs-delete.adoc @@ -0,0 +1,4 @@ + +== `rs delete` + +Delete an artifact. diff --git a/doc/rs-download.adoc b/doc/rs-download.adoc new file mode 100644 index 0000000..f4dee13 --- /dev/null +++ b/doc/rs-download.adoc @@ -0,0 +1,4 @@ + +== `rs download` + +Download an artifact. diff --git a/doc/rs-upload.adoc b/doc/rs-upload.adoc new file mode 100644 index 0000000..722a122 --- /dev/null +++ b/doc/rs-upload.adoc @@ -0,0 +1,8 @@ + +== `rs upload` + +Uplooad an artifact. + +Check the help with `rs upload --help` + + rs upload --bucket $AWS_BUCKET custom foo/bar s3 tests/fixtures/foo.txt diff --git a/doc/rs-version.adoc b/doc/rs-version.adoc new file mode 100644 index 0000000..537edfb --- /dev/null +++ b/doc/rs-version.adoc @@ -0,0 +1,10 @@ + +== `rs version` + +Get the version: + + docker run --rm -it paritytech/releng-scripts version + +output: + + 0.0.1 diff --git a/doc/rs.adoc b/doc/rs.adoc new file mode 100644 index 0000000..7c6822e --- /dev/null +++ b/doc/rs.adoc @@ -0,0 +1,18 @@ +== rs + +The commands offerered by `rs` can be access via script, GHS, Docker, etc.. +Those use cases are described in the documentation. + +The following chapters explain what those commands **are** and how to use them. + +You may find convenient testing using: + + alias rs='docker run --rm -it -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e AWS_BUCKET paritytech/releng-scripts' + +include::rs-version.adoc[leveloffset=+1] + +include::rs-upload.adoc[leveloffset=+1] + +include::rs-download.adoc[leveloffset=+1] + +include::rs-delete.adoc[leveloffset=+1] diff --git a/doc/usage.adoc b/doc/usage.adoc new file mode 100644 index 0000000..a69d010 --- /dev/null +++ b/doc/usage.adoc @@ -0,0 +1,40 @@ + += Usage + +== Docker + +Once the automation is in place, the image will be available as `paritytech/releng-scripts` and usable as: + +```bash +# Show the help +docker run --rm -it paritytech/releng-scripts +# Show the version +docker run --rm -it paritytech/releng-scripts version +``` + +== GitHub Actions + +```yaml +jobs: + upload-artifact: + name: My workflow + runs-on: ubuntu-latest + steps: + - name: First step + run: | + git clone --depth=1 https://github.com/paritytech/releng-scripts + ./releng-scripts/foo ... +``` + +== GitLab Jobs + +```yaml +job: + script: + - git clone --depth=1 https://github.com/paritytech/releng-scripts + - ./releng-scripts/foo ... +``` + +== Locally + +Clone this repository and run the scripts diff --git a/justfile b/justfile index 057a9a1..50e7ffd 100644 --- a/justfile +++ b/justfile @@ -1,11 +1,16 @@ set positional-arguments export docker_image_name := "releng-scripts" +export ENGINE := "podman" default_owner := "paritytech" # List available commands _default: just --choose --chooser "fzf +s -x --tac --cycle" +# Install required tooling +setup: + pip install pre-commit + # Show some help help: just --list @@ -19,11 +24,23 @@ linters *args: releng-scripts *args: ./releng-scripts "$@" - +# Run as container run *args: - docker run --rm -it releng-scripts "$@" + $ENGINE run --rm -it releng-scripts "$@" + +# Build the docker image +build_docker_image owner=default_owner: + $ENGINE build -t {{owner}}/$docker_image_name . + $ENGINE images | grep {{owner}}/$docker_image_name + +# Push the docker image +publish_docker_image owner=default_owner: (build_docker_image owner) + $ENGINE push {{owner}}/$docker_image_name + +# Publish everything +publish: publish_docker_image -# Generate the readme as .md +# Generate the readme as Markdown file md: #!/usr/bin/env bash asciidoctor -b docbook -a leveloffset=+1 -o - README_src.adoc | pandoc --markdown-headings=atx --wrap=preserve -t markdown_strict -f docbook - > README.md