From ddad1c392aa813cf05535f03a5e6a03fb2552c6c Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 28 Mar 2022 17:06:51 -0600 Subject: [PATCH 01/20] GHA workflow to build ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine --- .env | 1 + .../trafficserver-alpine/Dockerfile | 60 ++++++++++++++++++ .../trafficserver-alpine/docker-compose.yml | 27 ++++++++ .../container-trafficserver-alpine.yml | 61 +++++++++++++++++++ 4 files changed, 149 insertions(+) create mode 100644 .github/containers/trafficserver-alpine/Dockerfile create mode 100644 .github/containers/trafficserver-alpine/docker-compose.yml create mode 100644 .github/workflows/container-trafficserver-alpine.yml diff --git a/.env b/.env index 167797e43a..904d0e08e4 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ +ATS_VERSION=9.1.2 GO_VERSION=1.18 diff --git a/.github/containers/trafficserver-alpine/Dockerfile b/.github/containers/trafficserver-alpine/Dockerfile new file mode 100644 index 0000000000..4788c73f34 --- /dev/null +++ b/.github/containers/trafficserver-alpine/Dockerfile @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM alpine:latest AS build-trafficserver +ARG ATS_VERSION +ADD https://downloads.apache.org/trafficserver/trafficserver-${ATS_VERSION}.tar.bz2 /tmp/ +RUN set -o errexit -o nounset; \ + cd tmp; \ + dirname=trafficserver-${ATS_VERSION}; \ + tar xf ${dirname}.tar.bz2; \ + rm ${dirname}.tar.bz2; \ + apk add --no-cache \ + # configure dependencies + g++ \ + perl \ + openssl-dev \ + pcre-dev \ + make \ + # build dependencies + libexecinfo-dev \ + fortify-headers \ + linux-headers \ + zlib-dev; \ + cd $dirname; \ + ./configure \ + --disable-tests \ + --enable-experimental-plugins \ + --prefix=/ \ + --with-user=ats \ + --with-group=ats; \ + make -j; \ + adduser -D ats; \ + make install DESTDIR=/tmp/built; \ + cd ..; \ + rm -r $dirname + +FROM alpine:latest +COPY --from=build-trafficserver /tmp/built/ / +RUN apk add --no-cache \ + # runtime dependencies + libexecinfo \ + libstdc++ \ + pcre && \ + adduser -D ats +USER ats +CMD /bin/traffic_server diff --git a/.github/containers/trafficserver-alpine/docker-compose.yml b/.github/containers/trafficserver-alpine/docker-compose.yml new file mode 100644 index 0000000000..cb5e4d1936 --- /dev/null +++ b/.github/containers/trafficserver-alpine/docker-compose.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.9' +services: + trafficserver: + build: + context: . + dockerfile: Dockerfile + args: + ATS_VERSION: ${ATS_VERSION} + # for example, ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine:9.1.2 + image: ${CONTAINER}:${ATS_VERSION} diff --git a/.github/workflows/container-trafficserver-alpine.yml b/.github/workflows/container-trafficserver-alpine.yml new file mode 100644 index 0000000000..f821a3bdf7 --- /dev/null +++ b/.github/workflows/container-trafficserver-alpine.yml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Container ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine + +env: + DOCKER_BUILDKIT: '1' + COMPOSE_DOCKER_CLI_BUILD: '1' + CONTAINER: ghcr.io/${{ github.repository }}/ci/trafficserver-alpine + +on: + workflow_dispatch: # run manually + push: + paths: + - .env + - .github/containers/trafficserver-alpine/** + - .github/workflows/containers/trafficserver-alpine.yml + pull_request: + paths: + - .env + - .github/containers/trafficserver-alpine/** + - .github/workflows/containers/trafficserver-alpine.yml + types: [ opened, reopened, ready_for_review, synchronize ] + +jobs: + build: + if: ${{ github.repository_owner == 'apache' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@master + - name: Build ${{ env.CONTAINER }} + working-directory: .github/containers/trafficserver-alpine + run: | + set -o errexit -o nounset + while IFS= read -r line; do + export "$line" + done <${GITHUB_WORKSPACE}/.env + docker-compose build + - name: Push ${{ env.CONTAINER }} + working-directory: .github/containers/trafficserver-alpine + run: | + set -o errexit -o nounset + while IFS= read -r line; do + export "$line" + done <${GITHUB_WORKSPACE}/.env + docker-compose push From 43d04c3e2622ce7d54e29d06c154bd7df93549c1 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 28 Mar 2022 17:31:03 -0600 Subject: [PATCH 02/20] Get trafficserver from ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine --- dev/t3c/Dockerfile | 11 ++++------- docker-compose.yml | 1 + 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dev/t3c/Dockerfile b/dev/t3c/Dockerfile index 240b48f7ef..b9e1b713ee 100644 --- a/dev/t3c/Dockerfile +++ b/dev/t3c/Dockerfile @@ -14,14 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +ARG ATS_VERSION ARG GO_VERSION -FROM alpine:latest AS traffic-server-builder - -RUN apk add --no-cache build-base perl libexecinfo-dev pcre-dev libressl-dev libtool linux-headers openssl-dev zlib-dev -ADD https://downloads.apache.org/trafficserver/trafficserver-9.1.1.tar.bz2 / -RUN tar -xf trafficserver-9.1.1.tar.bz2 && cd trafficserver-9.1.1 && mkdir /ats && ./configure --prefix / --enable-experimental-plugins && make -j && make install && ls -R /ats - +FROM ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine:${ATS_VERSION} AS traffic-server-builder FROM golang:${GO_VERSION}-alpine AS t3c-dev +ARG ATS_VERSION ENV TC="/root/go/src/github.com/apache/trafficcontrol/" GOFLAGS="--buildvcs=false" VOLUME /root/go/src/github.com/apache/trafficcontrol @@ -33,7 +30,7 @@ COPY --from=traffic-server-builder /include/ts /include/ts COPY --from=traffic-server-builder /include/tscpp /include/tscpp COPY --from=traffic-server-builder /lib/perl5 /lib/perl5 COPY --from=traffic-server-builder /lib/pkgconfig/trafficserver.pc /lib/pkgconfig/trafficserver.pc -COPY --from=traffic-server-builder /lib/libtscore.la /lib/libtscore.so /lib/libtscore.so.9 /lib/libtscore.so.9.1.1 /lib/libtscppapi.la /lib/libtscppapi.so /lib/libtscppapi.so.9 /lib/libtscppapi.so.9.1.1 /lib/libtscpputil.la /lib/libtscpputil.so /lib/libtscpputil.so.9 /lib/libtscpputil.so.9.1.1 /lib/libtsmgmt.la /lib/libtsmgmt.so /lib/libtsmgmt.so.9 /lib/libtsmgmt.so.9.1.1 /lib/plugin_init_fail.la /lib/plugin_init_fail.so /lib/plugin_instinit_fail.la /lib/plugin_instinit_fail.so /lib/plugin_missing_deleteinstance.la /lib/plugin_missing_deleteinstance.so /lib/plugin_missing_doremap.la /lib/plugin_missing_doremap.so /lib/plugin_missing_init.la /lib/plugin_missing_init.so /lib/plugin_missing_newinstance.la /lib/plugin_missing_newinstance.so /lib/plugin_required_cb.la /lib/plugin_required_cb.so /lib/plugin_testing_calls.la /lib/plugin_testing_calls.so /lib/plugin_v1.la /lib/plugin_v1.so /lib/plugin_v2.la /lib/plugin_v2.so /lib/ +COPY --from=traffic-server-builder /lib/libtscore.la /lib/libtscore.so /lib/libtscore.so.9 /lib/libtscore.so.${ATS_VERSION} /lib/libtscppapi.la /lib/libtscppapi.so /lib/libtscppapi.so.9 /lib/libtscppapi.so.${ATS_VERSION} /lib/libtscpputil.la /lib/libtscpputil.so /lib/libtscpputil.so.9 /lib/libtscpputil.so.${ATS_VERSION} /lib/libtsmgmt.la /lib/libtsmgmt.so /lib/libtsmgmt.so.9 /lib/libtsmgmt.so.${ATS_VERSION} /lib/plugin_init_fail.la /lib/plugin_init_fail.so /lib/plugin_instinit_fail.la /lib/plugin_instinit_fail.so /lib/plugin_missing_deleteinstance.la /lib/plugin_missing_deleteinstance.so /lib/plugin_missing_doremap.la /lib/plugin_missing_doremap.so /lib/plugin_missing_init.la /lib/plugin_missing_init.so /lib/plugin_missing_newinstance.la /lib/plugin_missing_newinstance.so /lib/plugin_required_cb.la /lib/plugin_required_cb.so /lib/plugin_testing_calls.la /lib/plugin_testing_calls.so /lib/plugin_v1.la /lib/plugin_v1.so /lib/plugin_v2.la /lib/plugin_v2.so /lib/ COPY --from=traffic-server-builder /libexec/trafficserver /libexec/trafficserver COPY --from=traffic-server-builder /share/man /share/man RUN mkdir /share/trafficserver && mkdir -p /var/log/trafficserver && mkdir /var/trafficserver diff --git a/docker-compose.yml b/docker-compose.yml index fb13702bd0..639f7307c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,6 +95,7 @@ services: build: context: dev/t3c args: + - ATS_VERSION=${ATS_VERSION} - GO_VERSION=${GO_VERSION} depends_on: - trafficops From 91ef84ba84de892ac66f09676ac165b1e4e089b7 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 28 Mar 2022 18:21:56 -0600 Subject: [PATCH 03/20] Log into ghcr.io before pushing the image --- .github/workflows/container-trafficserver-alpine.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/container-trafficserver-alpine.yml b/.github/workflows/container-trafficserver-alpine.yml index f821a3bdf7..c7d504dd60 100644 --- a/.github/workflows/container-trafficserver-alpine.yml +++ b/.github/workflows/container-trafficserver-alpine.yml @@ -51,6 +51,12 @@ jobs: export "$line" done <${GITHUB_WORKSPACE}/.env docker-compose build + - name: docker login ghcr.io + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: . # The username can be anything but must not be empty + password: ${{ secrets.GITHUB_TOKEN }} - name: Push ${{ env.CONTAINER }} working-directory: .github/containers/trafficserver-alpine run: | From 06dc5b47730f728df4f5847deca36dab880d1f5f Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 28 Mar 2022 19:16:04 -0600 Subject: [PATCH 04/20] Base t3c image on trafficserver-alpine image --- dev/t3c/Dockerfile | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/dev/t3c/Dockerfile b/dev/t3c/Dockerfile index b9e1b713ee..0388398b99 100644 --- a/dev/t3c/Dockerfile +++ b/dev/t3c/Dockerfile @@ -16,32 +16,24 @@ # under the License. ARG ATS_VERSION ARG GO_VERSION -FROM ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine:${ATS_VERSION} AS traffic-server-builder -FROM golang:${GO_VERSION}-alpine AS t3c-dev -ARG ATS_VERSION +FROM golang:${GO_VERSION}-alpine AS get-go +FROM ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine:${ATS_VERSION} +COPY --from=get-go /usr/local/go /usr/local/go +ENV PATH=/usr/local/go/bin:${PATH} \ + GOPATH=/go +ENV PATH=${GOPATH}/bin:${PATH} ENV TC="/root/go/src/github.com/apache/trafficcontrol/" GOFLAGS="--buildvcs=false" VOLUME /root/go/src/github.com/apache/trafficcontrol EXPOSE 80 8081 -COPY --from=traffic-server-builder /bin/traffic_cache_tool /bin/traffic_crashlog /bin/traffic_ctl /bin/traffic_layout /bin/traffic_logcat /bin/traffic_logstats /bin/traffic_manager /bin/traffic_server /bin/traffic_via /bin/trafficserver /bin/tspush /bin/tsxs /bin/ -COPY --from=traffic-server-builder /etc/trafficserver /etc/trafficserver -COPY --from=traffic-server-builder /include/ts /include/ts -COPY --from=traffic-server-builder /include/tscpp /include/tscpp -COPY --from=traffic-server-builder /lib/perl5 /lib/perl5 -COPY --from=traffic-server-builder /lib/pkgconfig/trafficserver.pc /lib/pkgconfig/trafficserver.pc -COPY --from=traffic-server-builder /lib/libtscore.la /lib/libtscore.so /lib/libtscore.so.9 /lib/libtscore.so.${ATS_VERSION} /lib/libtscppapi.la /lib/libtscppapi.so /lib/libtscppapi.so.9 /lib/libtscppapi.so.${ATS_VERSION} /lib/libtscpputil.la /lib/libtscpputil.so /lib/libtscpputil.so.9 /lib/libtscpputil.so.${ATS_VERSION} /lib/libtsmgmt.la /lib/libtsmgmt.so /lib/libtsmgmt.so.9 /lib/libtsmgmt.so.${ATS_VERSION} /lib/plugin_init_fail.la /lib/plugin_init_fail.so /lib/plugin_instinit_fail.la /lib/plugin_instinit_fail.so /lib/plugin_missing_deleteinstance.la /lib/plugin_missing_deleteinstance.so /lib/plugin_missing_doremap.la /lib/plugin_missing_doremap.so /lib/plugin_missing_init.la /lib/plugin_missing_init.so /lib/plugin_missing_newinstance.la /lib/plugin_missing_newinstance.so /lib/plugin_required_cb.la /lib/plugin_required_cb.so /lib/plugin_testing_calls.la /lib/plugin_testing_calls.so /lib/plugin_v1.la /lib/plugin_v1.so /lib/plugin_v2.la /lib/plugin_v2.so /lib/ -COPY --from=traffic-server-builder /libexec/trafficserver /libexec/trafficserver -COPY --from=traffic-server-builder /share/man /share/man -RUN mkdir /share/trafficserver && mkdir -p /var/log/trafficserver && mkdir /var/trafficserver - -RUN apk add --no-cache make inotify-tools gcc libc-dev pcre libexecinfo && \ +USER root +RUN apk add --no-cache inotify-tools gcc && \ go install github.com/go-delve/delve/cmd/dlv@latest && \ ln -s /root/go/bin/dlv /usr/bin/dlv && \ mkdir /lib64 && \ ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 -RUN addgroup ats && adduser -S -s /bin/ash -G ats ats && chown -R ats:ats /etc/trafficserver /var/trafficserver /share/trafficserver /var/log/trafficserver RUN echo "stats_over_http.so" >> /etc/trafficserver/plugin.config && echo "system_stats.so" >> /etc/trafficserver/plugin.config CMD /root/go/src/github.com/apache/trafficcontrol/dev/t3c/run.sh From 3b78682a3d1b05fc64001f8a1d750b3c39323bee Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 28 Mar 2022 19:30:33 -0600 Subject: [PATCH 05/20] Other simplifications --- dev/t3c/Dockerfile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dev/t3c/Dockerfile b/dev/t3c/Dockerfile index 0388398b99..5653df5f0c 100644 --- a/dev/t3c/Dockerfile +++ b/dev/t3c/Dockerfile @@ -24,15 +24,12 @@ ENV PATH=/usr/local/go/bin:${PATH} \ ENV PATH=${GOPATH}/bin:${PATH} ENV TC="/root/go/src/github.com/apache/trafficcontrol/" GOFLAGS="--buildvcs=false" -VOLUME /root/go/src/github.com/apache/trafficcontrol +VOLUME $TC EXPOSE 80 8081 USER root -RUN apk add --no-cache inotify-tools gcc && \ - go install github.com/go-delve/delve/cmd/dlv@latest && \ - ln -s /root/go/bin/dlv /usr/bin/dlv && \ - mkdir /lib64 && \ - ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 +RUN apk add --no-cache inotify-tools gcc musl-dev && \ + go install github.com/go-delve/delve/cmd/dlv@latest RUN echo "stats_over_http.so" >> /etc/trafficserver/plugin.config && echo "system_stats.so" >> /etc/trafficserver/plugin.config From 314e9a799d13e621037ef184bdb86d55f9e74d09 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 09:11:59 -0600 Subject: [PATCH 06/20] Do not push to the repository on pull requests --- .github/workflows/container-trafficserver-alpine.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container-trafficserver-alpine.yml b/.github/workflows/container-trafficserver-alpine.yml index c7d504dd60..b0e1974335 100644 --- a/.github/workflows/container-trafficserver-alpine.yml +++ b/.github/workflows/container-trafficserver-alpine.yml @@ -58,6 +58,7 @@ jobs: username: . # The username can be anything but must not be empty password: ${{ secrets.GITHUB_TOKEN }} - name: Push ${{ env.CONTAINER }} + if: ${{ github.event_name != 'pull_request' }} working-directory: .github/containers/trafficserver-alpine run: | set -o errexit -o nounset From d1b60b283101af6e76626489af5c9cb9e7538608 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 09:13:56 -0600 Subject: [PATCH 07/20] Set username to repository owner --- .github/workflows/container-trafficserver-alpine.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-trafficserver-alpine.yml b/.github/workflows/container-trafficserver-alpine.yml index b0e1974335..246f20611a 100644 --- a/.github/workflows/container-trafficserver-alpine.yml +++ b/.github/workflows/container-trafficserver-alpine.yml @@ -55,7 +55,7 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: . # The username can be anything but must not be empty + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Push ${{ env.CONTAINER }} if: ${{ github.event_name != 'pull_request' }} From 9d1b0fe64b25a77f89d19780280d886b92d1a6e0 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 10:52:13 -0600 Subject: [PATCH 08/20] Only modify GO_VERSION value in env file when updating the Go version --- .../pr-to-update-go/pr_to_update_go/constants.py | 6 ++++++ .../pr_to_update_go/go_pr_maker.py | 15 ++++++++++++--- .github/actions/pr-to-update-go/requirements.txt | 1 + .github/actions/pr-to-update-go/setup.cfg | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/constants.py b/.github/actions/pr-to-update-go/pr_to_update_go/constants.py index 44316db9a4..f7ae6ec63a 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/constants.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/constants.py @@ -69,6 +69,11 @@ version (e.g. GO_VERSION=3.2.1). """ +GO_VERSION_KEY: Final = 'GO_VERSION' +""" +The key in the env file whose value corresponds to the Go version to be used by any project +using the env file +""" GIT_AUTHOR_EMAIL_TEMPLATE: Final = '{git_author_name}@users.noreply.github.com' """Template used to construct the Git Author's email address.""" @@ -94,4 +99,5 @@ "GO_REPO_NAME", "GO_VERSION_URL", "RELEASE_PAGE_URL", + "GO_VERSION_KEY", ] diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index acb358dadb..ff3d12711e 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -26,6 +26,7 @@ import sys from typing import Optional, TypedDict, Any +import dotenv import requests from github.Commit import Commit @@ -48,7 +49,8 @@ RELEASE_PAGE_URL, ENV_GO_VERSION_FILE, ENV_GIT_AUTHOR_NAME, - GIT_AUTHOR_EMAIL_TEMPLATE + GIT_AUTHOR_EMAIL_TEMPLATE, + GO_VERSION_KEY, ) class GoVersion(TypedDict): @@ -396,11 +398,16 @@ def set_go_version(self, go_version: str, commit_message: str, print(f'Updated {go_version_file} on {self.repo.name}') env_path = os.path.join(os.path.dirname(getenv(ENV_ENV_FILE)), ".env") - content = f"GO_VERSION={go_version}\n" sha = self.file_contents(env_path, source_branch_name).sha + with open(env_path, encoding='UTF-8') as env_stream: + previous_env_content = env_stream.read() + dotenv.set_key(dotenv_path=env_path, key_to_set=GO_VERSION_KEY, value_to_set=go_version, + quote_mode='never') + with open(env_path, encoding='UTF-8') as env_stream: + env_content = env_stream.read() commit = self.repo.update_file( branch=source_branch_name, - content=content, + content=env_content, path=env_path, message=commit_message, sha=sha @@ -408,6 +415,8 @@ def set_go_version(self, go_version: str, commit_message: str, if not isinstance(commit, Commit): raise TypeError("'commit' property of file update response was not a Commit") print(f"Updated {env_path} on {self.repo.name}") + with open(env_path, encoding='UTF-8', mode='w') as env_stream: + env_stream.write(previous_env_content) return commit def update_golang_org_x(self, previous_commit: Commit) -> Optional[GitCommit]: diff --git a/.github/actions/pr-to-update-go/requirements.txt b/.github/actions/pr-to-update-go/requirements.txt index 05cf2ead7c..f54204bd26 100644 --- a/.github/actions/pr-to-update-go/requirements.txt +++ b/.github/actions/pr-to-update-go/requirements.txt @@ -10,5 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # +python-dotenv PyGithub requests \ No newline at end of file diff --git a/.github/actions/pr-to-update-go/setup.cfg b/.github/actions/pr-to-update-go/setup.cfg index 939d0c83ca..2b6c0fa5bd 100644 --- a/.github/actions/pr-to-update-go/setup.cfg +++ b/.github/actions/pr-to-update-go/setup.cfg @@ -24,6 +24,7 @@ classifiers = OSI Approved :: Apache Software License python_requires = >=3.9 packages = pr_to_update_go install_requires = + python-dotenv PyGithub requests From 9c35f74c357dde94a6f6a8d9b7054d2b778fcc22 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 12:10:44 -0600 Subject: [PATCH 09/20] Load environment from env file --- .../workflows/container-trafficserver-alpine.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/container-trafficserver-alpine.yml b/.github/workflows/container-trafficserver-alpine.yml index 246f20611a..24ec72a57e 100644 --- a/.github/workflows/container-trafficserver-alpine.yml +++ b/.github/workflows/container-trafficserver-alpine.yml @@ -43,14 +43,11 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@master + - name: Load environment + run: cp .env "${{ github.env }}" - name: Build ${{ env.CONTAINER }} working-directory: .github/containers/trafficserver-alpine - run: | - set -o errexit -o nounset - while IFS= read -r line; do - export "$line" - done <${GITHUB_WORKSPACE}/.env - docker-compose build + run: docker-compose build - name: docker login ghcr.io uses: docker/login-action@v1 with: @@ -60,9 +57,4 @@ jobs: - name: Push ${{ env.CONTAINER }} if: ${{ github.event_name != 'pull_request' }} working-directory: .github/containers/trafficserver-alpine - run: | - set -o errexit -o nounset - while IFS= read -r line; do - export "$line" - done <${GITHUB_WORKSPACE}/.env - docker-compose push + run: docker-compose push From 02545c96d2a1c5e3c70a3cc7c250bbfc37604d2f Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 12:22:47 -0600 Subject: [PATCH 10/20] Include ATS version tag in step names --- .github/workflows/container-trafficserver-alpine.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-trafficserver-alpine.yml b/.github/workflows/container-trafficserver-alpine.yml index 24ec72a57e..da5c27a603 100644 --- a/.github/workflows/container-trafficserver-alpine.yml +++ b/.github/workflows/container-trafficserver-alpine.yml @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@master - name: Load environment run: cp .env "${{ github.env }}" - - name: Build ${{ env.CONTAINER }} + - name: Build ${{ env.CONTAINER }}:${{ env.ATS_VERSION }} working-directory: .github/containers/trafficserver-alpine run: docker-compose build - name: docker login ghcr.io @@ -54,7 +54,7 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Push ${{ env.CONTAINER }} + - name: Push ${{ env.CONTAINER }}:${{ env.ATS_VERSION }} if: ${{ github.event_name != 'pull_request' }} working-directory: .github/containers/trafficserver-alpine run: docker-compose push From 137bb0a23955f2452e932b973d43c666c948b837 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 12:51:30 -0600 Subject: [PATCH 11/20] Reuse arguments with dict double pointer --- .../pr_to_update_go/go_pr_maker.py | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index ff3d12711e..184091ac84 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -377,27 +377,22 @@ def set_go_version(self, go_version: str, commit_message: str, go_version_file = getenv(ENV_GO_VERSION_FILE) content = f"{go_version}\n" sha = self.file_contents(go_version_file, source_branch_name).sha + kwargs = { + 'branch': source_branch_name, + 'content': content, + 'path': go_version_file, + 'message': commit_message, + 'sha': sha + } if self.author: - self.repo.update_file( - author=self.author, - branch=source_branch_name, - content=content, - path=go_version_file, - message=commit_message, - sha=sha - ) + kwargs['author'] = self.author + kwargs['committer'] = self.author else: print('Committing using the default author') - self.repo.update_file( - branch=source_branch_name, - content=content, - path=go_version_file, - message=commit_message, - sha=sha - ) - + self.repo.update_file(**kwargs) print(f'Updated {go_version_file} on {self.repo.name}') - env_path = os.path.join(os.path.dirname(getenv(ENV_ENV_FILE)), ".env") + env_file = getenv(ENV_ENV_FILE) + env_path = os.path.join(os.path.dirname(env_file), ".env") sha = self.file_contents(env_path, source_branch_name).sha with open(env_path, encoding='UTF-8') as env_stream: previous_env_content = env_stream.read() @@ -405,13 +400,13 @@ def set_go_version(self, go_version: str, commit_message: str, quote_mode='never') with open(env_path, encoding='UTF-8') as env_stream: env_content = env_stream.read() - commit = self.repo.update_file( - branch=source_branch_name, - content=env_content, - path=env_path, - message=commit_message, - sha=sha - )["commit"] + kwargs.update({ + 'content': env_content, + 'path': env_path, + 'message': kwargs['message'] + ' in ' + env_file, + 'sha': sha + }) + commit = self.repo.update_file(**kwargs)["commit"] if not isinstance(commit, Commit): raise TypeError("'commit' property of file update response was not a Commit") print(f"Updated {env_path} on {self.repo.name}") @@ -448,20 +443,17 @@ def update_golang_org_x(self, previous_commit: Commit) -> Optional[GitCommit]: commit_message: str = f'Update golang.org/x/ dependencies for go{self.latest_go_version}' previous_git_commit = self.repo.get_git_commit(previous_commit.sha) git_commit: GitCommit + kwargs = { + 'message': commit_message, + 'tree': tree, + 'parents': [previous_git_commit], + 'author': self.author, + 'committer': self.author, + } if self.author: - git_commit = self.repo.create_git_commit( - message=commit_message, - tree=tree, - parents=[previous_git_commit], - author=self.author, - committer=self.author - ) - else: - git_commit = self.repo.create_git_commit( - message=commit_message, - tree=tree, - parents=[previous_git_commit] - ) + kwargs['author'] = self.author + kwargs['committer'] = self.author + git_commit = self.repo.create_git_commit(**kwargs) print('Updated golang.org/x/ dependencies') return git_commit From 7256b2bc551920aaf4926973b3932ba46e9136ae Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 14:58:10 -0600 Subject: [PATCH 12/20] Update GO_VERSION and .env files in single commit --- .../pr_to_update_go/go_pr_maker.py | 95 ++++++++----------- 1 file changed, 37 insertions(+), 58 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index 184091ac84..33fd037e50 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -24,7 +24,7 @@ import re import subprocess import sys -from typing import Optional, TypedDict, Any +from typing import Optional, TypedDict, Any, Union import dotenv import requests @@ -281,22 +281,20 @@ def run(self, update_version_only: bool = False) -> None: print(f'Go version is up-to-date on {target_branch}, nothing to do.') return - commit: Optional[Commit] = None + commit: Optional[GitCommit] = None if not self.branch_exists(source_branch_name): commit = self.set_go_version(self.latest_go_version, commit_message, source_branch_name) if commit is None: source_branch_ref: GitRef = self.repo.get_git_ref(f'heads/{source_branch_name}') - commit = self.repo.get_commit(source_branch_ref.object.sha) + commit = self.repo.get_git_commit(source_branch_ref.object.sha) subprocess.run(['git', 'fetch', 'origin'], check=True) subprocess.run(['git', 'checkout', commit.sha], check=True) if update_version_only: print(f'Branch {source_branch_name} has been created, exiting...') return - update_golang_org_x_commit = self.update_golang_org_x(commit) - if update_golang_org_x_commit: - self.update_branch(source_branch_name, update_golang_org_x_commit.sha) + self.update_golang_org_x(commit, source_branch_name) self.create_pr( self.latest_go_version, @@ -360,7 +358,7 @@ def get_repo_go_version(self, branch: str = 'master') -> str: return self.file_contents(getenv(ENV_GO_VERSION_FILE), branch).decoded_content.decode().strip() def set_go_version(self, go_version: str, commit_message: str, - source_branch_name: str) -> Commit: + source_branch_name: str) -> GitCommit: """ Makes the commits necessary to change the Go version used by the repository. @@ -368,61 +366,28 @@ def set_go_version(self, go_version: str, commit_message: str, This includes updating the GO_VERSION and .env files at the repository's root. """ - master = self.repo.get_branch('master') - sha = master.commit.sha + master_tip = self.repo.get_branch('master').commit + sha = master_tip.sha ref = f'refs/heads/{source_branch_name}' self.repo.create_git_ref(ref, sha) - print(f'Created branch {source_branch_name}') + go_version_file = getenv(ENV_GO_VERSION_FILE) - content = f"{go_version}\n" - sha = self.file_contents(go_version_file, source_branch_name).sha - kwargs = { - 'branch': source_branch_name, - 'content': content, - 'path': go_version_file, - 'message': commit_message, - 'sha': sha - } - if self.author: - kwargs['author'] = self.author - kwargs['committer'] = self.author - else: - print('Committing using the default author') - self.repo.update_file(**kwargs) - print(f'Updated {go_version_file} on {self.repo.name}') + with open(go_version_file, 'w') as go_version_file_stream: + go_version_file_stream.write(f'{go_version}\n') env_file = getenv(ENV_ENV_FILE) env_path = os.path.join(os.path.dirname(env_file), ".env") - sha = self.file_contents(env_path, source_branch_name).sha - with open(env_path, encoding='UTF-8') as env_stream: - previous_env_content = env_stream.read() dotenv.set_key(dotenv_path=env_path, key_to_set=GO_VERSION_KEY, value_to_set=go_version, quote_mode='never') - with open(env_path, encoding='UTF-8') as env_stream: - env_content = env_stream.read() - kwargs.update({ - 'content': env_content, - 'path': env_path, - 'message': kwargs['message'] + ' in ' + env_file, - 'sha': sha - }) - commit = self.repo.update_file(**kwargs)["commit"] - if not isinstance(commit, Commit): - raise TypeError("'commit' property of file update response was not a Commit") - print(f"Updated {env_path} on {self.repo.name}") - with open(env_path, encoding='UTF-8', mode='w') as env_stream: - env_stream.write(previous_env_content) - return commit - - def update_golang_org_x(self, previous_commit: Commit) -> Optional[GitCommit]: + return self.update_files_on_tree(head=master_tip, files_to_check=[go_version_file, + env_file], commit_message=commit_message, source_branch_name=source_branch_name) + + def update_files_on_tree(self, head: Union[Commit, GitCommit], files_to_check: list[str], + commit_message: str, source_branch_name: + str) -> Optional[GitCommit]: """ - Updates golang.org/x/ Go dependencies as necessary for the new Go - version. + Commits multiple files in a single Git commit, then reverts those changes locally. """ - subprocess.run(['git', 'fetch', 'origin'], check=True) - subprocess.run(['git', 'checkout', previous_commit.sha], check=True) - subprocess.run([os.path.join(os.path.dirname(__file__), 'update_golang_org_x.sh')], check=True) - files_to_check = ['go.mod', 'go.sum', os.path.join('vendor', 'modules.txt')] tree_elements: list[InputGitTreeElement] = [] for file in files_to_check: diff_process = subprocess.run(['git', 'diff', '--exit-code', '--', file], check=False) @@ -433,20 +398,18 @@ def update_golang_org_x(self, previous_commit: Commit) -> Optional[GitCommit]: tree_element: InputGitTreeElement = InputGitTreeElement(path=file, mode='100644', type='blob', content=content) tree_elements.append(tree_element) + subprocess.run(['git', 'checkout', '--', file], check=True) if len(tree_elements) == 0: - print('No golang.org/x/ dependencies need to be updated.') + print('No files need to be updated.') return None tree_hash = subprocess.check_output( - ['git', 'log', '-1', '--pretty=%T', previous_commit.sha]).decode().strip() + ['git', 'log', '-1', '--pretty=%T', head.sha]).decode().strip() base_tree = self.repo.get_git_tree(sha=tree_hash) tree = self.repo.create_git_tree(tree_elements, base_tree) - commit_message: str = f'Update golang.org/x/ dependencies for go{self.latest_go_version}' - previous_git_commit = self.repo.get_git_commit(previous_commit.sha) - git_commit: GitCommit kwargs = { 'message': commit_message, 'tree': tree, - 'parents': [previous_git_commit], + 'parents': [self.repo.get_git_commit(head.sha)], 'author': self.author, 'committer': self.author, } @@ -454,6 +417,22 @@ def update_golang_org_x(self, previous_commit: Commit) -> Optional[GitCommit]: kwargs['author'] = self.author kwargs['committer'] = self.author git_commit = self.repo.create_git_commit(**kwargs) + self.update_branch(source_branch_name, git_commit.sha) + return git_commit + + def update_golang_org_x(self, head: GitCommit, source_branch_name: str) -> Optional[GitCommit]: + """ + Updates golang.org/x/ Go dependencies as necessary for the new Go + version. + """ + subprocess.run(['git', 'fetch', 'origin'], check=True) + subprocess.run(['git', 'checkout', head.sha], check=True) + subprocess.run([os.path.join(os.path.dirname(__file__), 'update_golang_org_x.sh')], check=True) + + commit_message: str = f'Update golang.org/x/ dependencies for go{self.latest_go_version}' + git_commit = self.update_files_on_tree(head=head, files_to_check=['go.mod', 'go.sum', + os.path.join('vendor', 'modules.txt')], commit_message=commit_message, + source_branch_name=source_branch_name) print('Updated golang.org/x/ dependencies') return git_commit From 1210ef19fb8749d3aa689d0e8772a5dde3cf41e8 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 29 Mar 2022 15:00:13 -0600 Subject: [PATCH 13/20] Reformat file --- .../pr_to_update_go/go_pr_maker.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index 33fd037e50..729d68af5d 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -53,6 +53,7 @@ GO_VERSION_KEY, ) + class GoVersion(TypedDict): """ A single entry in the list returned by the Go website's version listing API. @@ -62,6 +63,7 @@ class GoVersion(TypedDict): stable: bool version: str + def _get_pr_body(go_version: str, milestone_url: str) -> str: """ Generates the body of a Pull Request given a Go release version and a @@ -77,6 +79,7 @@ def _get_pr_body(go_version: str, milestone_url: str) -> str: print('Templated PR body') return pr_body + def get_major_version(from_go_version: str) -> str: """ Extracts the "major" version part of a full Go release version. ("major" to @@ -95,6 +98,7 @@ def get_major_version(from_go_version: str) -> str: return match.group(0) return "" + def getenv(var: str) -> str: """ Returns the value of the environment variable with the given name. @@ -107,6 +111,7 @@ def getenv(var: str) -> str: """ return os.environ[var] + def parse_release_notes(version: str, content: str) -> str: """ Parses Go version release notes. @@ -139,7 +144,7 @@ def parse_release_notes(version: str, content: str) -> str: """ go_version_pattern = version.replace('.', r"\.") release_notes_pattern = re.compile( - r"

\s*\n\s*go"+go_version_pattern+r".*?

", + r"

\s*\n\s*go" + go_version_pattern + r".*?

", re.MULTILINE | re.DOTALL ) matches = release_notes_pattern.search(content) @@ -147,13 +152,15 @@ def parse_release_notes(version: str, content: str) -> str: raise Exception(f'could not find release notes for Go {version}') return " ".join(matches.group(0).split()) + def _get_release_notes(go_version: str) -> str: """ Gets the release notes for the given Go version. """ release_history_response = requests.get(RELEASE_PAGE_URL) release_history_response.raise_for_status() - return parse_release_notes(go_version, release_history_response .content.decode()) + return parse_release_notes(go_version, release_history_response.content.decode()) + def find_latest_major_upgrade(major_version: str, versions: list[GoVersion]) -> str: """ @@ -202,6 +209,7 @@ def find_latest_major_upgrade(major_version: str, versions: list[GoVersion]) -> raise Exception(f'no supported {major_version} Go versions exist') + def _get_latest_major_upgrade(from_go_version: str) -> str: """ Gets the version of the latest Go release that is the same "major" @@ -218,6 +226,7 @@ def _get_latest_major_upgrade(from_go_version: str) -> str: print(f'Latest version of Go {major_version} is {fetched_go_version}') return fetched_go_version + class GoPRMaker: """ A class to generate pull requests for the purpose of updating the Go version @@ -355,7 +364,8 @@ def get_repo_go_version(self, branch: str = 'master') -> str: Gets the current Go version used at the head of the given branch (or not given to use "master" by default) for the repository. """ - return self.file_contents(getenv(ENV_GO_VERSION_FILE), branch).decoded_content.decode().strip() + return self.file_contents(getenv(ENV_GO_VERSION_FILE), + branch).decoded_content.decode().strip() def set_go_version(self, go_version: str, commit_message: str, source_branch_name: str) -> GitCommit: @@ -384,7 +394,7 @@ def set_go_version(self, go_version: str, commit_message: str, def update_files_on_tree(self, head: Union[Commit, GitCommit], files_to_check: list[str], commit_message: str, source_branch_name: - str) -> Optional[GitCommit]: + str) -> Optional[GitCommit]: """ Commits multiple files in a single Git commit, then reverts those changes locally. """ @@ -427,7 +437,8 @@ def update_golang_org_x(self, head: GitCommit, source_branch_name: str) -> Optio """ subprocess.run(['git', 'fetch', 'origin'], check=True) subprocess.run(['git', 'checkout', head.sha], check=True) - subprocess.run([os.path.join(os.path.dirname(__file__), 'update_golang_org_x.sh')], check=True) + subprocess.run([os.path.join(os.path.dirname(__file__), 'update_golang_org_x.sh')], + check=True) commit_message: str = f'Update golang.org/x/ dependencies for go{self.latest_go_version}' git_commit = self.update_files_on_tree(head=head, files_to_check=['go.mod', 'go.sum', @@ -447,7 +458,8 @@ def create_pr(self, latest_go_version: str, commit_message: str, owner: str, pull_request = self.repo.get_pull(list_item.number) if pull_request.head.ref != source_branch_name: continue - print(f'Pull request for branch {source_branch_name} already exists:\n{pull_request.html_url}') + print( + f'Pull request for branch {source_branch_name} already exists:\n{pull_request.html_url}') return milestone_url = self.get_go_milestone(latest_go_version) @@ -466,6 +478,8 @@ def create_pr(self, latest_go_version: str, commit_message: str, owner: str, print('Unable to find a label named "go version"', file=sys.stderr) print(f'Created pull request {pull_request.html_url}') + if __name__ == "__main__": import doctest + doctest.testmod() From 99e26e5c07fee30c5f6286fab9fe15ce3fbb11cf Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 30 Mar 2022 14:53:47 -0600 Subject: [PATCH 14/20] Re-add make dependency --- dev/t3c/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dev/t3c/Dockerfile b/dev/t3c/Dockerfile index 5653df5f0c..411dc029fb 100644 --- a/dev/t3c/Dockerfile +++ b/dev/t3c/Dockerfile @@ -28,7 +28,13 @@ VOLUME $TC EXPOSE 80 8081 USER root -RUN apk add --no-cache inotify-tools gcc musl-dev && \ +RUN apk add --no-cache \ + # inotify-tools is used for inotifywait in run.sh + inotify-tools \ + # make is used for the t3c Makefile + make \ + # gcc and musl-dev are used to build packages using CGO + gcc musl-dev && \ go install github.com/go-delve/delve/cmd/dlv@latest RUN echo "stats_over_http.so" >> /etc/trafficserver/plugin.config && echo "system_stats.so" >> /etc/trafficserver/plugin.config From 28c76c2c8c4ad13e53474e664824d21d0a276156 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 30 Mar 2022 20:08:11 -0600 Subject: [PATCH 15/20] Do not include author and committer if no author is provided --- .github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index 729d68af5d..8dd11403f8 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -420,8 +420,6 @@ def update_files_on_tree(self, head: Union[Commit, GitCommit], files_to_check: l 'message': commit_message, 'tree': tree, 'parents': [self.repo.get_git_commit(head.sha)], - 'author': self.author, - 'committer': self.author, } if self.author: kwargs['author'] = self.author From 167ca6db6e11d8c14142bad2e55f21b91dd51a39 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 31 Mar 2022 16:21:18 -0600 Subject: [PATCH 16/20] Make return type optional --- .github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index 8dd11403f8..fedf2a721c 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -368,7 +368,7 @@ def get_repo_go_version(self, branch: str = 'master') -> str: branch).decoded_content.decode().strip() def set_go_version(self, go_version: str, commit_message: str, - source_branch_name: str) -> GitCommit: + source_branch_name: str) -> Optional[GitCommit]: """ Makes the commits necessary to change the Go version used by the repository. From d733c56a148eed1d83ddaa4255228ef3b433047b Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 31 Mar 2022 16:23:01 -0600 Subject: [PATCH 17/20] Add to dict using dict.update() --- .github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index fedf2a721c..aabc22c3c8 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -422,8 +422,7 @@ def update_files_on_tree(self, head: Union[Commit, GitCommit], files_to_check: l 'parents': [self.repo.get_git_commit(head.sha)], } if self.author: - kwargs['author'] = self.author - kwargs['committer'] = self.author + kwargs.update({'author': self.author, 'committer': self.author}) git_commit = self.repo.create_git_commit(**kwargs) self.update_branch(source_branch_name, git_commit.sha) return git_commit From 190640b3a6e22e6d77ebd5b46d7ec003df6dc9b7 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 31 Mar 2022 16:23:22 -0600 Subject: [PATCH 18/20] Make env_path PathLike --- .github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index aabc22c3c8..81d245880f 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -24,6 +24,7 @@ import re import subprocess import sys +from pathlib import PurePath from typing import Optional, TypedDict, Any, Union import dotenv @@ -386,7 +387,7 @@ def set_go_version(self, go_version: str, commit_message: str, with open(go_version_file, 'w') as go_version_file_stream: go_version_file_stream.write(f'{go_version}\n') env_file = getenv(ENV_ENV_FILE) - env_path = os.path.join(os.path.dirname(env_file), ".env") + env_path = PurePath(os.path.dirname(env_file), ".env") dotenv.set_key(dotenv_path=env_path, key_to_set=GO_VERSION_KEY, value_to_set=go_version, quote_mode='never') return self.update_files_on_tree(head=master_tip, files_to_check=[go_version_file, From 3070bf9cfb5944056a731a5fff9e47c72e992164 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 1 Apr 2022 11:27:45 -0600 Subject: [PATCH 19/20] Do not use more than one type in kwargs --- .../pr-to-update-go/pr_to_update_go/go_pr_maker.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index 81d245880f..360857408f 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -417,14 +417,11 @@ def update_files_on_tree(self, head: Union[Commit, GitCommit], files_to_check: l ['git', 'log', '-1', '--pretty=%T', head.sha]).decode().strip() base_tree = self.repo.get_git_tree(sha=tree_hash) tree = self.repo.create_git_tree(tree_elements, base_tree) - kwargs = { - 'message': commit_message, - 'tree': tree, - 'parents': [self.repo.get_git_commit(head.sha)], - } + kwargs = {} if self.author: - kwargs.update({'author': self.author, 'committer': self.author}) - git_commit = self.repo.create_git_commit(**kwargs) + kwargs = {'author': self.author, 'committer': self.author} + git_commit = self.repo.create_git_commit(message=commit_message, tree=tree, + parents=[self.repo.get_git_commit(head.sha)], **kwargs) self.update_branch(source_branch_name, git_commit.sha) return git_commit From b1050897b22f5b53411988591b35ac223bf2a9f1 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 1 Apr 2022 11:47:24 -0600 Subject: [PATCH 20/20] Import set_key() directly --- .../actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py index 360857408f..25fda0d981 100644 --- a/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py +++ b/.github/actions/pr-to-update-go/pr_to_update_go/go_pr_maker.py @@ -27,8 +27,8 @@ from pathlib import PurePath from typing import Optional, TypedDict, Any, Union -import dotenv import requests +from dotenv import set_key from github.Commit import Commit from github.ContentFile import ContentFile @@ -388,7 +388,7 @@ def set_go_version(self, go_version: str, commit_message: str, go_version_file_stream.write(f'{go_version}\n') env_file = getenv(ENV_ENV_FILE) env_path = PurePath(os.path.dirname(env_file), ".env") - dotenv.set_key(dotenv_path=env_path, key_to_set=GO_VERSION_KEY, value_to_set=go_version, + set_key(dotenv_path=env_path, key_to_set=GO_VERSION_KEY, value_to_set=go_version, quote_mode='never') return self.update_files_on_tree(head=master_tip, files_to_check=[go_version_file, env_file], commit_message=commit_message, source_branch_name=source_branch_name)