\s*\n\s*go"+go_version_pattern+r".*?
", + r"\s*\n\s*go" + go_version_pattern + r".*?
", re.MULTILINE | re.DOTALL ) matches = release_notes_pattern.search(content) @@ -145,13 +153,15 @@ def parse_release_notes(version: str, content: str) -> str: raise Exception(f'could not find release notes for Go {version}') return " ".join(matches.group(0).split()) + def _get_release_notes(go_version: str) -> str: """ Gets the release notes for the given Go version. """ release_history_response = requests.get(RELEASE_PAGE_URL) release_history_response.raise_for_status() - return parse_release_notes(go_version, release_history_response .content.decode()) + return parse_release_notes(go_version, release_history_response.content.decode()) + def find_latest_major_upgrade(major_version: str, versions: list[GoVersion]) -> str: """ @@ -200,6 +210,7 @@ def find_latest_major_upgrade(major_version: str, versions: list[GoVersion]) -> raise Exception(f'no supported {major_version} Go versions exist') + def _get_latest_major_upgrade(from_go_version: str) -> str: """ Gets the version of the latest Go release that is the same "major" @@ -216,6 +227,7 @@ def _get_latest_major_upgrade(from_go_version: str) -> str: print(f'Latest version of Go {major_version} is {fetched_go_version}') return fetched_go_version + class GoPRMaker: """ A class to generate pull requests for the purpose of updating the Go version @@ -279,22 +291,20 @@ def run(self, update_version_only: bool = False) -> None: print(f'Go version is up-to-date on {target_branch}, nothing to do.') return - commit: Optional[Commit] = None + commit: Optional[GitCommit] = None if not self.branch_exists(source_branch_name): commit = self.set_go_version(self.latest_go_version, commit_message, source_branch_name) if commit is None: source_branch_ref: GitRef = self.repo.get_git_ref(f'heads/{source_branch_name}') - commit = self.repo.get_commit(source_branch_ref.object.sha) + commit = self.repo.get_git_commit(source_branch_ref.object.sha) subprocess.run(['git', 'fetch', 'origin'], check=True) subprocess.run(['git', 'checkout', commit.sha], check=True) if update_version_only: print(f'Branch {source_branch_name} has been created, exiting...') return - update_golang_org_x_commit = self.update_golang_org_x(commit) - if update_golang_org_x_commit: - self.update_branch(source_branch_name, update_golang_org_x_commit.sha) + self.update_golang_org_x(commit, source_branch_name) self.create_pr( self.latest_go_version, @@ -355,10 +365,11 @@ def get_repo_go_version(self, branch: str = 'master') -> str: Gets the current Go version used at the head of the given branch (or not given to use "master" by default) for the repository. """ - return self.file_contents(getenv(ENV_GO_VERSION_FILE), branch).decoded_content.decode().strip() + return self.file_contents(getenv(ENV_GO_VERSION_FILE), + branch).decoded_content.decode().strip() def set_go_version(self, go_version: str, commit_message: str, - source_branch_name: str) -> Commit: + source_branch_name: str) -> Optional[GitCommit]: """ Makes the commits necessary to change the Go version used by the repository. @@ -366,59 +377,28 @@ def set_go_version(self, go_version: str, commit_message: str, This includes updating the GO_VERSION and .env files at the repository's root. """ - master = self.repo.get_branch('master') - sha = master.commit.sha + master_tip = self.repo.get_branch('master').commit + sha = master_tip.sha ref = f'refs/heads/{source_branch_name}' self.repo.create_git_ref(ref, sha) - print(f'Created branch {source_branch_name}') + go_version_file = getenv(ENV_GO_VERSION_FILE) - content = f"{go_version}\n" - sha = self.file_contents(go_version_file, source_branch_name).sha - if self.author: - self.repo.update_file( - author=self.author, - branch=source_branch_name, - content=content, - path=go_version_file, - message=commit_message, - sha=sha - ) - else: - print('Committing using the default author') - self.repo.update_file( - branch=source_branch_name, - content=content, - path=go_version_file, - message=commit_message, - sha=sha - ) - - print(f'Updated {go_version_file} on {self.repo.name}') - env_path = os.path.join(os.path.dirname(getenv(ENV_ENV_FILE)), ".env") - content = f"GO_VERSION={go_version}\n" - sha = self.file_contents(env_path, source_branch_name).sha - commit = self.repo.update_file( - branch=source_branch_name, - content=content, - path=env_path, - message=commit_message, - sha=sha - )["commit"] - if not isinstance(commit, Commit): - raise TypeError("'commit' property of file update response was not a Commit") - print(f"Updated {env_path} on {self.repo.name}") - return commit - - def update_golang_org_x(self, previous_commit: Commit) -> Optional[GitCommit]: + with open(go_version_file, 'w') as go_version_file_stream: + go_version_file_stream.write(f'{go_version}\n') + env_file = getenv(ENV_ENV_FILE) + env_path = PurePath(os.path.dirname(env_file), ".env") + set_key(dotenv_path=env_path, key_to_set=GO_VERSION_KEY, value_to_set=go_version, + quote_mode='never') + return self.update_files_on_tree(head=master_tip, files_to_check=[go_version_file, + env_file], commit_message=commit_message, source_branch_name=source_branch_name) + + def update_files_on_tree(self, head: Union[Commit, GitCommit], files_to_check: list[str], + commit_message: str, source_branch_name: + str) -> Optional[GitCommit]: """ - Updates golang.org/x/ Go dependencies as necessary for the new Go - version. + Commits multiple files in a single Git commit, then reverts those changes locally. """ - subprocess.run(['git', 'fetch', 'origin'], check=True) - subprocess.run(['git', 'checkout', previous_commit.sha], check=True) - subprocess.run([os.path.join(os.path.dirname(__file__), 'update_golang_org_x.sh')], check=True) - files_to_check = ['go.mod', 'go.sum', os.path.join('vendor', 'modules.txt')] tree_elements: list[InputGitTreeElement] = [] for file in files_to_check: diff_process = subprocess.run(['git', 'diff', '--exit-code', '--', file], check=False) @@ -429,30 +409,36 @@ def update_golang_org_x(self, previous_commit: Commit) -> Optional[GitCommit]: tree_element: InputGitTreeElement = InputGitTreeElement(path=file, mode='100644', type='blob', content=content) tree_elements.append(tree_element) + subprocess.run(['git', 'checkout', '--', file], check=True) if len(tree_elements) == 0: - print('No golang.org/x/ dependencies need to be updated.') + print('No files need to be updated.') return None tree_hash = subprocess.check_output( - ['git', 'log', '-1', '--pretty=%T', previous_commit.sha]).decode().strip() + ['git', 'log', '-1', '--pretty=%T', head.sha]).decode().strip() base_tree = self.repo.get_git_tree(sha=tree_hash) tree = self.repo.create_git_tree(tree_elements, base_tree) - commit_message: str = f'Update golang.org/x/ dependencies for go{self.latest_go_version}' - previous_git_commit = self.repo.get_git_commit(previous_commit.sha) - git_commit: GitCommit + kwargs = {} if self.author: - git_commit = self.repo.create_git_commit( - message=commit_message, - tree=tree, - parents=[previous_git_commit], - author=self.author, - committer=self.author - ) - else: - git_commit = self.repo.create_git_commit( - message=commit_message, - tree=tree, - parents=[previous_git_commit] - ) + kwargs = {'author': self.author, 'committer': self.author} + git_commit = self.repo.create_git_commit(message=commit_message, tree=tree, + parents=[self.repo.get_git_commit(head.sha)], **kwargs) + self.update_branch(source_branch_name, git_commit.sha) + return git_commit + + def update_golang_org_x(self, head: GitCommit, source_branch_name: str) -> Optional[GitCommit]: + """ + Updates golang.org/x/ Go dependencies as necessary for the new Go + version. + """ + subprocess.run(['git', 'fetch', 'origin'], check=True) + subprocess.run(['git', 'checkout', head.sha], check=True) + subprocess.run([os.path.join(os.path.dirname(__file__), 'update_golang_org_x.sh')], + check=True) + + commit_message: str = f'Update golang.org/x/ dependencies for go{self.latest_go_version}' + git_commit = self.update_files_on_tree(head=head, files_to_check=['go.mod', 'go.sum', + os.path.join('vendor', 'modules.txt')], commit_message=commit_message, + source_branch_name=source_branch_name) print('Updated golang.org/x/ dependencies') return git_commit @@ -467,7 +453,8 @@ def create_pr(self, latest_go_version: str, commit_message: str, owner: str, pull_request = self.repo.get_pull(list_item.number) if pull_request.head.ref != source_branch_name: continue - print(f'Pull request for branch {source_branch_name} already exists:\n{pull_request.html_url}') + print( + f'Pull request for branch {source_branch_name} already exists:\n{pull_request.html_url}') return milestone_url = self.get_go_milestone(latest_go_version) @@ -486,6 +473,8 @@ def create_pr(self, latest_go_version: str, commit_message: str, owner: str, print('Unable to find a label named "go version"', file=sys.stderr) print(f'Created pull request {pull_request.html_url}') + if __name__ == "__main__": import doctest + doctest.testmod() diff --git a/.github/actions/pr-to-update-go/requirements.txt b/.github/actions/pr-to-update-go/requirements.txt index 05cf2ead7c..f54204bd26 100644 --- a/.github/actions/pr-to-update-go/requirements.txt +++ b/.github/actions/pr-to-update-go/requirements.txt @@ -10,5 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # +python-dotenv PyGithub requests \ No newline at end of file diff --git a/.github/actions/pr-to-update-go/setup.cfg b/.github/actions/pr-to-update-go/setup.cfg index 939d0c83ca..2b6c0fa5bd 100644 --- a/.github/actions/pr-to-update-go/setup.cfg +++ b/.github/actions/pr-to-update-go/setup.cfg @@ -24,6 +24,7 @@ classifiers = OSI Approved :: Apache Software License python_requires = >=3.9 packages = pr_to_update_go install_requires = + python-dotenv PyGithub requests diff --git a/.github/containers/trafficserver-alpine/Dockerfile b/.github/containers/trafficserver-alpine/Dockerfile new file mode 100644 index 0000000000..4788c73f34 --- /dev/null +++ b/.github/containers/trafficserver-alpine/Dockerfile @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM alpine:latest AS build-trafficserver +ARG ATS_VERSION +ADD https://downloads.apache.org/trafficserver/trafficserver-${ATS_VERSION}.tar.bz2 /tmp/ +RUN set -o errexit -o nounset; \ + cd tmp; \ + dirname=trafficserver-${ATS_VERSION}; \ + tar xf ${dirname}.tar.bz2; \ + rm ${dirname}.tar.bz2; \ + apk add --no-cache \ + # configure dependencies + g++ \ + perl \ + openssl-dev \ + pcre-dev \ + make \ + # build dependencies + libexecinfo-dev \ + fortify-headers \ + linux-headers \ + zlib-dev; \ + cd $dirname; \ + ./configure \ + --disable-tests \ + --enable-experimental-plugins \ + --prefix=/ \ + --with-user=ats \ + --with-group=ats; \ + make -j; \ + adduser -D ats; \ + make install DESTDIR=/tmp/built; \ + cd ..; \ + rm -r $dirname + +FROM alpine:latest +COPY --from=build-trafficserver /tmp/built/ / +RUN apk add --no-cache \ + # runtime dependencies + libexecinfo \ + libstdc++ \ + pcre && \ + adduser -D ats +USER ats +CMD /bin/traffic_server diff --git a/.github/containers/trafficserver-alpine/docker-compose.yml b/.github/containers/trafficserver-alpine/docker-compose.yml new file mode 100644 index 0000000000..cb5e4d1936 --- /dev/null +++ b/.github/containers/trafficserver-alpine/docker-compose.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.9' +services: + trafficserver: + build: + context: . + dockerfile: Dockerfile + args: + ATS_VERSION: ${ATS_VERSION} + # for example, ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine:9.1.2 + image: ${CONTAINER}:${ATS_VERSION} diff --git a/.github/workflows/container-trafficserver-alpine.yml b/.github/workflows/container-trafficserver-alpine.yml new file mode 100644 index 0000000000..da5c27a603 --- /dev/null +++ b/.github/workflows/container-trafficserver-alpine.yml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Container ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine + +env: + DOCKER_BUILDKIT: '1' + COMPOSE_DOCKER_CLI_BUILD: '1' + CONTAINER: ghcr.io/${{ github.repository }}/ci/trafficserver-alpine + +on: + workflow_dispatch: # run manually + push: + paths: + - .env + - .github/containers/trafficserver-alpine/** + - .github/workflows/containers/trafficserver-alpine.yml + pull_request: + paths: + - .env + - .github/containers/trafficserver-alpine/** + - .github/workflows/containers/trafficserver-alpine.yml + types: [ opened, reopened, ready_for_review, synchronize ] + +jobs: + build: + if: ${{ github.repository_owner == 'apache' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@master + - name: Load environment + run: cp .env "${{ github.env }}" + - name: Build ${{ env.CONTAINER }}:${{ env.ATS_VERSION }} + working-directory: .github/containers/trafficserver-alpine + run: docker-compose build + - name: docker login ghcr.io + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push ${{ env.CONTAINER }}:${{ env.ATS_VERSION }} + if: ${{ github.event_name != 'pull_request' }} + working-directory: .github/containers/trafficserver-alpine + run: docker-compose push diff --git a/dev/t3c/Dockerfile b/dev/t3c/Dockerfile index 240b48f7ef..411dc029fb 100644 --- a/dev/t3c/Dockerfile +++ b/dev/t3c/Dockerfile @@ -14,37 +14,29 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +ARG ATS_VERSION ARG GO_VERSION -FROM alpine:latest AS traffic-server-builder - -RUN apk add --no-cache build-base perl libexecinfo-dev pcre-dev libressl-dev libtool linux-headers openssl-dev zlib-dev -ADD https://downloads.apache.org/trafficserver/trafficserver-9.1.1.tar.bz2 / -RUN tar -xf trafficserver-9.1.1.tar.bz2 && cd trafficserver-9.1.1 && mkdir /ats && ./configure --prefix / --enable-experimental-plugins && make -j && make install && ls -R /ats - -FROM golang:${GO_VERSION}-alpine AS t3c-dev +FROM golang:${GO_VERSION}-alpine AS get-go +FROM ghcr.io/apache/trafficcontrol/ci/trafficserver-alpine:${ATS_VERSION} +COPY --from=get-go /usr/local/go /usr/local/go +ENV PATH=/usr/local/go/bin:${PATH} \ + GOPATH=/go +ENV PATH=${GOPATH}/bin:${PATH} ENV TC="/root/go/src/github.com/apache/trafficcontrol/" GOFLAGS="--buildvcs=false" -VOLUME /root/go/src/github.com/apache/trafficcontrol +VOLUME $TC EXPOSE 80 8081 -COPY --from=traffic-server-builder /bin/traffic_cache_tool /bin/traffic_crashlog /bin/traffic_ctl /bin/traffic_layout /bin/traffic_logcat /bin/traffic_logstats /bin/traffic_manager /bin/traffic_server /bin/traffic_via /bin/trafficserver /bin/tspush /bin/tsxs /bin/ -COPY --from=traffic-server-builder /etc/trafficserver /etc/trafficserver -COPY --from=traffic-server-builder /include/ts /include/ts -COPY --from=traffic-server-builder /include/tscpp /include/tscpp -COPY --from=traffic-server-builder /lib/perl5 /lib/perl5 -COPY --from=traffic-server-builder /lib/pkgconfig/trafficserver.pc /lib/pkgconfig/trafficserver.pc -COPY --from=traffic-server-builder /lib/libtscore.la /lib/libtscore.so /lib/libtscore.so.9 /lib/libtscore.so.9.1.1 /lib/libtscppapi.la /lib/libtscppapi.so /lib/libtscppapi.so.9 /lib/libtscppapi.so.9.1.1 /lib/libtscpputil.la /lib/libtscpputil.so /lib/libtscpputil.so.9 /lib/libtscpputil.so.9.1.1 /lib/libtsmgmt.la /lib/libtsmgmt.so /lib/libtsmgmt.so.9 /lib/libtsmgmt.so.9.1.1 /lib/plugin_init_fail.la /lib/plugin_init_fail.so /lib/plugin_instinit_fail.la /lib/plugin_instinit_fail.so /lib/plugin_missing_deleteinstance.la /lib/plugin_missing_deleteinstance.so /lib/plugin_missing_doremap.la /lib/plugin_missing_doremap.so /lib/plugin_missing_init.la /lib/plugin_missing_init.so /lib/plugin_missing_newinstance.la /lib/plugin_missing_newinstance.so /lib/plugin_required_cb.la /lib/plugin_required_cb.so /lib/plugin_testing_calls.la /lib/plugin_testing_calls.so /lib/plugin_v1.la /lib/plugin_v1.so /lib/plugin_v2.la /lib/plugin_v2.so /lib/ -COPY --from=traffic-server-builder /libexec/trafficserver /libexec/trafficserver -COPY --from=traffic-server-builder /share/man /share/man -RUN mkdir /share/trafficserver && mkdir -p /var/log/trafficserver && mkdir /var/trafficserver - -RUN apk add --no-cache make inotify-tools gcc libc-dev pcre libexecinfo && \ - go install github.com/go-delve/delve/cmd/dlv@latest && \ - ln -s /root/go/bin/dlv /usr/bin/dlv && \ - mkdir /lib64 && \ - ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 +USER root +RUN apk add --no-cache \ + # inotify-tools is used for inotifywait in run.sh + inotify-tools \ + # make is used for the t3c Makefile + make \ + # gcc and musl-dev are used to build packages using CGO + gcc musl-dev && \ + go install github.com/go-delve/delve/cmd/dlv@latest -RUN addgroup ats && adduser -S -s /bin/ash -G ats ats && chown -R ats:ats /etc/trafficserver /var/trafficserver /share/trafficserver /var/log/trafficserver RUN echo "stats_over_http.so" >> /etc/trafficserver/plugin.config && echo "system_stats.so" >> /etc/trafficserver/plugin.config CMD /root/go/src/github.com/apache/trafficcontrol/dev/t3c/run.sh diff --git a/docker-compose.yml b/docker-compose.yml index fb13702bd0..639f7307c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,6 +95,7 @@ services: build: context: dev/t3c args: + - ATS_VERSION=${ATS_VERSION} - GO_VERSION=${GO_VERSION} depends_on: - trafficops