From a8f5e7b9e7898407107cd91de5ba3f07af56c848 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 27 Nov 2024 23:57:08 +0000 Subject: [PATCH 1/2] Attempt to make version management sane --- .github/workflows/reusable_build.yml | 1 - cmake/autogenerated_versions.txt | 8 +++++--- cmake/version.cmake | 7 ++++--- tests/ci/build_check.py | 29 +++++++++++++++++++++------- tests/ci/git_helper.py | 16 ++++++++++++++- tests/ci/pr_info.py | 3 ++- tests/ci/version_helper.py | 16 ++------------- 7 files changed, 50 insertions(+), 30 deletions(-) diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index ebe85b72bdc0..c64d0aaec500 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -4,7 +4,6 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - CLICKHOUSE_STABLE_VERSION_SUFFIX: altinitystable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 6824302ea36a..46ab7b8bf48d 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -8,10 +8,12 @@ SET(VERSION_MINOR 8) SET(VERSION_PATCH 8) SET(VERSION_GITHASH e28553d4f2ba78643f9ef47b698954a2c54e6bcc) -SET(VERSION_TWEAK 18) +#1000 for altinitystable candidates +#2000 for altinityedge candidates +SET(VERSION_TWEAK 181000) SET(VERSION_FLAVOUR altinitystable) -SET(VERSION_DESCRIBE v24.8.8.18.altinitystable) -SET(VERSION_STRING 24.8.8.18.altinitystable) +SET(VERSION_DESCRIBE v24.8.8.181000.altinitystable) +SET(VERSION_STRING 24.8.8.181000.altinitystable) # end of autochange diff --git a/cmake/version.cmake b/cmake/version.cmake index 06fb783b88f2..b008c989c0b0 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -3,9 +3,10 @@ include(${PROJECT_SOURCE_DIR}/cmake/autogenerated_versions.txt) set(VERSION_EXTRA "" CACHE STRING "") set(VERSION_TWEAK "" CACHE STRING "") -if (VERSION_TWEAK) - string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_TWEAK}) -endif () +# NOTE(vnemkov): we rely on VERSION_TWEAK portion to be already present in VERSION_STRING +# if (VERSION_TWEAK) +# string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_TWEAK}) +# endif () if (VERSION_EXTRA) string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA}) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 17cf409293c7..c5b028ca5e87 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -12,12 +12,13 @@ from ci_config import CI from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY from git_helper import Git -from pr_info import PRInfo +from pr_info import PRInfo, EventType from report import FAILURE, SUCCESS, JobReport, StatusType from stopwatch import Stopwatch from tee_popen import TeePopen from version_helper import ( ClickHouseVersion, + VersionType, get_version_from_repo, update_version_local, ) @@ -164,16 +165,30 @@ def main(): version = get_version_from_repo(git=Git(True)) logging.info("Got version from repo %s", version.string) - official_flag = pr_info.number == 0 + # official_flag = pr_info.number == 0 - version_type = "testing" - if is_release_pr(pr_info): - version_type = "stable" - official_flag = True + # version_type = "testing" + # if is_release_pr(pr_info): + # version_type = "stable" + # official_flag = True + + # NOTE(vnemkov): For Altinity Stable builds, version flavor + # (last part of version, like 'altinitystable') is obtained from tag. + # If there is no tag, then version is considered to be 'testing' + version_type = version._flavour = VersionType.TESTING + + if pr_info.event_type == EventType.PUSH \ + and pr_info.ref.startswith('/ref/tags/'): + tag_name = pr_info.ref.removeprefix('/ref/tags/') + version_type = tag_name.split('.')[-1] + version._flavour = version_type + logging.info("Using version from tag: %s => %s", tag_name, version) + + # TODO(vnemkov): make sure tweak part is incremented by 1 each time we merge a PR update_version_local(version, version_type) - logging.info("Updated local files with version") + logging.info("Updated local files with version %s", version) logging.info("Build short name %s", build_name) diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 1de8fe6ef5b0..ca924a378a45 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -11,6 +11,20 @@ logger = logging.getLogger(__name__) +class VersionType: + LTS = "lts" + NEW = "new" + PRESTABLE = "altinityedge" + STABLE = "altinitystable" + TESTING = "altinitytest" + + VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS, + # NOTE (vnemkov): we don't use those directly, but it is used in unit-tests + "stable", + "prestable", + "testing", + ) + # ^ and $ match subline in `multiple\nlines` # \A and \Z match only start and end of the whole string # NOTE (vnemkov): support both upstream tag style: v22.x.y.z-lts and Altinity tag style: v22.x.y.z.altinitystable @@ -19,7 +33,7 @@ TAG_REGEXP = ( r"\Av\d{2}" # First two digits of major part r"([.][1-9]\d*){3}" # minor.patch.tweak parts - r"-(new|testing|prestable|stable|lts|altinitystable)\Z" # suffix with a version type + fr"[.-]({'|'.join(VersionType.VALID)})\Z" # suffix with a version type ) SHA_REGEXP = re.compile(r"\A([0-9]|[a-f]){40}\Z") diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 09a8cb563a1f..520786ad9107 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -132,8 +132,9 @@ def __init__( ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] + self.ref = ref # type: str e.g. "refs/pull/509/merge" or "refs/tags/v24.3.12.76.altinitystable" # Default values - self.base_ref = "" # type: str + self.base_ref = github_event.get("base_ref","") # type: str self.base_name = "" # type: str self.head_ref = "" # type: str self.head_name = "" # type: str diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 84ad3a50d87c..f3726ca3b6fa 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple, Union -from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix +from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix, VersionType FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt" CHANGELOG_IN_PATH = "debian/changelog.in" @@ -250,20 +250,8 @@ def __repr__(self): ClickHouseVersions = List[ClickHouseVersion] - -class VersionType: - LTS = "lts" - NEW = "new" - PRESTABLE = "prestable" - STABLE = "altinitystable" - TESTING = "testing" - VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS, - "stable" # NOTE (vnemkov): we don't use that directly, but it is used in unit-tests - ) - - def validate_version(version: str) -> None: - # NOTE(vnemkov): minor but imporant fixes, so versions with 'flavour' are threated as valid (e.g. 22.8.8.4.altinitystable) + # NOTE(vnemkov): minor but important fixes, so versions with 'flavour' are threated as valid (e.g. 22.8.8.4.altinitystable) parts = version.split(".") if len(parts) < 4: raise ValueError(f"{version} does not contain 4 parts") From 1555ef7826ad0a06eb7109d88031b3a5ea0111ac Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 28 Nov 2024 09:34:27 +0000 Subject: [PATCH 2/2] Fixed minor hiccup --- tests/ci/build_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index c5b028ca5e87..384435311069 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -176,6 +176,7 @@ def main(): # (last part of version, like 'altinitystable') is obtained from tag. # If there is no tag, then version is considered to be 'testing' version_type = version._flavour = VersionType.TESTING + official_flag = True if pr_info.event_type == EventType.PUSH \ and pr_info.ref.startswith('/ref/tags/'):