From 745dd692380059477266fa8d51cb8531186284cc Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 28 Nov 2024 19:14:51 +0000 Subject: [PATCH 1/6] Disabled getting AZURE_CONNECTION_STRING from SSM To allow Stateless and Statefull tests to start --- tests/ci/functional_test_check.py | 5 +++-- tests/config/config.d/azure_storage_conf.xml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 744ec0a715cf..e340a43cb49d 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -52,8 +52,9 @@ def get_additional_envs( check_name: str, run_by_hash_num: int, run_by_hash_total: int ) -> List[str]: result = [] - azure_connection_string = get_parameter_from_ssm("azure_connection_string") - result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") + # TODO(vnemkov): put proper connection string into SSM and re-enable this one + # azure_connection_string = get_parameter_from_ssm("azure_connection_string") + # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") if "DatabaseReplicated" in check_name: result.append("USE_DATABASE_REPLICATED=1") if "DatabaseOrdinary" in check_name: diff --git a/tests/config/config.d/azure_storage_conf.xml b/tests/config/config.d/azure_storage_conf.xml index f24b62b87b10..de416824b37d 100644 --- a/tests/config/config.d/azure_storage_conf.xml +++ b/tests/config/config.d/azure_storage_conf.xml @@ -7,7 +7,7 @@ false 33554432 openbucketforpublicci - + cache From 479c3a1737bb7a506958591ffbe7c617e82714ae Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 28 Nov 2024 19:40:23 +0000 Subject: [PATCH 2/6] Also for stress test --- tests/ci/functional_test_check.py | 2 +- tests/ci/stress_check.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index e340a43cb49d..e3451f8f541a 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -52,7 +52,7 @@ def get_additional_envs( check_name: str, run_by_hash_num: int, run_by_hash_total: int ) -> List[str]: result = [] - # TODO(vnemkov): put proper connection string into SSM and re-enable this one + # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one # azure_connection_string = get_parameter_from_ssm("azure_connection_string") # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") if "DatabaseReplicated" in check_name: diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 9f1f36a5db28..94991a60a3bf 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -34,8 +34,9 @@ def format(self, record): def get_additional_envs(check_name: str) -> List[str]: result = [] - azure_connection_string = get_parameter_from_ssm("azure_connection_string") - result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") + # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one + # azure_connection_string = get_parameter_from_ssm("azure_connection_string") + # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") # some cloud-specific features require feature flags enabled # so we need this ENV to be able to disable the randomization # of feature flags From 2c31b57c89d1e2e6e7feda04506a1a21cb073128 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 29 Nov 2024 10:21:26 +0000 Subject: [PATCH 3/6] Fixed CH startup --- tests/config/config.d/azure_storage_conf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/config.d/azure_storage_conf.xml b/tests/config/config.d/azure_storage_conf.xml index de416824b37d..f24b62b87b10 100644 --- a/tests/config/config.d/azure_storage_conf.xml +++ b/tests/config/config.d/azure_storage_conf.xml @@ -7,7 +7,7 @@ false 33554432 openbucketforpublicci - + cache From adec54911440786cd5cc355b3f64d5c7e26f8736 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 5 Dec 2024 18:34:12 +0000 Subject: [PATCH 4/6] Revert "Merge pull request #62565 from ClickHouse/ci_add_azure_tests" This reverts commit 9c3d40749ba20ee37dbc353ac6a691aee6d01ffa, reversing changes made to f6d443c927806e21cec1c7caead833deb3cdaee7. This is to temporarily switch away from real AZURE to minio --- docker/test/stateful/run.sh | 3 +-- docker/test/stateless/run.sh | 6 ++++++ docker/test/stress/run.sh | 1 + tests/ci/functional_test_check.py | 4 ---- tests/ci/stress_check.py | 5 +---- tests/config/config.d/azure_storage_conf.xml | 7 +++++-- tests/config/config.d/azure_storage_policy_by_default.xml | 5 ----- 7 files changed, 14 insertions(+), 17 deletions(-) delete mode 100644 tests/config/config.d/azure_storage_policy_by_default.xml diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index c072eeb0fa8b..af578423b043 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -22,8 +22,7 @@ source /utils.lib # install test configs /usr/share/clickhouse-test/config/install.sh -azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & - +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & ./setup_minio.sh stateful ./mc admin trace clickminio > /test_output/minio.log & MC_ADMIN_PID=$! diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index ad0cd321cc55..13d79243713b 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -50,6 +50,12 @@ source /utils.lib # install test configs /usr/share/clickhouse-test/config/install.sh +if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then + echo "Azure is disabled" +else + azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & +fi + ./setup_minio.sh stateless ./setup_hdfs_minicluster.sh diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index b21114e456f4..48348aa131ca 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -55,6 +55,7 @@ export ZOOKEEPER_FAULT_INJECTION=1 # available for dump via clickhouse-local configure +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & ./setup_minio.sh stateless # to have a proper environment config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index e3451f8f541a..0fd8ba4dd638 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -16,7 +16,6 @@ from docker_images_helper import DockerImage, get_docker_image, pull_image from download_release_packages import download_last_release from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH -from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import ( ERROR, @@ -68,9 +67,6 @@ def get_additional_envs( result.append("RANDOMIZE_OBJECT_KEY_TYPE=1") if "analyzer" in check_name: result.append("USE_OLD_ANALYZER=1") - if "azure" in check_name: - assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in result - result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1") if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 94991a60a3bf..c9b6fc28d5f2 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -13,7 +13,6 @@ from clickhouse_helper import CiLogsCredentials from docker_images_helper import DockerImage, get_docker_image, pull_image from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH -from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import ERROR, JobReport, TestResults, read_test_results from stopwatch import Stopwatch @@ -41,8 +40,6 @@ def get_additional_envs(check_name: str) -> List[str]: # so we need this ENV to be able to disable the randomization # of feature flags result.append("RANDOMIZE_KEEPER_FEATURE_FLAGS=1") - if "azure" in check_name: - result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1") if "s3" in check_name: result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") @@ -168,7 +165,7 @@ def run_stress_test(docker_image_name: str) -> None: pr_info, stopwatch.start_time_str, check_name ) - additional_envs = get_additional_envs(check_name) + additional_envs = get_additional_envs() run_command = get_run_command( packages_path, diff --git a/tests/config/config.d/azure_storage_conf.xml b/tests/config/config.d/azure_storage_conf.xml index f24b62b87b10..412d40111a70 100644 --- a/tests/config/config.d/azure_storage_conf.xml +++ b/tests/config/config.d/azure_storage_conf.xml @@ -4,10 +4,13 @@ object_storage azure + http://localhost:10000/devstoreaccount1 + cont false + + devstoreaccount1 + Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== 33554432 - openbucketforpublicci - cache diff --git a/tests/config/config.d/azure_storage_policy_by_default.xml b/tests/config/config.d/azure_storage_policy_by_default.xml deleted file mode 100644 index cab8a106f1b8..000000000000 --- a/tests/config/config.d/azure_storage_policy_by_default.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - azure_cache - - From 51522b818754ae647e38ccdea975f1bad479c641 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sat, 7 Dec 2024 17:11:33 +0000 Subject: [PATCH 5/6] Udpdated azurite version --- cmake/autogenerated_versions.txt | 1 - docker/test/stateless/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index d91c075cc557..6824302ea36a 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -14,5 +14,4 @@ SET(VERSION_FLAVOUR altinitystable) SET(VERSION_DESCRIBE v24.8.8.18.altinitystable) SET(VERSION_STRING 24.8.8.18.altinitystable) ->>>>>>> altinity/customizations/24.8.7 # end of autochange diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 71a68df7e0b7..7e57ad900a49 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -87,7 +87,7 @@ ENV MINIO_ROOT_PASSWORD="clickhouse" ENV EXPORT_S3_STORAGE_POLICIES=1 ENV CLICKHOUSE_GRPC_CLIENT="/usr/share/clickhouse-utils/grpc-client/clickhouse-grpc-client.py" -RUN npm install -g azurite@3.30.0 \ +RUN npm install -g azurite@^3.33.0 \ && npm install -g tslib && npm install -g node COPY run.sh / From 9883f8824150872488da5767cb81adf95f5b10e0 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 9 Dec 2024 08:23:32 +0000 Subject: [PATCH 6/6] Pushing events to proper database --- tests/ci/ci.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 27fa0b104c01..465820a7488f 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -388,7 +388,7 @@ def _pre_action(s3, job_name, batch, indata, pr_info): _get_ext_check_name(job_name), ) ClickHouseHelper().insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) print(f"Pre action done. Report files [{reports_files}] have been downloaded") @@ -966,7 +966,7 @@ def _add_build_to_version_history( print(f"::notice ::Log Adding record to versions history: {data}") - ch_helper.insert_event_into(db="default", table="version_history", event=data) + ch_helper.insert_event_into(db="gh-data", table="version_history", event=data) def _run_test(job_name: str, run_command: str) -> int: @@ -1376,7 +1376,7 @@ def main() -> int: _get_ext_check_name(args.job_name), ) ClickHouseHelper().insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) ### POST action: end