From 5c3e003eec1749aab61ee31f7d7fb6348998b933 Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Fri, 22 Nov 2024 10:27:57 +0800 Subject: [PATCH 1/5] [chore](check) Add code check for some mow code --- be/src/cloud/cloud_cumulative_compaction.cpp | 2 ++ be/src/cloud/cloud_cumulative_compaction.h | 2 ++ .../cloud_cumulative_compaction_policy.cpp | 2 ++ .../cloud_cumulative_compaction_policy.h | 2 ++ be/src/cloud/cloud_delete_bitmap_action.cpp | 2 ++ be/src/cloud/cloud_delete_bitmap_action.h | 2 ++ .../cloud_engine_calc_delete_bitmap_task.cpp | 2 ++ be/src/cloud/cloud_meta_mgr.cpp | 2 ++ be/src/cloud/cloud_meta_mgr.h | 2 ++ be/src/cloud/cloud_storage_engine.cpp | 2 ++ be/src/cloud/cloud_tablet.cpp | 2 ++ be/src/cloud/config.cpp | 2 ++ be/src/cloud/config.h | 2 ++ be/src/common/compile_check_begin.h | 4 +-- be/src/common/compile_check_end.h | 2 +- be/src/olap/rowset/beta_rowset_writer.cpp | 2 ++ be/src/olap/tablet_meta.cpp | 2 ++ be/src/olap/tablet_meta.h | 2 ++ be/src/service/http_service.cpp | 2 ++ cloud/src/common/compile_check_begin.h | 30 +++++++++++++++++++ cloud/src/common/compile_check_end.h | 23 ++++++++++++++ cloud/src/meta-service/meta_service.cpp | 2 ++ cloud/src/meta-service/meta_service.h | 2 ++ cloud/src/meta-service/meta_service_helper.h | 2 ++ 24 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 cloud/src/common/compile_check_begin.h create mode 100644 cloud/src/common/compile_check_end.h diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index c466c35e2a2ab6..72264178173a67 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -33,6 +33,7 @@ #include "util/uuid_generator.h" namespace doris { +#include "common/compile_check_begin.h" using namespace ErrorCode; bvar::Adder cumu_output_size("cumu_compaction", "output_size"); @@ -621,4 +622,5 @@ void CloudCumulativeCompaction::do_lease() { } } +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_cumulative_compaction.h b/be/src/cloud/cloud_cumulative_compaction.h index 1159dcb59ceef1..87fc0b62c9c389 100644 --- a/be/src/cloud/cloud_cumulative_compaction.h +++ b/be/src/cloud/cloud_cumulative_compaction.h @@ -24,6 +24,7 @@ #include "olap/compaction.h" namespace doris { +#include "common/compile_check_begin.h" class CloudCumulativeCompaction : public CloudCompactionMixin { public: @@ -60,4 +61,5 @@ class CloudCumulativeCompaction : public CloudCompactionMixin { Version _last_delete_version {-1, -1}; }; +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 5a9879387b2327..6a259c78102cb3 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -31,6 +31,7 @@ #include "olap/tablet_meta.h" namespace doris { +#include "common/compile_check_begin.h" CloudSizeBasedCumulativeCompactionPolicy::CloudSizeBasedCumulativeCompactionPolicy( int64_t promotion_size, double promotion_ratio, int64_t promotion_min_size, @@ -377,4 +378,5 @@ int64_t CloudTimeSeriesCumulativeCompactionPolicy::new_cumulative_point( return output_rowset->end_version() + 1; } +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.h b/be/src/cloud/cloud_cumulative_compaction_policy.h index c142a8a6d3dffe..2680bc3eef0cbf 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.h +++ b/be/src/cloud/cloud_cumulative_compaction_policy.h @@ -30,6 +30,7 @@ #include "olap/rowset/rowset_meta.h" namespace doris { +#include "common/compile_check_begin.h" class Tablet; struct Version; @@ -115,4 +116,5 @@ class CloudTimeSeriesCumulativeCompactionPolicy : public CloudCumulativeCompacti bool allow_delete = false) override; }; +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/cloud/cloud_delete_bitmap_action.cpp index 86cc535e1bc88e..3d834bfe7b373c 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.cpp +++ b/be/src/cloud/cloud_delete_bitmap_action.cpp @@ -50,6 +50,7 @@ #include "util/stopwatch.hpp" namespace doris { +#include "common/compile_check_begin.h" using namespace ErrorCode; namespace { @@ -177,4 +178,5 @@ void CloudDeleteBitmapAction::handle(HttpRequest* req) { } } +#include "common/compile_check_end.h" } // namespace doris \ No newline at end of file diff --git a/be/src/cloud/cloud_delete_bitmap_action.h b/be/src/cloud/cloud_delete_bitmap_action.h index 35739a7373efc8..ce507ee9991757 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.h +++ b/be/src/cloud/cloud_delete_bitmap_action.h @@ -27,6 +27,7 @@ #include "olap/tablet.h" namespace doris { +#include "common/compile_check_begin.h" class HttpRequest; class ExecEnv; @@ -52,4 +53,5 @@ class CloudDeleteBitmapAction : public HttpHandlerWithAuth { CloudStorageEngine& _engine; DeleteBitmapActionType _delete_bitmap_action_type; }; +#include "common/compile_check_end.h" } // namespace doris \ No newline at end of file diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index 91611d20c6270b..a6fee5f1a690d1 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -34,6 +34,7 @@ #include "runtime/memory/mem_tracker_limiter.h" namespace doris { +#include "common/compile_check_begin.h" CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask( CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req, @@ -325,4 +326,5 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( return status; } +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 8e21498b0d873d..53787bc134e10a 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -64,6 +64,7 @@ #include "util/thrift_rpc_helper.h" namespace doris::cloud { +#include "common/compile_check_begin.h" using namespace ErrorCode; Status bthread_fork_join(const std::vector>& tasks, int concurrency) { @@ -1286,4 +1287,5 @@ int64_t CloudMetaMgr::get_inverted_index_file_szie(const RowsetMeta& rs_meta) { return total_inverted_index_size; } +#include "common/compile_check_end.h" } // namespace doris::cloud diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index c49b036ad90c15..913ef59489a1b3 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -27,6 +27,7 @@ #include "util/s3_util.h" namespace doris { +#include "common/compile_check_begin.h" class DeleteBitmap; class StreamLoadContext; @@ -124,4 +125,5 @@ class CloudMetaMgr { }; } // namespace cloud +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index dc6abbac31ba1b..992e3e97af0450 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -52,6 +52,7 @@ #include "util/parse_util.h" namespace doris { +#include "common/compile_check_begin.h" using namespace std::literals; @@ -857,4 +858,5 @@ std::shared_ptr CloudStorageEngine::cumu_compac return _cumulative_compaction_policies.at(compaction_policy); } +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index ea86f3b40ff1dc..c4b7daa1d64dc0 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -50,6 +50,7 @@ #include "vec/common/schema_util.h" namespace doris { +#include "common/compile_check_begin.h" using namespace ErrorCode; static constexpr int COMPACTION_DELETE_BITMAP_LOCK_ID = -1; @@ -924,4 +925,5 @@ void CloudTablet::build_tablet_report_info(TTabletInfo* tablet_info) { // but it may be used in the future. } +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index e95c295ae1daa5..f90bf536f63018 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -20,6 +20,7 @@ #include "common/status.h" namespace doris::config { +#include "common/compile_check_begin.h" DEFINE_String(deploy_mode, ""); DEFINE_mString(cloud_unique_id, ""); @@ -76,4 +77,5 @@ DEFINE_mInt32(tablet_txn_info_min_expired_seconds, "120"); DEFINE_mBool(enable_use_cloud_unique_id_from_fe, "true"); DEFINE_mBool(enable_cloud_tablet_report, "true"); +#include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index b345e6355921a4..a8a7c0c48ec91f 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -20,6 +20,7 @@ #include "common/config.h" namespace doris::config { +#include "common/compile_check_begin.h" DECLARE_String(deploy_mode); // deprecated do not configure directly @@ -110,4 +111,5 @@ DECLARE_mBool(enable_use_cloud_unique_id_from_fe); DECLARE_Bool(enable_cloud_tablet_report); +#include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/common/compile_check_begin.h b/be/src/common/compile_check_begin.h index 6da403f2894885..722b0dc7507e76 100644 --- a/be/src/common/compile_check_begin.h +++ b/be/src/common/compile_check_begin.h @@ -22,9 +22,9 @@ #define COMPILE_CHECK #ifdef __clang__ #pragma clang diagnostic push -#pragma clang diagnostic error "-Wconversion" +#pragma clang diagnostic warning "-Wconversion" #pragma clang diagnostic ignored "-Wsign-conversion" #pragma clang diagnostic ignored "-Wfloat-conversion" #endif -//#include "common/compile_check_begin.h" \ No newline at end of file +//#include "common/compile_check_begin.h" diff --git a/be/src/common/compile_check_end.h b/be/src/common/compile_check_end.h index 0897965dc74a3d..40df41b6bdfc6c 100644 --- a/be/src/common/compile_check_end.h +++ b/be/src/common/compile_check_end.h @@ -20,4 +20,4 @@ #endif #undef COMPILE_CHECK -// #include "common/compile_check_end.h" \ No newline at end of file +// #include "common/compile_check_end.h" diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 198b4e8595ed20..36affba74544c7 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -60,6 +60,7 @@ #include "vec/data_types/data_type_factory.hpp" namespace doris { +#include "common/compile_check_begin.h" using namespace ErrorCode; namespace { @@ -1111,4 +1112,5 @@ Status BetaRowsetWriter::flush_segment_writer_for_segcompaction( return Status::OK(); } +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 0570aff349c583..ad2d7393f1d982 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -57,6 +57,7 @@ using std::unordered_map; using std::vector; namespace doris { +#include "common/compile_check_begin.h" using namespace ErrorCode; TabletMetaSharedPtr TabletMeta::create( @@ -1312,4 +1313,5 @@ std::string tablet_state_name(TabletState state) { } } +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 34794ef6d0a760..4ab6950888c832 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -51,6 +51,7 @@ #include "util/uid_util.h" namespace json2pb { +#include "common/compile_check_begin.h" struct Pb2JsonOptions; } // namespace json2pb @@ -778,4 +779,5 @@ std::string tablet_state_name(TabletState state); bool operator==(const TabletMeta& a, const TabletMeta& b); bool operator!=(const TabletMeta& a, const TabletMeta& b); +#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 57600d1f56aae9..912f9f5ff403e7 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -80,6 +80,7 @@ #include "util/doris_metrics.h" namespace doris { +#include "common/compile_check_begin.h" namespace { std::shared_ptr get_rate_limit_group(event_base* event_base) { auto rate_limit = config::download_binlog_rate_limit_kbs; @@ -473,4 +474,5 @@ int HttpService::get_real_port() const { return _ev_http_server->get_real_port(); } +#include "common/compile_check_end.h" } // namespace doris diff --git a/cloud/src/common/compile_check_begin.h b/cloud/src/common/compile_check_begin.h new file mode 100644 index 00000000000000..722b0dc7507e76 --- /dev/null +++ b/cloud/src/common/compile_check_begin.h @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifdef COMPILE_CHECK +#error The handling of compile_check_begin.h and compile_check_end.h is not done correctly. +#endif + +#define COMPILE_CHECK +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic warning "-Wconversion" +#pragma clang diagnostic ignored "-Wsign-conversion" +#pragma clang diagnostic ignored "-Wfloat-conversion" +#endif + +//#include "common/compile_check_begin.h" diff --git a/cloud/src/common/compile_check_end.h b/cloud/src/common/compile_check_end.h new file mode 100644 index 00000000000000..40df41b6bdfc6c --- /dev/null +++ b/cloud/src/common/compile_check_end.h @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif +#undef COMPILE_CHECK + +// #include "common/compile_check_end.h" diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 9da5750d8d83f9..d2845f4344a732 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -67,6 +67,7 @@ using namespace std::chrono; namespace doris::cloud { +#include "common/compile_check_begin.h" MetaServiceImpl::MetaServiceImpl(std::shared_ptr txn_kv, std::shared_ptr resource_mgr, @@ -2398,4 +2399,5 @@ std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB& rowset return ret; } +#include "common/compile_check_end.h" } // namespace doris::cloud diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 7af96cbc14b8ee..7fa4992928677f 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -35,6 +35,7 @@ #include "resource-manager/resource_manager.h" namespace doris::cloud { +#include "common/compile_check_begin.h" class Transaction; @@ -781,4 +782,5 @@ class MetaServiceProxy final : public MetaService { std::unique_ptr impl_; }; +#include "common/compile_check_end.h" } // namespace doris::cloud diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 5355ac0eb61ed2..ae2963cac37104 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -38,6 +38,7 @@ #include "resource-manager/resource_manager.h" namespace doris::cloud { +#include "common/compile_check_begin.h" inline std::string md5(const std::string& str) { unsigned char digest[MD5_DIGEST_LENGTH]; MD5_CTX context; @@ -249,4 +250,5 @@ bool is_dropped_tablet(Transaction* txn, const std::string& instance_id, int64_t int64_t partition_id); std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB& rowset_meta); +#include "common/compile_check_end.h" } // namespace doris::cloud From 641731d06f7016b9f3a319eef2a8ddd5631c60fa Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Fri, 22 Nov 2024 10:35:23 +0800 Subject: [PATCH 2/5] edit --- be/src/common/compile_check_begin.h | 2 +- cloud/src/common/compile_check_begin.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/common/compile_check_begin.h b/be/src/common/compile_check_begin.h index 722b0dc7507e76..8f19b3242def18 100644 --- a/be/src/common/compile_check_begin.h +++ b/be/src/common/compile_check_begin.h @@ -22,7 +22,7 @@ #define COMPILE_CHECK #ifdef __clang__ #pragma clang diagnostic push -#pragma clang diagnostic warning "-Wconversion" +#pragma clang diagnostic error "-Wconversion" #pragma clang diagnostic ignored "-Wsign-conversion" #pragma clang diagnostic ignored "-Wfloat-conversion" #endif diff --git a/cloud/src/common/compile_check_begin.h b/cloud/src/common/compile_check_begin.h index 722b0dc7507e76..8f19b3242def18 100644 --- a/cloud/src/common/compile_check_begin.h +++ b/cloud/src/common/compile_check_begin.h @@ -22,7 +22,7 @@ #define COMPILE_CHECK #ifdef __clang__ #pragma clang diagnostic push -#pragma clang diagnostic warning "-Wconversion" +#pragma clang diagnostic error "-Wconversion" #pragma clang diagnostic ignored "-Wsign-conversion" #pragma clang diagnostic ignored "-Wfloat-conversion" #endif From c14db578c8844c6415341e804af29f7737e1594f Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Wed, 4 Dec 2024 17:20:29 +0800 Subject: [PATCH 3/5] edit --- be/src/cloud/cloud_cumulative_compaction.cpp | 6 ++-- .../cloud_cumulative_compaction_policy.cpp | 10 +++---- .../cloud_cumulative_compaction_policy.h | 6 ++-- be/src/cloud/cloud_meta_mgr.cpp | 2 +- be/src/cloud/cloud_storage_engine.cpp | 13 ++++---- be/src/cloud/cloud_tablet.cpp | 7 +++-- be/src/cloud/cloud_tablet.h | 2 +- be/src/olap/base_tablet.cpp | 2 +- be/src/olap/base_tablet.h | 2 +- be/src/olap/rowset/beta_rowset_writer.cpp | 14 ++++----- be/src/olap/rowset/beta_rowset_writer.h | 8 ++--- be/src/olap/rowset/rowset_writer.h | 2 +- be/src/olap/tablet_meta.cpp | 7 +++-- be/src/olap/tablet_meta.h | 6 ++-- be/src/util/hash_util.hpp | 2 +- cloud/src/common/compile_check_begin.h | 30 ------------------- cloud/src/common/compile_check_end.h | 23 -------------- 17 files changed, 48 insertions(+), 94 deletions(-) delete mode 100644 cloud/src/common/compile_check_begin.h delete mode 100644 cloud/src/common/compile_check_end.h diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index 72264178173a67..b28082268f92f3 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -491,8 +491,10 @@ Status CloudCumulativeCompaction::pick_rowsets_to_compact() { } int64_t max_score = config::cumulative_compaction_max_deltas; - auto process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage(); - bool memory_usage_high = process_memory_usage > MemInfo::soft_mem_limit() * 0.8; + auto process_memory_usage = + static_cast(doris::GlobalMemoryArbitrator::process_memory_usage()); + bool memory_usage_high = + process_memory_usage > static_cast(MemInfo::soft_mem_limit()) * 0.8; if (cloud_tablet()->last_compaction_status.is() || memory_usage_high) { max_score = std::max(config::cumulative_compaction_max_deltas / diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 6a259c78102cb3..6445b47fc5963f 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -49,7 +49,7 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::_level_size(const int64_t size return (int64_t)1 << (sizeof(size) * 8 - 1 - __builtin_clzl(size)); } -int32_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets( +int64_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets( CloudTablet* tablet, const std::vector& candidate_rowsets, const int64_t max_compaction_score, const int64_t min_compaction_score, std::vector* input_rowsets, Version* last_delete_version, @@ -115,8 +115,8 @@ int32_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets( size_t new_compaction_score = *compaction_score; while (rs_begin != input_rowsets->end()) { auto& rs_meta = (*rs_begin)->rowset_meta(); - int current_level = _level_size(rs_meta->total_disk_size()); - int remain_level = _level_size(total_size - rs_meta->total_disk_size()); + int64_t current_level = _level_size(rs_meta->total_disk_size()); + int64_t remain_level = _level_size(total_size - rs_meta->total_disk_size()); // if current level less then remain level, input rowsets contain current rowset // and process return; otherwise, input rowsets do not contain current rowset. if (current_level <= remain_level) { @@ -186,7 +186,7 @@ int32_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets( } int64_t CloudSizeBasedCumulativeCompactionPolicy::cloud_promotion_size(CloudTablet* t) const { - int64_t promotion_size = int64_t(t->base_size() * _promotion_ratio); + int64_t promotion_size = int64_t(static_cast(t->base_size()) * _promotion_ratio); // promotion_size is between _size_based_promotion_size and _size_based_promotion_min_size return promotion_size > _promotion_size ? _promotion_size : promotion_size < _promotion_min_size ? _promotion_min_size @@ -216,7 +216,7 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point( : last_cumulative_point; } -int32_t CloudTimeSeriesCumulativeCompactionPolicy::pick_input_rowsets( +int64_t CloudTimeSeriesCumulativeCompactionPolicy::pick_input_rowsets( CloudTablet* tablet, const std::vector& candidate_rowsets, const int64_t max_compaction_score, const int64_t min_compaction_score, std::vector* input_rowsets, Version* last_delete_version, diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.h b/be/src/cloud/cloud_cumulative_compaction_policy.h index 2680bc3eef0cbf..9373728547241b 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.h +++ b/be/src/cloud/cloud_cumulative_compaction_policy.h @@ -45,7 +45,7 @@ class CloudCumulativeCompactionPolicy { virtual int64_t new_compaction_level(const std::vector& input_rowsets) = 0; - virtual int32_t pick_input_rowsets(CloudTablet* tablet, + virtual int64_t pick_input_rowsets(CloudTablet* tablet, const std::vector& candidate_rowsets, const int64_t max_compaction_score, const int64_t min_compaction_score, @@ -72,7 +72,7 @@ class CloudSizeBasedCumulativeCompactionPolicy : public CloudCumulativeCompactio return 0; } - int32_t pick_input_rowsets(CloudTablet* tablet, + int64_t pick_input_rowsets(CloudTablet* tablet, const std::vector& candidate_rowsets, const int64_t max_compaction_score, const int64_t min_compaction_score, @@ -107,7 +107,7 @@ class CloudTimeSeriesCumulativeCompactionPolicy : public CloudCumulativeCompacti int64_t new_compaction_level(const std::vector& input_rowsets) override; - int32_t pick_input_rowsets(CloudTablet* tablet, + int64_t pick_input_rowsets(CloudTablet* tablet, const std::vector& candidate_rowsets, const int64_t max_compaction_score, const int64_t min_compaction_score, diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 53787bc134e10a..477cc802910869 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -718,7 +718,7 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_ "rowset_ids.size={},segment_ids.size={},vers.size={},delete_bitmaps.size={}", rowset_ids.size(), segment_ids.size(), vers.size(), delete_bitmaps.size()); } - for (size_t i = 0; i < rowset_ids.size(); i++) { + for (int i = 0; i < rowset_ids.size(); i++) { RowsetId rst_id; rst_id.init(rowset_ids[i]); delete_bitmap->merge( diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 992e3e97af0450..40a314f751a125 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -167,7 +167,8 @@ Status CloudStorageEngine::open() { _memtable_flush_executor = std::make_unique(); // Use file cache disks number - _memtable_flush_executor->init(io::FileCacheFactory::instance()->get_cache_instance_size()); + _memtable_flush_executor->init( + static_cast(io::FileCacheFactory::instance()->get_cache_instance_size())); _calc_delete_bitmap_executor = std::make_unique(); _calc_delete_bitmap_executor->init(); @@ -323,7 +324,8 @@ void CloudStorageEngine::_check_file_cache_ttl_block_valid() { int64_t ttl_seconds = tablet->tablet_meta()->ttl_seconds(); if (rowset->newest_write_timestamp() + ttl_seconds <= UnixSeconds()) continue; for (int64_t seg_id = 0; seg_id < rowset->num_segments(); seg_id++) { - auto hash = Segment::file_cache_key(rowset->rowset_id().to_string(), seg_id); + auto hash = Segment::file_cache_key(rowset->rowset_id().to_string(), + static_cast(seg_id)); auto* file_cache = io::FileCacheFactory::instance()->get_by_path(hash); file_cache->update_ttl_atime(hash); } @@ -545,7 +547,8 @@ std::vector CloudStorageEngine::_generate_cloud_compaction_task int num_cumu = std::accumulate(submitted_cumu_compactions.begin(), submitted_cumu_compactions.end(), 0, [](int a, auto& b) { return a + b.second.size(); }); - int num_base = submitted_base_compactions.size() + submitted_full_compactions.size(); + int num_base = + static_cast(submitted_base_compactions.size() + submitted_full_compactions.size()); int n = thread_per_disk - num_cumu - num_base; if (compaction_type == CompactionType::BASE_COMPACTION) { // We need to reserve at least one thread for cumulative compaction, @@ -823,7 +826,7 @@ Status CloudStorageEngine::get_compaction_status_json(std::string* result) { // cumu std::string_view cumu = "CumulativeCompaction"; rapidjson::Value cumu_key; - cumu_key.SetString(cumu.data(), cumu.length(), root.GetAllocator()); + cumu_key.SetString(cumu.data(), static_cast(cumu.length()), root.GetAllocator()); rapidjson::Document cumu_arr; cumu_arr.SetArray(); for (auto& [tablet_id, v] : _submitted_cumu_compactions) { @@ -835,7 +838,7 @@ Status CloudStorageEngine::get_compaction_status_json(std::string* result) { // base std::string_view base = "BaseCompaction"; rapidjson::Value base_key; - base_key.SetString(base.data(), base.length(), root.GetAllocator()); + base_key.SetString(base.data(), static_cast(base.length()), root.GetAllocator()); rapidjson::Document base_arr; base_arr.SetArray(); for (auto& [tablet_id, _] : _submitted_base_compactions) { diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index c4b7daa1d64dc0..99fdb8ddf1ce6b 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -381,7 +381,7 @@ void CloudTablet::delete_rowsets(const std::vector& to_delete, _tablet_meta->modify_rs_metas({}, rs_metas, false); } -int CloudTablet::delete_expired_stale_rowsets() { +uint64_t CloudTablet::delete_expired_stale_rowsets() { std::vector expired_rowsets; int64_t expired_stale_sweep_endtime = ::time(nullptr) - config::tablet_rowset_stale_sweep_time_sec; @@ -618,7 +618,8 @@ void CloudTablet::get_compaction_status(std::string* json_result) { } rapidjson::Value value; std::string version_str = rowset->get_rowset_info_str(); - value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator()); + value.SetString(version_str.c_str(), static_cast(version_str.length()), + versions_arr.GetAllocator()); versions_arr.PushBack(value, versions_arr.GetAllocator()); last_version = ver.second; } @@ -631,7 +632,7 @@ void CloudTablet::get_compaction_status(std::string* json_result) { for (auto& rowset : stale_rowsets) { rapidjson::Value value; std::string version_str = rowset->get_rowset_info_str(); - value.SetString(version_str.c_str(), version_str.length(), + value.SetString(version_str.c_str(), static_cast(version_str.length()), stale_versions_arr.GetAllocator()); stale_versions_arr.PushBack(value, stale_versions_arr.GetAllocator()); } diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 80038e569ba2fc..fc0d64a493d316 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -92,7 +92,7 @@ class CloudTablet final : public BaseTablet { void clear_cache() override; // Return number of deleted stale rowsets - int delete_expired_stale_rowsets(); + uint64_t delete_expired_stale_rowsets(); bool has_stale_rowsets() const { return !_stale_rs_version_map.empty(); } diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index a499a27b07f6e2..97f6f665c4a779 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -1762,7 +1762,7 @@ std::vector BaseTablet::get_snapshot_rowset(bool include_stale_ void BaseTablet::calc_consecutive_empty_rowsets( std::vector* empty_rowsets, - const std::vector& candidate_rowsets, int limit) { + const std::vector& candidate_rowsets, int64_t limit) { int len = cast_set(candidate_rowsets.size()); for (int i = 0; i < len - 1; ++i) { auto rowset = candidate_rowsets[i]; diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index f961f4c49eedd6..bd46cdbbe14854 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -276,7 +276,7 @@ class BaseTablet { // Find the first consecutive empty rowsets. output->size() >= limit void calc_consecutive_empty_rowsets(std::vector* empty_rowsets, const std::vector& candidate_rowsets, - int limit); + int64_t limit); // Return the merged schema of all rowsets virtual TabletSchemaSPtr merged_tablet_schema() const { return _max_version_schema; } diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 36affba74544c7..024a975e27e10d 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -390,7 +390,7 @@ Status BetaRowsetWriter::_find_longest_consecutive_small_segment( SegCompactionCandidatesSharedPtr& segments) { segments = std::make_shared(); // skip last (maybe active) segment - int32_t last_segment = _num_segment - 1; + int64_t last_segment = _num_segment - 1; size_t task_bytes = 0; uint32_t task_rows = 0; int32_t segid; @@ -476,15 +476,15 @@ Status BetaRowsetWriter::_rename_compacted_segments(int64_t begin, int64_t end) return Status::OK(); } -void BetaRowsetWriter::_clear_statistics_for_deleting_segments_unsafe(uint64_t begin, - uint64_t end) { +void BetaRowsetWriter::_clear_statistics_for_deleting_segments_unsafe(uint32_t begin, + uint32_t end) { VLOG_DEBUG << "_segid_statistics_map clear record segid range from:" << begin << " to:" << end; - for (int i = begin; i <= end; ++i) { + for (uint32_t i = begin; i <= end; ++i) { _segid_statistics_map.erase(i); } } -Status BetaRowsetWriter::_rename_compacted_segment_plain(uint64_t seg_id) { +Status BetaRowsetWriter::_rename_compacted_segment_plain(uint32_t seg_id) { if (seg_id == _num_segcompacted) { ++_num_segcompacted; return Status::OK(); @@ -1044,7 +1044,7 @@ Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStati if (segment_id >= _segment_num_rows.size()) { _segment_num_rows.resize(segment_id + 1); } - _segment_num_rows[segid_offset] = segstat.row_num; + _segment_num_rows[segid_offset] = static_cast(segstat.row_num); } VLOG_DEBUG << "_segid_statistics_map add new record. segment_id:" << segment_id << " row_num:" << segstat.row_num << " data_size:" << segstat.data_size @@ -1053,7 +1053,7 @@ Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStati { std::lock_guard lock(_segment_set_mutex); _segment_set.add(segid_offset); - while (_segment_set.contains(_num_segment)) { + while (_segment_set.contains(static_cast(_num_segment))) { _num_segment++; } } diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h index d96301af22630d..112d5a101ee921 100644 --- a/be/src/olap/rowset/beta_rowset_writer.h +++ b/be/src/olap/rowset/beta_rowset_writer.h @@ -173,7 +173,7 @@ class BaseBetaRowsetWriter : public RowsetWriter { int32_t allocate_segment_id() override { return _segment_creator.allocate_segment_id(); }; - void set_segment_start_id(int32_t start_id) override { + void set_segment_start_id(int64_t start_id) override { _segment_creator.set_segment_start_id(start_id); _segment_start_id = start_id; } @@ -226,7 +226,7 @@ class BaseBetaRowsetWriter : public RowsetWriter { return Status::OK(); } - std::atomic _num_segment; // number of consecutive flushed segments + std::atomic _num_segment; // number of consecutive flushed segments roaring::Roaring _segment_set; // bitmap set to record flushed segment id std::mutex _segment_set_mutex; // mutex for _segment_set int32_t _segment_start_id; // basic write start from 0, partial update may be different @@ -298,9 +298,9 @@ class BetaRowsetWriter : public BaseBetaRowsetWriter { Status _load_noncompacted_segment(segment_v2::SegmentSharedPtr& segment, int32_t segment_id); Status _find_longest_consecutive_small_segment(SegCompactionCandidatesSharedPtr& segments); Status _rename_compacted_segments(int64_t begin, int64_t end); - Status _rename_compacted_segment_plain(uint64_t seg_id); + Status _rename_compacted_segment_plain(uint32_t seg_id); Status _rename_compacted_indices(int64_t begin, int64_t end, uint64_t seg_id); - void _clear_statistics_for_deleting_segments_unsafe(uint64_t begin, uint64_t end); + void _clear_statistics_for_deleting_segments_unsafe(uint32_t begin, uint32_t end); StorageEngine& _engine; diff --git a/be/src/olap/rowset/rowset_writer.h b/be/src/olap/rowset/rowset_writer.h index f84ff964ea3051..d6594bfdf08007 100644 --- a/be/src/olap/rowset/rowset_writer.h +++ b/be/src/olap/rowset/rowset_writer.h @@ -170,7 +170,7 @@ class RowsetWriter { virtual int32_t allocate_segment_id() = 0; - virtual void set_segment_start_id(int num_segment) { LOG(FATAL) << "not supported!"; } + virtual void set_segment_start_id(int64_t num_segment) { LOG(FATAL) << "not supported!"; } virtual int64_t delete_bitmap_ns() { return 0; } diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index ad2d7393f1d982..655da615f95cc0 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -107,7 +107,7 @@ TabletMeta::TabletMeta() _delete_bitmap(new DeleteBitmap(_tablet_id)) {} TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, - int64_t replica_id, int32_t schema_hash, uint64_t shard_id, + int64_t replica_id, int32_t schema_hash, int32_t shard_id, const TTabletSchema& tablet_schema, uint32_t next_unique_id, const std::unordered_map& col_ordinal_to_unique_id, TabletUid tablet_uid, TTabletType::type tabletType, @@ -572,7 +572,8 @@ void TabletMeta::serialize(string* meta_binary) { Status TabletMeta::deserialize(std::string_view meta_binary) { TabletMetaPB tablet_meta_pb; - bool parsed = tablet_meta_pb.ParseFromArray(meta_binary.data(), meta_binary.size()); + bool parsed = tablet_meta_pb.ParseFromArray(meta_binary.data(), + static_cast(meta_binary.size())); if (!parsed) { return Status::Error("parse tablet meta failed"); } @@ -665,7 +666,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { int seg_maps_size = tablet_meta_pb.delete_bitmap().segment_delete_bitmaps_size(); CHECK(rst_ids_size == seg_ids_size && seg_ids_size == seg_maps_size && seg_maps_size == versions_size); - for (size_t i = 0; i < rst_ids_size; ++i) { + for (int i = 0; i < rst_ids_size; ++i) { RowsetId rst_id; rst_id.init(tablet_meta_pb.delete_bitmap().rowset_ids(i)); auto seg_id = tablet_meta_pb.delete_bitmap().segment_ids(i); diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 4ab6950888c832..e0337fb3628e20 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -101,7 +101,7 @@ class TabletMeta : public MetadataAdder { TabletMeta(); TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int64_t replica_id, - int32_t schema_hash, uint64_t shard_id, const TTabletSchema& tablet_schema, + int32_t schema_hash, int32_t shard_id, const TTabletSchema& tablet_schema, uint32_t next_unique_id, const std::unordered_map& col_ordinal_to_unique_id, TabletUid tablet_uid, TTabletType::type tabletType, @@ -157,7 +157,7 @@ class TabletMeta : public MetadataAdder { int64_t replica_id() const; void set_replica_id(int64_t replica_id) { _replica_id = replica_id; } int32_t schema_hash() const; - int16_t shard_id() const; + int32_t shard_id() const; void set_shard_id(int32_t shard_id); int64_t creation_time() const; void set_creation_time(int64_t creation_time); @@ -615,7 +615,7 @@ inline int32_t TabletMeta::schema_hash() const { return _schema_hash; } -inline int16_t TabletMeta::shard_id() const { +inline int32_t TabletMeta::shard_id() const { return _shard_id; } diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp index d444daa8c68d11..68845313bd49c2 100644 --- a/be/src/util/hash_util.hpp +++ b/be/src/util/hash_util.hpp @@ -305,7 +305,7 @@ class HashUtil { #endif } - static uint64_t hash64(const void* data, uint32_t bytes, uint64_t seed) { + static uint64_t hash64(const void* data, uint64_t bytes, uint64_t seed) { #ifdef _SSE4_2_ if (LIKELY(CpuInfo::is_supported(CpuInfo::SSE4_2))) { return crc_hash64(data, bytes, seed); diff --git a/cloud/src/common/compile_check_begin.h b/cloud/src/common/compile_check_begin.h deleted file mode 100644 index 8f19b3242def18..00000000000000 --- a/cloud/src/common/compile_check_begin.h +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifdef COMPILE_CHECK -#error The handling of compile_check_begin.h and compile_check_end.h is not done correctly. -#endif - -#define COMPILE_CHECK -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic error "-Wconversion" -#pragma clang diagnostic ignored "-Wsign-conversion" -#pragma clang diagnostic ignored "-Wfloat-conversion" -#endif - -//#include "common/compile_check_begin.h" diff --git a/cloud/src/common/compile_check_end.h b/cloud/src/common/compile_check_end.h deleted file mode 100644 index 40df41b6bdfc6c..00000000000000 --- a/cloud/src/common/compile_check_end.h +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif -#undef COMPILE_CHECK - -// #include "common/compile_check_end.h" From 17c0f0f59b42deedc68690337eae5b9a40d7a868 Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Wed, 4 Dec 2024 17:26:12 +0800 Subject: [PATCH 4/5] edit --- cloud/src/meta-service/meta_service.cpp | 2 -- cloud/src/meta-service/meta_service.h | 2 -- cloud/src/meta-service/meta_service_helper.h | 2 -- 3 files changed, 6 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index d2845f4344a732..9da5750d8d83f9 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -67,7 +67,6 @@ using namespace std::chrono; namespace doris::cloud { -#include "common/compile_check_begin.h" MetaServiceImpl::MetaServiceImpl(std::shared_ptr txn_kv, std::shared_ptr resource_mgr, @@ -2399,5 +2398,4 @@ std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB& rowset return ret; } -#include "common/compile_check_end.h" } // namespace doris::cloud diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 7fa4992928677f..7af96cbc14b8ee 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -35,7 +35,6 @@ #include "resource-manager/resource_manager.h" namespace doris::cloud { -#include "common/compile_check_begin.h" class Transaction; @@ -782,5 +781,4 @@ class MetaServiceProxy final : public MetaService { std::unique_ptr impl_; }; -#include "common/compile_check_end.h" } // namespace doris::cloud diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index ae2963cac37104..5355ac0eb61ed2 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -38,7 +38,6 @@ #include "resource-manager/resource_manager.h" namespace doris::cloud { -#include "common/compile_check_begin.h" inline std::string md5(const std::string& str) { unsigned char digest[MD5_DIGEST_LENGTH]; MD5_CTX context; @@ -250,5 +249,4 @@ bool is_dropped_tablet(Transaction* txn, const std::string& instance_id, int64_t int64_t partition_id); std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB& rowset_meta); -#include "common/compile_check_end.h" } // namespace doris::cloud From b5fdfa04d0227753e97e008d5cb04db59ccbd6de Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Thu, 5 Dec 2024 12:28:53 +0800 Subject: [PATCH 5/5] edit2 --- be/src/cloud/cloud_cumulative_compaction.cpp | 6 +++--- be/src/cloud/cloud_cumulative_compaction_policy.cpp | 2 +- be/src/cloud/cloud_storage_engine.cpp | 13 ++++++------- be/src/cloud/cloud_tablet.cpp | 6 +++--- be/src/olap/rowset/beta_rowset_writer.cpp | 8 ++++---- be/src/olap/rowset/beta_rowset_writer.h | 4 ++-- be/src/olap/rowset/rowset_writer.h | 2 +- 7 files changed, 20 insertions(+), 21 deletions(-) diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index b28082268f92f3..9aa448350d9b75 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -491,10 +491,10 @@ Status CloudCumulativeCompaction::pick_rowsets_to_compact() { } int64_t max_score = config::cumulative_compaction_max_deltas; - auto process_memory_usage = - static_cast(doris::GlobalMemoryArbitrator::process_memory_usage()); + double process_memory_usage = + cast_set(doris::GlobalMemoryArbitrator::process_memory_usage()); bool memory_usage_high = - process_memory_usage > static_cast(MemInfo::soft_mem_limit()) * 0.8; + process_memory_usage > cast_set(MemInfo::soft_mem_limit()) * 0.8; if (cloud_tablet()->last_compaction_status.is() || memory_usage_high) { max_score = std::max(config::cumulative_compaction_max_deltas / diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 6445b47fc5963f..92a47fcc69f8d7 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -186,7 +186,7 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets( } int64_t CloudSizeBasedCumulativeCompactionPolicy::cloud_promotion_size(CloudTablet* t) const { - int64_t promotion_size = int64_t(static_cast(t->base_size()) * _promotion_ratio); + int64_t promotion_size = int64_t(cast_set(t->base_size()) * _promotion_ratio); // promotion_size is between _size_based_promotion_size and _size_based_promotion_min_size return promotion_size > _promotion_size ? _promotion_size : promotion_size < _promotion_min_size ? _promotion_min_size diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 40a314f751a125..2944ccf21f276b 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -168,7 +168,7 @@ Status CloudStorageEngine::open() { _memtable_flush_executor = std::make_unique(); // Use file cache disks number _memtable_flush_executor->init( - static_cast(io::FileCacheFactory::instance()->get_cache_instance_size())); + cast_set(io::FileCacheFactory::instance()->get_cache_instance_size())); _calc_delete_bitmap_executor = std::make_unique(); _calc_delete_bitmap_executor->init(); @@ -323,9 +323,8 @@ void CloudStorageEngine::_check_file_cache_ttl_block_valid() { for (const auto& rowset : rowsets) { int64_t ttl_seconds = tablet->tablet_meta()->ttl_seconds(); if (rowset->newest_write_timestamp() + ttl_seconds <= UnixSeconds()) continue; - for (int64_t seg_id = 0; seg_id < rowset->num_segments(); seg_id++) { - auto hash = Segment::file_cache_key(rowset->rowset_id().to_string(), - static_cast(seg_id)); + for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); seg_id++) { + auto hash = Segment::file_cache_key(rowset->rowset_id().to_string(), seg_id); auto* file_cache = io::FileCacheFactory::instance()->get_by_path(hash); file_cache->update_ttl_atime(hash); } @@ -548,7 +547,7 @@ std::vector CloudStorageEngine::_generate_cloud_compaction_task std::accumulate(submitted_cumu_compactions.begin(), submitted_cumu_compactions.end(), 0, [](int a, auto& b) { return a + b.second.size(); }); int num_base = - static_cast(submitted_base_compactions.size() + submitted_full_compactions.size()); + cast_set(submitted_base_compactions.size() + submitted_full_compactions.size()); int n = thread_per_disk - num_cumu - num_base; if (compaction_type == CompactionType::BASE_COMPACTION) { // We need to reserve at least one thread for cumulative compaction, @@ -826,7 +825,7 @@ Status CloudStorageEngine::get_compaction_status_json(std::string* result) { // cumu std::string_view cumu = "CumulativeCompaction"; rapidjson::Value cumu_key; - cumu_key.SetString(cumu.data(), static_cast(cumu.length()), root.GetAllocator()); + cumu_key.SetString(cumu.data(), cast_set(cumu.length()), root.GetAllocator()); rapidjson::Document cumu_arr; cumu_arr.SetArray(); for (auto& [tablet_id, v] : _submitted_cumu_compactions) { @@ -838,7 +837,7 @@ Status CloudStorageEngine::get_compaction_status_json(std::string* result) { // base std::string_view base = "BaseCompaction"; rapidjson::Value base_key; - base_key.SetString(base.data(), static_cast(base.length()), root.GetAllocator()); + base_key.SetString(base.data(), cast_set(base.length()), root.GetAllocator()); rapidjson::Document base_arr; base_arr.SetArray(); for (auto& [tablet_id, _] : _submitted_base_compactions) { diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 99fdb8ddf1ce6b..0ee7a016b3a441 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -540,7 +540,7 @@ Result> CloudTablet::create_transient_rowset_write return RowsetFactory::create_rowset_writer(_engine, context, false) .transform([&](auto&& writer) { - writer->set_segment_start_id(rowset.num_segments()); + writer->set_segment_start_id(cast_set(rowset.num_segments())); return writer; }); } @@ -618,7 +618,7 @@ void CloudTablet::get_compaction_status(std::string* json_result) { } rapidjson::Value value; std::string version_str = rowset->get_rowset_info_str(); - value.SetString(version_str.c_str(), static_cast(version_str.length()), + value.SetString(version_str.c_str(), cast_set(version_str.length()), versions_arr.GetAllocator()); versions_arr.PushBack(value, versions_arr.GetAllocator()); last_version = ver.second; @@ -632,7 +632,7 @@ void CloudTablet::get_compaction_status(std::string* json_result) { for (auto& rowset : stale_rowsets) { rapidjson::Value value; std::string version_str = rowset->get_rowset_info_str(); - value.SetString(version_str.c_str(), static_cast(version_str.length()), + value.SetString(version_str.c_str(), cast_set(version_str.length()), stale_versions_arr.GetAllocator()); stale_versions_arr.PushBack(value, stale_versions_arr.GetAllocator()); } diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 024a975e27e10d..5f0603e0ce9742 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -390,7 +390,7 @@ Status BetaRowsetWriter::_find_longest_consecutive_small_segment( SegCompactionCandidatesSharedPtr& segments) { segments = std::make_shared(); // skip last (maybe active) segment - int64_t last_segment = _num_segment - 1; + int32_t last_segment = _num_segment - 1; size_t task_bytes = 0; uint32_t task_rows = 0; int32_t segid; @@ -654,7 +654,7 @@ Status BaseBetaRowsetWriter::add_rowset(RowsetSharedPtr rowset) { _num_rows_written += rowset->num_rows(); _total_data_size += rowset->rowset_meta()->data_disk_size(); _total_index_size += rowset->rowset_meta()->index_disk_size(); - _num_segment += rowset->num_segments(); + _num_segment += cast_set(rowset->num_segments()); // append key_bounds to current rowset RETURN_IF_ERROR(rowset->get_segments_key_bounds(&_segments_encoded_key_bounds)); @@ -1044,7 +1044,7 @@ Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStati if (segment_id >= _segment_num_rows.size()) { _segment_num_rows.resize(segment_id + 1); } - _segment_num_rows[segid_offset] = static_cast(segstat.row_num); + _segment_num_rows[segid_offset] = cast_set(segstat.row_num); } VLOG_DEBUG << "_segid_statistics_map add new record. segment_id:" << segment_id << " row_num:" << segstat.row_num << " data_size:" << segstat.data_size @@ -1053,7 +1053,7 @@ Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStati { std::lock_guard lock(_segment_set_mutex); _segment_set.add(segid_offset); - while (_segment_set.contains(static_cast(_num_segment))) { + while (_segment_set.contains(_num_segment)) { _num_segment++; } } diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h index 112d5a101ee921..a69d1063a55086 100644 --- a/be/src/olap/rowset/beta_rowset_writer.h +++ b/be/src/olap/rowset/beta_rowset_writer.h @@ -173,7 +173,7 @@ class BaseBetaRowsetWriter : public RowsetWriter { int32_t allocate_segment_id() override { return _segment_creator.allocate_segment_id(); }; - void set_segment_start_id(int64_t start_id) override { + void set_segment_start_id(int32_t start_id) override { _segment_creator.set_segment_start_id(start_id); _segment_start_id = start_id; } @@ -226,7 +226,7 @@ class BaseBetaRowsetWriter : public RowsetWriter { return Status::OK(); } - std::atomic _num_segment; // number of consecutive flushed segments + std::atomic _num_segment; // number of consecutive flushed segments roaring::Roaring _segment_set; // bitmap set to record flushed segment id std::mutex _segment_set_mutex; // mutex for _segment_set int32_t _segment_start_id; // basic write start from 0, partial update may be different diff --git a/be/src/olap/rowset/rowset_writer.h b/be/src/olap/rowset/rowset_writer.h index d6594bfdf08007..f84ff964ea3051 100644 --- a/be/src/olap/rowset/rowset_writer.h +++ b/be/src/olap/rowset/rowset_writer.h @@ -170,7 +170,7 @@ class RowsetWriter { virtual int32_t allocate_segment_id() = 0; - virtual void set_segment_start_id(int64_t num_segment) { LOG(FATAL) << "not supported!"; } + virtual void set_segment_start_id(int num_segment) { LOG(FATAL) << "not supported!"; } virtual int64_t delete_bitmap_ns() { return 0; }