Branch 2.0 var merge

.dockerignore

@@ -0,0 +1,7 @@
+be*
+fs_brokers*
+thirdparty*
+fe*
+.git*
+fs_brokers*
+

.github/workflows/auto_trigger_teamcity.yml

@@ -95,6 +95,7 @@ jobs:
 
           for pipeline in ${trigger_pipelines}
           do
+            same_build_sign="false"
             echo "-----------------------------------------------------------------"
             running_builds_command="curl -s -X GET ${teamcity_url}/app/rest/builds\?locator\=buildType\:${pipeline}\,branch:pull/${pull_request_num}\,running:true"
             echo "${running_builds_command}" 

.github/workflows/be-ut-mac.yml

@@ -29,7 +29,7 @@ concurrency:
 jobs:
   run-ut:
     name: BE UT (macOS)
-    runs-on: macos-12
+    runs-on: macos-13
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -81,7 +81,7 @@ jobs:
             'node'
             'llvm@16'
           )
-          brew install "${cellars[@]}"
+          brew install "${cellars[@]}" || true
 
           pushd thirdparty
           branch="${{ github.base_ref }}"

.github/workflows/build-extension.yml

@@ -100,6 +100,7 @@ jobs:
         run: |
           git clone https://github.com/apache/doris-website.git website
           cd website
+          echo "[\"current\"]" > versions.json 
           mkdir -p docs
           cp -R ../docs/en/docs/* docs/
           cp -R ../docs/sidebars.json sidebars.json

.github/workflows/code-checks.yml

@@ -49,7 +49,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           sh_checker_comment: true
-          sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene
+          sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest
 
   clang-tidy:
     name: "Clang Tidy"

.gitignore

@@ -1,4 +1,5 @@
 # ignore file with wildcard anywhere
+*.swo
 *.swp
 *.pyc
 *.class
@@ -20,6 +21,7 @@ dependency-reduced-pom.xml
 yarn.lock
 ut_dir
 package-lock.json
+__pycache__
 
 # ignore all sub `/bin` and `/build` directory
 **/build/
@@ -91,6 +93,7 @@ be/tags
 be/test/olap/test_data/tablet_meta_test.hdr
 be/.devcontainer/
 be/src/apache-orc/
+zoneinfo/
 
 ## tools
 tools/ssb-tools/ssb-data/

.licenserc.yaml

@@ -18,6 +18,8 @@ header:
     - ".gitmodules"
     - ".licenserc.yaml"
     - ".rat-excludes"
+    - ".github/**"
+    - ".dockerignore"
     - "be/src/common/status.cpp"
     - "be/src/common/status.h"
     - "be/src/env/env.h"
@@ -88,5 +90,8 @@ header:
     - "regression-test/ssl_default_certificate/*"
     - "extension/beats/go.mod"
     - "extension/beats/go.sum"
-
+    - "pytest/hdfs"
+    - "pytest/qe"
+    - "pytest/sys/data"
+    - "pytest/deploy/*.conf"
   comment: on-failure

be/CMakeLists.txt

@@ -259,7 +259,8 @@ if (COMPILER_GCC)
     add_compile_options(-fdiagnostics-color=always
                         -Wno-nonnull
                         -Wno-stringop-overread
-                        -Wno-stringop-overflow)
+                        -Wno-stringop-overflow
+                        -Wno-array-bounds)
 endif ()
 
 if (COMPILER_CLANG)
@@ -369,7 +370,7 @@ set(CXX_FLAGS_LSAN "${CXX_GCC_FLAGS} -O0 -fsanitize=leak -DLEAK_SANITIZER")
 
 # Set the flags to the undefined behavior sanitizer, also known as "ubsan"
 # Turn on sanitizer and debug symbols to get stack traces:
-set(CXX_FLAGS_UBSAN "${CXX_GCC_FLAGS} -O0 -fno-wrapv -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
+set(CXX_FLAGS_UBSAN "${CXX_GCC_FLAGS} -O0 -fno-wrapv -mcmodel=medium -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
 
 # Set the flags to the thread sanitizer, also known as "tsan"
 # Turn on sanitizer and debug symbols to get stack traces:
@@ -584,8 +585,14 @@ endif()
 if (absl_FOUND)
     set(COMMON_THIRDPARTY
         ${COMMON_THIRDPARTY}
-        absl::flat_hash_set
-        absl::str_format
+        absl::flags_internal
+        absl::flags_reflection
+        absl::random_internal_pool_urbg
+        absl::random_internal_randen
+        absl::spinlock_wait
+        absl::status
+        absl::statusor
+        absl::strings
     )
 endif()
 

be/cmake/thirdparty.cmake

@@ -283,6 +283,10 @@ set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/li
 add_library(gssapi_krb5 STATIC IMPORTED)
 set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libgssapi_krb5.a)
 
+add_library(dragonbox_to_chars STATIC IMPORTED)
+set_target_properties(dragonbox_to_chars PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libdragonbox_to_chars.a)
+target_include_directories(dragonbox_to_chars INTERFACE ${THIRDPARTY_DIR}/include/dragonbox-1.1.3)
+
 find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)
 
 if (OS_MACOSX)

be/src/agent/task_worker_pool.cpp

@@ -1524,18 +1524,22 @@ void PublishVersionTaskPool::_publish_version_worker_thread_callback() {
         DorisMetrics::instance()->publish_task_request_total->increment(1);
         VLOG_NOTICE << "get publish version task. signature=" << agent_task_req.signature;
 
-        std::vector<TTabletId> error_tablet_ids;
-        std::vector<TTabletId> succ_tablet_ids;
+        std::set<TTabletId> error_tablet_ids;
+        std::map<TTabletId, TVersion> succ_tablets;
         // partition_id, tablet_id, publish_version
         std::vector<std::tuple<int64_t, int64_t, int64_t>> discontinuous_version_tablets;
+        std::map<TTableId, int64_t> table_id_to_num_delta_rows;
         uint32_t retry_time = 0;
         Status status;
         bool is_task_timeout = false;
         while (retry_time < PUBLISH_VERSION_MAX_RETRY) {
+            succ_tablets.clear();
             error_tablet_ids.clear();
+            table_id_to_num_delta_rows.clear();
             EnginePublishVersionTask engine_task(publish_version_req, &error_tablet_ids,
-                                                 &succ_tablet_ids, &discontinuous_version_tablets);
-            status = _env->storage_engine()->execute_task(&engine_task);
+                                                 &succ_tablets, &discontinuous_version_tablets,
+                                                 &table_id_to_num_delta_rows);
+            status = StorageEngine::instance()->execute_task(&engine_task);
             if (status.ok()) {
                 break;
             } else if (status.is<PUBLISH_VERSION_NOT_CONTINUOUS>()) {
@@ -1583,25 +1587,22 @@ void PublishVersionTaskPool::_publish_version_worker_thread_callback() {
                     .tag("transaction_id", publish_version_req.transaction_id)
                     .tag("error_tablets_num", error_tablet_ids.size())
                     .error(status);
-            finish_task_request.__set_error_tablet_ids(error_tablet_ids);
         } else {
             if (!config::disable_auto_compaction &&
                 !MemInfo::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
-                for (int i = 0; i < succ_tablet_ids.size(); i++) {
+                for (auto [tablet_id, _] : succ_tablets) {
                     TabletSharedPtr tablet =
-                            StorageEngine::instance()->tablet_manager()->get_tablet(
-                                    succ_tablet_ids[i]);
+                            StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id);
                     if (tablet != nullptr) {
                         tablet->publised_count++;
                         if (tablet->publised_count % 10 == 0) {
                             StorageEngine::instance()->submit_compaction_task(
                                     tablet, CompactionType::CUMULATIVE_COMPACTION, true);
-                            LOG(INFO) << "trigger compaction succ, tabletid:" << succ_tablet_ids[i]
+                            LOG(INFO) << "trigger compaction succ, tablet_id:" << tablet_id
                                       << ", publised:" << tablet->publised_count;
                         }
                     } else {
-                        LOG(WARNING)
-                                << "trigger compaction failed, tabletid:" << succ_tablet_ids[i];
+                        LOG(WARNING) << "trigger compaction failed, tablet_id:" << tablet_id;
                     }
                 }
             }
@@ -1610,7 +1611,7 @@ void PublishVersionTaskPool::_publish_version_worker_thread_callback() {
             LOG_INFO("successfully publish version")
                     .tag("signature", agent_task_req.signature)
                     .tag("transaction_id", publish_version_req.transaction_id)
-                    .tag("tablets_num", succ_tablet_ids.size())
+                    .tag("tablets_num", succ_tablets.size())
                     .tag("cost(s)", cost_second);
         }
 
@@ -1619,8 +1620,10 @@ void PublishVersionTaskPool::_publish_version_worker_thread_callback() {
         finish_task_request.__set_task_type(agent_task_req.task_type);
         finish_task_request.__set_signature(agent_task_req.signature);
         finish_task_request.__set_report_version(_s_report_version);
-        finish_task_request.__set_error_tablet_ids(error_tablet_ids);
-
+        finish_task_request.__set_succ_tablets(succ_tablets);
+        finish_task_request.__set_error_tablet_ids(
+                std::vector<TTabletId>(error_tablet_ids.begin(), error_tablet_ids.end()));
+        finish_task_request.__set_table_id_to_delta_num_rows(table_id_to_num_delta_rows);
         _finish_task(finish_task_request);
         _remove_task_info(agent_task_req.task_type, agent_task_req.signature);
     }
@@ -1880,9 +1883,10 @@ void CloneTaskPool::_clone_worker_thread_callback() {
             LOG_INFO("successfully clone tablet")
                     .tag("signature", agent_task_req.signature)
                     .tag("tablet_id", clone_req.tablet_id);
+            ++_s_report_version;
             finish_task_request.__set_finish_tablet_infos(tablet_infos);
         }
-
+        finish_task_request.__set_report_version(_s_report_version);
         _finish_task(finish_task_request);
         _remove_task_info(agent_task_req.task_type, agent_task_req.signature);
     }
@@ -1921,6 +1925,10 @@ void StorageMediumMigrateTaskPool::_storage_medium_migrate_worker_thread_callbac
             EngineStorageMigrationTask engine_task(tablet, dest_store);
             status = _env->storage_engine()->execute_task(&engine_task);
         }
+        // fe should ignore this err
+        if (status.is<FILE_ALREADY_EXIST>()) {
+            status = Status::OK();
+        }
         if (!status.ok()) {
             LOG_WARNING("failed to migrate storage medium")
                     .tag("signature", agent_task_req.signature)
@@ -1983,8 +1991,9 @@ Status StorageMediumMigrateTaskPool::_check_migrate_request(const TStorageMedium
         *dest_store = stores[0];
     }
     if (tablet->data_dir()->path() == (*dest_store)->path()) {
-        return Status::InternalError("tablet is already on specified path {}",
-                                     tablet->data_dir()->path());
+        LOG_WARNING("tablet is already on specified path").tag("path", tablet->data_dir()->path());
+        return Status::Error<FILE_ALREADY_EXIST, false>("tablet is already on specified path: {}",
+                                                        tablet->data_dir()->path());
     }
 
     // check local disk capacity
@@ -1993,7 +2002,6 @@ Status StorageMediumMigrateTaskPool::_check_migrate_request(const TStorageMedium
         return Status::InternalError("reach the capacity limit of path {}, tablet_size={}",
                                      (*dest_store)->path(), tablet_size);
     }
-
     return Status::OK();
 }
 

be/src/common/config.cpp

@@ -130,9 +130,9 @@ DEFINE_mBool(enable_query_memory_overcommit, "true");
 
 DEFINE_mBool(disable_memory_gc, "false");
 
-DEFINE_mInt64(large_memory_check_bytes, "1073741824");
+DEFINE_mInt64(large_memory_check_bytes, "2147483648");
 
-// The maximum time a thread waits for a full GC. Currently only query will wait for full gc.
+// The maximum time a thread waits for full GC. Currently only query will wait for full gc.
 DEFINE_mInt32(thread_wait_gc_max_milliseconds, "1000");
 
 DEFINE_mInt64(pre_serialize_keys_limit_bytes, "16777216");
@@ -302,6 +302,7 @@ DEFINE_mInt32(trash_file_expire_time_sec, "259200");
 // minimum file descriptor number
 // modify them upon necessity
 DEFINE_Int32(min_file_descriptor_number, "60000");
+DEFINE_mBool(disable_segment_cache, "false");
 DEFINE_Int64(index_stream_cache_capacity, "10737418240");
 DEFINE_String(row_cache_mem_limit, "20%");
 
@@ -316,7 +317,7 @@ DEFINE_Int32(index_page_cache_percentage, "10");
 // whether to disable page cache feature in storage
 DEFINE_Bool(disable_storage_page_cache, "false");
 // whether to disable row cache feature in storage
-DEFINE_Bool(disable_storage_row_cache, "true");
+DEFINE_mBool(disable_storage_row_cache, "true");
 // whether to disable pk page cache feature in storage
 DEFINE_Bool(disable_pk_storage_page_cache, "false");
 
@@ -375,6 +376,12 @@ DEFINE_mDouble(compaction_promotion_ratio, "0.05");
 // rowset will be not given to base compaction. The unit is m byte.
 DEFINE_mInt64(compaction_promotion_min_size_mbytes, "128");
 
+// When output rowset of cumulative compaction total version count (end_version - start_version)
+// exceed this config count, the rowset will be moved to base compaction
+// NOTE: this config will work for unique key merge-on-write table only, to reduce version count
+// related cost on delete bitmap more effectively.
+DEFINE_mInt64(compaction_promotion_version_count, "1000");
+
 // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
 // this size, size_based policy may not do to cumulative compaction. The unit is m byte.
 DEFINE_mInt64(compaction_min_size_mbytes, "64");
@@ -707,7 +714,8 @@ DEFINE_mInt32(zone_map_row_num_threshold, "20");
 //    Info = 4,
 //    Debug = 5,
 //    Trace = 6
-DEFINE_Int32(aws_log_level, "3");
+// Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs
+DEFINE_Int32(aws_log_level, "0");
 
 // the buffer size when read data from remote storage like s3
 DEFINE_mInt32(remote_storage_read_buffer_mb, "16");
@@ -892,7 +900,7 @@ DEFINE_mInt64(file_cache_max_size_per_disk, "0"); // zero for no limit
 DEFINE_Int32(s3_transfer_executor_pool_size, "2");
 
 DEFINE_Bool(enable_time_lut, "true");
-DEFINE_Bool(enable_simdjson_reader, "true");
+DEFINE_mBool(enable_simdjson_reader, "true");
 
 DEFINE_mBool(enable_query_like_bloom_filter, "true");
 // number of s3 scanner thread pool size
@@ -944,8 +952,10 @@ DEFINE_Bool(enable_java_support, "true");
 // Set config randomly to check more issues in github workflow
 DEFINE_Bool(enable_fuzzy_mode, "false");
 
+DEFINE_Bool(enable_debug_points, "false");
+
 DEFINE_Int32(pipeline_executor_size, "0");
-DEFINE_mInt16(pipeline_short_query_timeout_s, "20");
+DEFINE_mBool(enable_workload_group_for_scan, "false");
 
 // Temp config. True to use optimization for bitmap_index apply predicate except leaf node of the and node.
 // Will remove after fully test.
@@ -1046,6 +1056,7 @@ DEFINE_Bool(allow_invalid_decimalv2_literal, "false");
 DEFINE_mInt64(kerberos_expiration_time_seconds, "43200");
 
 DEFINE_mString(get_stack_trace_tool, "libunwind");
+DEFINE_mString(dwarf_location_info_mode, "FAST");
 
 // the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer
 DEFINE_mInt64(auto_inc_prefetch_size_ratio, "10");
@@ -1061,10 +1072,6 @@ DEFINE_mInt64(lookup_connection_cache_bytes_limit, "4294967296");
 // level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
 DEFINE_mInt64(LZ4_HC_compression_level, "9");
 
-DEFINE_Bool(enable_hdfs_hedged_read, "false");
-DEFINE_Int32(hdfs_hedged_read_thread_num, "128");
-DEFINE_Int32(hdfs_hedged_read_threshold_time, "500");
-
 DEFINE_mBool(enable_merge_on_write_correctness_check, "true");
 
 // The secure path with user files, used in the `local` table function.
@@ -1076,11 +1083,25 @@ DEFINE_Int16(bitmap_serialize_version, "1");
 DEFINE_Int32(group_commit_insert_threads, "10");
 
 DEFINE_mInt32(scan_thread_nice_value, "0");
+
 DEFINE_mInt32(tablet_schema_cache_recycle_interval, "86400");
 
 DEFINE_Bool(exit_on_exception, "false");
 
-DEFINE_Bool(enable_scan_thread_low_thread_priority, "false");
+DEFINE_Bool(ignore_always_true_predicate_for_segment, "true");
+
+// Dir of default timezone files
+DEFINE_String(default_tzfiles_path, "${DORIS_HOME}/zoneinfo");
+
+// the max package bytes be thrift server can receive
+// avoid accepting error or too large package causing OOM,default 20000000(20M)
+DEFINE_Int32(be_thrift_max_pkg_bytes, "20000000");
+
+// Ingest binlog work pool size, -1 is disable, 0 is hardware concurrency
+DEFINE_Int32(ingest_binlog_work_pool_size, "-1");
+
+// Download binlog rate limit, unit is KB/s, 0 means no limit
+DEFINE_Int32(download_binlog_rate_limit_kbs, "0");
 
 DEFINE_mInt64(threshold_rows_to_estimate_sparse_column, "1000");
 // clang-format off
@@ -1501,7 +1522,6 @@ void set_fuzzy_configs() {
     // random value true or false
     set_fuzzy_config("disable_storage_page_cache", ((rand() % 2) == 0) ? "true" : "false");
     set_fuzzy_config("enable_system_metrics", ((rand() % 2) == 0) ? "true" : "false");
-    set_fuzzy_config("enable_simdjson_reader", ((rand() % 2) == 0) ? "true" : "false");
     // random value from 8 to 48
     // s = set_fuzzy_config("doris_scanner_thread_pool_thread_num", std::to_string((rand() % 41) + 8));
     // LOG(INFO) << s.to_string();