From d05eb6db729a3fe8ebed20d31b751431aebf5461 Mon Sep 17 00:00:00 2001
From: yiguolei <guolei@selectdb.com>
Date: Mon, 3 Mar 2025 16:44:31 +0800
Subject: [PATCH] [enhancement](threadpool) reduce thread pool for arrow flight
 and spill io threads (#48530)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What problem does this PR solve?

Reduce these two pool size to allow BE start normally.

1. spill io threads could started as needed
2. arrow flight thread pool is async thread， should not use too much.



### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [ ] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 be/src/service/internal_service.cpp       | 2 +-
 be/src/vec/spill/spill_stream_manager.cpp | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index ff29a7de2dad80..8665fd310675e8 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -227,7 +227,7 @@ PInternalServiceImpl::PInternalServiceImpl(ExecEnv* exec_env)
                            "brpc_light"),
           _arrow_flight_work_pool(config::brpc_arrow_flight_work_pool_threads != -1
                                           ? config::brpc_arrow_flight_work_pool_threads
-                                          : std::max(512, CpuInfo::num_cores() * 16),
+                                          : std::max(512, CpuInfo::num_cores() * 2),
                                   config::brpc_arrow_flight_work_pool_max_queue_size != -1
                                           ? config::brpc_arrow_flight_work_pool_max_queue_size
                                           : std::max(20480, CpuInfo::num_cores() * 640),
diff --git a/be/src/vec/spill/spill_stream_manager.cpp b/be/src/vec/spill/spill_stream_manager.cpp
index 12a1e59037fe0d..9925b44e8899bd 100644
--- a/be/src/vec/spill/spill_stream_manager.cpp
+++ b/be/src/vec/spill/spill_stream_manager.cpp
@@ -81,8 +81,9 @@ Status SpillStreamManager::init() {
         }
         store->update_spill_data_usage(spill_data_size);
     }
+    // Reduce min threads to 1, to avoid occupy too many threads at start time.
     static_cast<void>(ThreadPoolBuilder("SpillIOThreadPool")
-                              .set_min_threads(config::spill_io_thread_pool_thread_num)
+                              .set_min_threads(1)
                               .set_max_threads(config::spill_io_thread_pool_thread_num)
                               .set_max_queue_size(config::spill_io_thread_pool_queue_size)
                               .build(&_spill_io_thread_pool));